diff --git a/4b284b12bc4opt2/3494406.err b/4b284b12bc4opt2/3494406.err new file mode 100644 index 0000000000000000000000000000000000000000..5140ff781af1b5a4809a6629f4ee5cb3e5b488e8 --- /dev/null +++ b/4b284b12bc4opt2/3494406.err @@ -0,0 +1,3642 @@ + 9: 2023-05-11 00:13:21.402682: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-11 00:13:21.402699: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-11 00:13:21.402701: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-11 00:13:21.402755: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-11 00:13:21.402777: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-11 00:13:21.402789: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-11 00:13:21.402753: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-11 00:13:21.402819: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-11 00:13:21.403432: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-11 00:13:21.403450: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-11 00:13:21.403464: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: 2023-05-11 00:13:21.403708: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-11 00:13:21.403723: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-11 00:13:21.403733: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: 2023-05-11 00:13:21.403526: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-11 00:13:21.403558: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-11 00:13:21.403574: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: 2023-05-11 00:13:21.403505: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-11 00:13:21.403538: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-11 00:13:21.403565: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-11 00:13:21.403505: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-11 00:13:21.403513: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-11 00:13:21.403750: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-11 00:13:21.403773: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-11 00:13:21.403752: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: 2023-05-11 00:13:21.403625: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-11 00:13:21.403644: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-11 00:13:21.403648: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-11 00:13:21.403585: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-11 00:13:21.403614: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-11 00:13:21.403585: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-11 00:13:21.403607: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-11 00:13:21.403594: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: 2023-05-11 00:13:21.403526: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-11 00:13:21.403555: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-11 00:13:21.403895: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-11 00:13:21.403891: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-11 00:13:21.403891: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-11 00:13:21.403681: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-11 00:13:21.403700: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-11 00:13:21.403826: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-11 00:13:21.403850: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-11 00:13:21.403826: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-11 00:13:21.403750: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-11 00:13:21.403763: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-11 00:13:21.403801: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-11 00:13:21.403804: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-11 00:13:21.403854: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-11 00:13:21.403602: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-11 00:13:21.403606: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-11 00:13:21.403635: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-11 00:13:21.403637: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-11 00:13:21.403669: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-11 00:13:21.403665: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-11 00:13:21.403594: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-11 00:13:21.403910: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-11 00:13:21.403915: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-11 00:13:21.403952: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-11 00:13:21.404013: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-11 00:13:21.404032: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-11 00:13:21.403794: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-11 00:13:21.403815: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-11 00:13:21.403686: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-11 00:13:21.403683: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-11 00:13:21.403752: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-11 00:13:21.403901: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-11 00:13:21.403998: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-11 00:13:21.404038: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-11 00:13:21.403777: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-11 00:13:21.403615: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-11 00:13:21.403596: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-11 00:13:21.403643: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: 2023-05-11 00:13:21.403937: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-11 00:13:21.403978: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-11 00:13:21.403994: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: 2023-05-11 00:13:21.403670: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-11 00:13:21.403681: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-11 00:13:21.403799: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-11 00:13:21.403822: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: 2023-05-11 00:13:21.404059: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-11 00:13:21.404075: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-11 00:13:21.404044: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-11 00:13:21.404052: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-11 00:13:21.404104: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: 2023-05-11 00:13:21.403939: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-11 00:13:21.403945: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-11 00:13:21.403950: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-11 00:13:21.404037: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-11 00:13:21.404048: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-11 00:13:21.403814: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-11 00:13:21.403920: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-11 00:13:21.403938: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-11 00:13:21.403939: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: 2023-05-11 00:13:21.404062: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-11 00:13:21.403795: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-11 00:13:21.404072: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-11 00:13:21.404069: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-11 00:13:21.404113: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-11 00:13:21.404124: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-11 00:13:21.404129: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-11 00:13:21.404062: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-11 00:13:21.404091: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-11 00:13:21.403829: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-11 00:13:21.403838: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-11 00:13:21.404042: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-11 00:13:21.404078: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-11 00:13:21.403867: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-11 00:13:21.404024: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-11 00:13:21.404057: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-11 00:13:21.404080: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: 2023-05-11 00:13:21.403884: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-11 00:13:21.403894: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-11 00:13:21.403897: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-11 00:13:21.404009: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-11 00:13:21.404209: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-11 00:13:21.404220: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-11 00:13:21.404248: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: 2023-05-11 00:13:21.404248: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-11 00:13:21.404251: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-11 00:13:21.404271: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: 2023-05-11 00:13:21.404177: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-11 00:13:21.404194: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-11 00:13:21.404196: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: 2023-05-11 00:13:21.404138: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-11 00:13:21.404102: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-11 00:13:21.404139: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-11 00:13:21.404308: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-11 00:13:21.404320: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-11 00:13:21.404304: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: 2023-05-11 00:13:21.403987: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-11 00:13:21.404102: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-11 00:13:21.404111: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-11 00:13:21.404082: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-11 00:13:21.404125: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-11 00:13:21.404311: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-11 00:13:21.404343: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-11 00:13:21.404314: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-11 00:13:21.404104: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-11 00:13:21.404132: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-11 00:13:21.403911: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-11 00:13:21.403919: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-11 00:13:21.404338: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-11 00:13:21.404382: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-11 00:13:21.404411: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: 2023-05-11 00:13:21.404013: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-11 00:13:21.404028: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-11 00:13:21.404040: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-11 00:13:21.404269: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-11 00:13:21.404281: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-11 00:13:21.404198: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-11 00:13:21.404204: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-11 00:13:21.404435: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-11 00:13:21.404454: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-11 00:13:21.404456: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-11 00:13:21.404436: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-11 00:13:21.404489: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-11 00:13:21.404518: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: 2023-05-11 00:13:21.404078: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-11 00:13:21.404384: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-11 00:13:21.404425: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-11 00:13:21.404147: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-11 00:13:21.404167: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-11 00:13:21.404205: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-11 00:13:21.404368: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-11 00:13:21.404384: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-11 00:13:21.404111: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-11 00:13:21.404147: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-11 00:13:21.404046: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-11 00:13:21.404299: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-11 00:13:21.404313: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-11 00:13:21.404325: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-11 00:13:21.404278: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-11 00:13:21.404300: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-11 00:13:21.404285: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-11 00:13:21.404295: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-11 00:13:21.404385: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-11 00:13:21.404396: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-11 00:13:21.404410: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-11 00:13:21.404394: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-11 00:13:21.404155: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-11 00:13:21.404414: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-11 00:13:21.404418: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-11 00:13:21.404434: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: 2023-05-11 00:13:21.404485: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-11 00:13:21.404517: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-11 00:13:21.404546: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-11 00:13:21.404304: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-11 00:13:21.404315: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-11 00:13:21.404381: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-11 00:13:21.404524: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-11 00:13:21.404550: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-11 00:13:21.404559: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: 2023-05-11 00:13:21.404438: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-11 00:13:21.404412: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-11 00:13:21.404347: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-11 00:13:21.404428: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-11 00:13:21.404568: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-11 00:13:21.404470: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-11 00:13:21.404426: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-11 00:13:21.404475: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: 2023-05-11 00:13:21.404513: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-11 00:13:21.404517: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-11 00:13:21.404555: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-11 00:13:21.404567: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-11 00:13:21.404579: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: 2023-05-11 00:13:21.404587: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-11 00:13:21.404479: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-11 00:13:21.404496: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-11 00:13:21.404511: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-11 00:13:21.404557: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-11 00:13:21.404566: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-11 00:13:21.404616: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-11 00:13:21.404622: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-11 00:13:21.404513: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-11 00:13:21.404519: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-11 00:13:21.404528: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: 2023-05-11 00:13:21.404585: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-11 00:13:21.404635: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-11 00:13:21.404533: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-11 00:13:21.404679: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-11 00:13:21.404690: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-11 00:13:21.404651: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-11 00:13:21.404658: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-11 00:13:21.404590: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-11 00:13:21.405265: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-11 00:13:21.405277: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-11 00:13:21.405273: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: 2023-05-11 00:13:21.405247: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-11 00:13:21.405246: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-11 00:13:21.405276: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-11 00:13:21.405281: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-11 00:13:21.405261: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-11 00:13:21.405325: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-11 00:13:21.405332: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-11 00:13:21.405318: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: 2023-05-11 00:13:21.405291: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-11 00:13:21.405295: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-11 00:13:21.405300: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-11 00:13:21.405353: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-11 00:13:21.405366: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-11 00:13:21.405353: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-11 00:13:21.405288: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-11 00:13:21.405278: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-11 00:13:21.405380: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-11 00:13:21.405381: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-11 00:13:21.405347: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-11 00:13:21.405360: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-11 00:13:21.405362: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-11 00:13:21.405473: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-11 00:13:21.405487: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-11 00:13:21.405532: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-11 00:13:21.405542: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-11 00:13:21.405557: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-11 00:13:21.405571: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-11 00:13:21.405542: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-11 00:13:21.405591: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-11 00:13:38.838154: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:13:38.838169: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:13:38.838152: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:13:38.838180: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:13:38.838173: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:13:38.838199: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:13:38.838184: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:13:38.838197: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:13:38.838201: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:13:38.838181: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:13:38.838204: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:13:38.838210: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:13:38.838214: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:13:38.838215: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:13:38.838222: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:13:38.838218: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:13:38.839020: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-11 00:13:38.839037: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-11 00:13:38.839049: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-11 00:13:38.839057: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-11 00:13:38.839066: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-11 00:13:38.839080: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-11 00:13:38.839050: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-11 00:13:38.839071: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-11 00:13:38.839083: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-11 00:13:38.839085: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-11 00:13:38.839082: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-11 00:13:38.839105: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-11 00:13:38.839110: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-11 00:13:38.839118: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-11 00:13:38.839122: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-11 00:13:38.839132: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-11 00:13:38.838880: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-11 00:13:38.838902: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-11 00:13:38.838914: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-11 00:13:38.838955: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-11 00:13:38.838966: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-11 00:13:38.838983: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-11 00:13:38.838989: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-11 00:13:38.838994: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-11 00:13:38.839585: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-11 00:13:38.839604: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-11 00:13:38.839618: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-11 00:13:38.839628: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-11 00:13:38.839642: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-11 00:13:38.839651: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-11 00:13:38.839653: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-11 00:13:38.839659: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-11 00:13:38.839498: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-11 00:13:38.839520: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-11 00:13:38.839529: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-11 00:13:38.839548: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-11 00:13:38.839550: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-11 00:13:38.839560: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-11 00:13:38.839555: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-11 00:13:38.839566: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-11 00:13:38.840008: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-11 00:13:38.840010: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-11 00:13:38.840037: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-11 00:13:38.840037: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-11 00:13:38.840035: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-11 00:13:38.840059: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-11 00:13:38.840062: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-11 00:13:38.840064: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-11 00:13:38.840115: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-11 00:13:38.840135: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-11 00:13:38.840152: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-11 00:13:38.840158: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-11 00:13:38.840166: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-11 00:13:38.840178: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-11 00:13:38.840191: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-11 00:13:38.840191: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-11 00:13:38.840660: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-11 00:13:38.840682: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-11 00:13:38.840691: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-11 00:13:38.840709: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-11 00:13:38.840722: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-11 00:13:38.840728: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-11 00:13:38.840735: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-11 00:13:38.840747: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-11 00:13:38.841133: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:13:38.841177: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:13:38.841240: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:13:38.841192: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:13:38.841241: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:13:38.841256: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:13:38.841286: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:13:38.841244: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:13:38.841810: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-11 00:13:38.841831: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-11 00:13:38.841837: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-11 00:13:38.841857: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-11 00:13:38.841870: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-11 00:13:38.841878: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-11 00:13:38.841882: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-11 00:13:38.841891: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-11 00:13:38.841943: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:13:38.841958: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:13:38.841983: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:13:38.841935: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:13:38.841988: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:13:38.842039: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:13:38.841968: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:13:38.841963: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:13:38.841997: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:13:38.841997: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:13:38.842000: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:13:38.842016: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:13:38.842121: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:13:38.842001: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:13:38.842051: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:13:38.842032: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:13:38.842065: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:13:38.842027: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:13:38.842057: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:13:38.842138: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:13:38.842113: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:13:38.842046: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:13:38.842150: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:13:38.842074: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:13:38.842055: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:13:38.842054: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:13:38.842175: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:13:38.842103: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:13:38.842081: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:13:38.842077: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:13:38.842171: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:13:38.842092: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:13:38.842089: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:13:38.842102: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:13:38.842176: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:13:38.842115: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:13:38.842106: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:13:38.842185: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:13:38.842109: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:13:38.842190: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:13:38.842127: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:13:38.842119: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:13:38.842137: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:13:38.842139: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:13:38.842138: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:13:38.842182: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:13:38.842135: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:13:38.842201: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:13:38.842152: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:13:38.842249: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:13:38.842175: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:13:38.842256: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:13:38.842160: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:13:38.842229: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:13:38.842166: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:13:38.842267: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:13:38.842901: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-11 00:13:38.842920: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-11 00:13:38.842934: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-11 00:13:38.842951: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-11 00:13:38.842954: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-11 00:13:38.842974: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-11 00:13:38.842932: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-11 00:13:38.842941: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-11 00:13:38.842971: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-11 00:13:38.842979: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-11 00:13:38.842956: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-11 00:13:38.842988: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-11 00:13:38.842994: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-11 00:13:38.842962: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-11 00:13:38.842988: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-11 00:13:38.843002: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-11 00:13:38.843023: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-11 00:13:38.843042: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-11 00:13:38.842975: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-11 00:13:38.842976: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-11 00:13:38.842994: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-11 00:13:38.843002: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-11 00:13:38.843008: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-11 00:13:38.843006: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-11 00:13:38.843053: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-11 00:13:38.843014: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-11 00:13:38.843039: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-11 00:13:38.843056: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-11 00:13:38.843071: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-11 00:13:38.843067: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-11 00:13:38.843020: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-11 00:13:38.843083: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-11 00:13:38.843066: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-11 00:13:38.843076: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-11 00:13:38.843081: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-11 00:13:38.843101: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-11 00:13:38.843099: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-11 00:13:38.843102: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-11 00:13:38.843085: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-11 00:13:38.843108: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-11 00:13:38.843083: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-11 00:13:38.843084: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-11 00:13:38.843094: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-11 00:13:38.843104: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-11 00:13:38.843118: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-11 00:13:38.843121: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-11 00:13:38.843138: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-11 00:13:38.843145: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-11 00:13:38.843149: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-11 00:13:38.843158: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-11 00:13:38.843161: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-11 00:13:38.843146: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-11 00:13:38.843159: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-11 00:13:38.843162: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-11 00:13:38.843162: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-11 00:13:38.843168: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-11 00:13:38.843281: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:13:38.843285: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:13:38.843305: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:13:38.843312: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:13:38.843423: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:13:38.843322: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:13:38.843308: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:13:38.843332: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:13:38.843856: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-11 00:13:38.843863: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-11 00:13:38.843883: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-11 00:13:38.843890: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-11 00:13:38.843909: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-11 00:13:38.843907: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-11 00:13:38.843909: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-11 00:13:38.843920: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-11 00:13:38.844057: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:13:38.844079: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:13:38.844099: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:13:38.844110: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:13:38.844142: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:13:38.844126: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:13:38.844150: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:13:38.844158: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:13:38.844607: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-11 00:13:38.844624: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-11 00:13:38.844634: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-11 00:13:38.844645: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-11 00:13:38.844652: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-11 00:13:38.844657: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-11 00:13:38.844661: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-11 00:13:38.844664: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-11 00:13:38.844711: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:13:38.844736: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:13:38.844742: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:13:38.844770: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:13:38.844781: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:13:38.844782: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:13:38.844792: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:13:38.844793: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:13:38.845225: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-11 00:13:38.845245: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-11 00:13:38.845256: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-11 00:13:38.845276: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-11 00:13:38.845281: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-11 00:13:38.845291: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-11 00:13:38.845295: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-11 00:13:38.845304: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-11 00:13:38.845066: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:13:38.844974: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:13:38.845068: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:13:38.844985: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:13:38.845100: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:13:38.844998: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:13:38.845092: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:13:38.845007: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:13:38.845095: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:13:38.845014: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:13:38.845098: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:13:38.845015: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:13:38.845117: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:13:38.845030: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:13:38.845120: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:13:38.845038: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:13:38.845642: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-11 00:13:38.845662: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-11 00:13:38.845619: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-11 00:13:38.845649: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-11 00:13:38.845666: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-11 00:13:38.845660: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-11 00:13:38.845660: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-11 00:13:38.845681: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-11 00:13:38.845683: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-11 00:13:38.845669: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-11 00:13:38.845694: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-11 00:13:38.845683: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-11 00:13:38.845692: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-11 00:13:38.845705: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-11 00:13:38.845711: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-11 00:13:38.845720: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-11 00:13:38.845820: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:13:38.845876: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-11 00:13:38.845850: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-11 00:13:38.845838: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-11 00:13:38.845859: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-11 00:13:38.845865: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-11 00:13:38.845877: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-11 00:13:38.845875: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:13:38.845904: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-11 00:13:38.845862: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:13:38.845923: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:13:38.845942: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:13:38.845934: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-11 00:13:38.846375: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-11 00:13:38.846395: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-11 00:13:38.845932: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:13:38.845954: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-11 00:13:38.846407: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-11 00:13:38.845957: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-11 00:13:38.846424: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-11 00:13:38.846426: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-11 00:13:38.846446: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-11 00:13:38.846438: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-11 00:13:38.846440: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-11 00:13:38.846442: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-11 00:13:38.846466: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-11 00:13:38.846481: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-11 00:13:38.846494: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-11 00:13:38.846506: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-11 00:13:38.846516: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-11 00:13:38.846526: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-11 00:13:38.846533: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-11 00:13:38.846949: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:13:38.846999: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:13:38.847011: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:13:38.847015: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:13:38.847025: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:13:38.847006: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:13:38.847034: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:13:38.847037: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:13:38.847056: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:13:38.847052: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:13:38.847060: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:13:38.847088: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:13:38.847104: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:13:38.847119: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:13:38.847648: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-11 00:13:38.847685: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-11 00:13:38.847382: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:13:38.847701: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-11 00:13:38.847404: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:13:38.847716: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-11 00:13:38.847410: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:13:38.847723: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-11 00:13:38.847736: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-11 00:13:38.847437: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:13:38.847736: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-11 00:13:38.847453: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:13:38.847748: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-11 00:13:38.847444: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-11 00:13:38.847465: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:13:38.847135: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-11 00:13:38.847462: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:13:38.847103: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:13:38.847681: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-11 00:13:38.847703: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-11 00:13:38.847715: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-11 00:13:38.847861: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-11 00:13:38.847728: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-11 00:13:38.847733: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-11 00:13:38.847548: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:13:38.847743: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-11 00:13:38.847880: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-11 00:13:38.847889: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-11 00:13:38.847876: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-11 00:13:38.847879: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-11 00:13:38.847569: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-11 00:13:38.847895: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-11 00:13:38.847893: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-11 00:13:38.847901: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-11 00:13:38.847902: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-11 00:13:38.847580: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-11 00:13:38.847598: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-11 00:13:38.847605: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-11 00:13:38.847923: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-11 00:13:38.847624: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-11 00:13:38.847610: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-11 00:13:38.847639: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-11 00:13:38.848066: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-11 00:13:38.848080: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-11 00:13:38.848091: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-11 00:13:38.848095: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-11 00:13:38.848101: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-11 00:13:38.848129: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-11 00:13:38.848132: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-11 00:13:38.848137: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-11 00:13:38.847925: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:13:38.847944: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:13:38.847955: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:13:38.847975: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:13:38.847988: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:13:38.847993: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:13:38.848006: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:13:38.848022: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-11 00:13:38.848064: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-11 00:13:38.848083: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:13:38.848457: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-11 00:13:38.848098: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:13:38.848477: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-11 00:13:38.848101: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:13:38.848488: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-11 00:13:38.848120: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:13:38.848506: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-11 00:13:38.848108: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:13:38.848517: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-11 00:13:38.848523: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-11 00:13:38.848119: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:13:38.848529: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-11 00:13:38.848130: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-11 00:13:38.848586: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-11 00:13:38.848557: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-11 00:13:38.848605: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-11 00:13:38.848611: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-11 00:13:38.848623: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-11 00:13:38.848626: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-11 00:13:38.848213: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-11 00:13:38.848638: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-11 00:13:38.848638: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-11 00:13:38.848643: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-11 00:13:38.848234: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-11 00:13:38.848240: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-11 00:13:38.848290: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-11 00:13:38.848290: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-11 00:13:38.848281: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-11 00:13:38.848271: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-11 00:13:38.848305: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-11 00:13:38.848794: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-11 00:13:38.848816: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-11 00:13:38.848823: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-11 00:13:38.848836: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-11 00:13:38.848851: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-11 00:13:38.848850: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-11 00:13:38.848860: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-11 00:13:38.848872: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-11 00:13:38.849299: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:13:38.849323: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:13:38.849335: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:13:38.849355: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:13:38.849360: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:13:38.849361: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:13:38.849372: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:13:38.849414: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:13:38.849377: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:13:38.849828: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-11 00:13:38.849433: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-11 00:13:38.849531: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:13:38.849453: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:13:38.849476: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:13:38.849484: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:13:38.849854: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-11 00:13:38.849863: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-11 00:13:38.849499: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:13:38.849868: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-11 00:13:38.849880: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-11 00:13:38.849552: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:13:38.849484: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:13:38.849895: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-11 00:13:38.849900: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-11 00:13:38.849560: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:13:38.849495: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-11 00:13:38.849594: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:13:38.849905: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-11 00:13:38.849584: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:13:38.849958: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-11 00:13:38.849603: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:13:38.849984: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-11 00:13:38.849988: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-11 00:13:38.849578: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-11 00:13:38.849613: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:13:38.850007: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-11 00:13:38.849998: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-11 00:13:38.850013: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-11 00:13:38.850017: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-11 00:13:38.850021: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-11 00:13:38.850025: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-11 00:13:38.850028: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-11 00:13:38.850031: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-11 00:13:38.850039: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-11 00:13:38.850039: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-11 00:13:38.850054: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-11 00:13:38.850058: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-11 00:13:38.850061: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-11 00:13:38.854190: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-11 00:13:38.854210: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-11 00:13:38.854216: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-11 00:13:38.854230: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-11 00:13:38.854235: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:13:38.862474: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:13:38.862509: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:13:38.862536: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:13:38.862551: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:13:38.862549: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:13:38.862584: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:13:38.862597: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:13:38.862592: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:13:38.863170: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-11 00:13:38.863192: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-11 00:13:38.863198: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-11 00:13:38.863223: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-11 00:13:38.863224: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-11 00:13:38.863230: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-11 00:13:38.863228: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-11 00:13:38.863240: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-11 00:13:38.854237: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-11 00:13:38.854252: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-11 00:13:38.854250: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-11 00:13:38.855338: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-11 00:13:38.855355: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-11 00:13:38.855375: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-11 00:13:38.855397: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-11 00:13:38.855403: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-11 00:13:38.855408: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-11 00:13:38.855414: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-11 00:13:38.855416: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-11 00:14:14.370259: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:14:14.370285: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:14:14.370288: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:14:14.370299: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:14:14.370305: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:14:14.370315: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:14:14.370319: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:14:14.370320: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:14:14.370747: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:14:14.370772: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:14:14.370792: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-11 00:14:14.370900: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:14:14.370803: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-11 00:14:14.370927: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:14:14.370820: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-11 00:14:14.370953: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:14:14.370824: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-11 00:14:14.370959: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-11 00:14:14.371189: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:14:14.370832: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-11 00:14:14.370975: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:14:14.371001: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-11 00:14:14.370981: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:14:14.370849: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-11 00:14:14.370980: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:14:14.370809: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:14:14.370831: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-11 00:14:14.371195: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-11 00:14:14.371005: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:14:14.370833: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:14:14.370852: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-11 00:14:14.371196: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:14:14.370880: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-11 00:14:14.371010: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:14:14.370865: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:14:14.370852: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:14:14.370893: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-11 00:14:14.371021: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:14:14.370857: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-11 00:14:14.371240: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:14:14.370865: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:14:14.370907: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-11 00:14:14.371026: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:14:14.370877: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-11 00:14:14.371247: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:14:14.370866: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:14:14.371252: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:14:14.371098: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:14:14.370922: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-11 00:14:14.371010: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-11 00:14:14.371032: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:14:14.370882: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-11 00:14:14.371269: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:14:14.370882: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:14:14.371289: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:14:14.371279: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:14:14.371128: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:14:14.370933: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-11 00:14:14.371040: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-11 00:14:14.371037: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:14:14.370892: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-11 00:14:14.371272: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:14:14.370884: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:14:14.371307: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:14:14.371304: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:14:14.371143: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:14:14.370939: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-11 00:14:14.371053: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-11 00:14:14.371189: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:14:14.370904: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-11 00:14:14.371282: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:14:14.370886: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:14:14.371336: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:14:14.371327: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:14:14.371167: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:14:14.371124: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-11 00:14:14.371051: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-11 00:14:14.371157: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-11 00:14:14.371424: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:14:14.371358: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:14:14.371336: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:14:14.371167: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-11 00:14:14.371323: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-11 00:14:14.371191: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-11 00:14:14.371445: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:14:14.371367: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:14:14.371348: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:14:14.371459: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:14:14.371450: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:14:14.371181: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-11 00:14:14.371337: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:14:14.371319: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-11 00:14:14.371203: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:14:14.371528: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:14:14.371368: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:14:14.371354: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:14:14.371502: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:14:14.371526: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:14:14.371483: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:14:14.371193: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-11 00:14:14.371337: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:14:14.371354: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-11 00:14:14.371221: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:14:14.371554: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:14:14.371593: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:14:14.371363: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:14:14.371516: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-11 00:14:14.371343: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:14:14.371556: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:14:14.371519: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:14:14.371393: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:14:14.371366: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-11 00:14:14.371235: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:14:14.371575: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:14:14.371645: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:14:14.371574: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:14:14.371545: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-11 00:14:14.371378: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:14:14.371566: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:14:14.371533: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:14:14.371387: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-11 00:14:14.371259: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:14:14.371585: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:14:14.371677: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:14:14.371550: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-11 00:14:14.371391: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:14:14.371586: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:14:14.371550: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:14:14.371406: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-11 00:14:14.371261: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:14:14.371599: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:14:14.371686: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:14:14.371498: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:14:14.371557: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-11 00:14:14.371414: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:14:14.371594: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:14:14.371556: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:14:14.371424: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-11 00:14:14.371288: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:14:14.371595: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:14:14.371705: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:14:14.371568: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-11 00:14:14.371422: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:14:14.371603: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:14:14.371730: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:14:14.371428: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:14:14.371610: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:14:14.371716: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:14:14.371522: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:14:14.371741: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-11 00:14:14.371437: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:14:14.371599: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:14:14.371750: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:14:14.371432: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:14:14.371811: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:14:14.371733: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:14:14.371548: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-11 00:14:14.371442: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:14:14.371752: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:14:14.371748: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:14:14.371568: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-11 00:14:14.371642: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:14:14.371572: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:14:14.371575: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:14:14.371583: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:14:14.371759: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:14:14.371916: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:14:14.371684: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:14:14.371706: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:14:14.371722: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:14:14.371743: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:14:14.371755: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:14:14.371745: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:14:14.371768: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:14:14.371986: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:14:14.373062: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:14:14.373062: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:14:14.373059: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:14:14.373064: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:14:14.373061: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:14:14.373061: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:14:14.373070: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:14:14.373071: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-11 00:14:14.373089: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-11 00:14:14.373089: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-11 00:14:14.373097: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-11 00:14:14.373101: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-11 00:14:14.373102: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-11 00:14:14.373102: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-11 00:14:14.373105: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-11 00:14:14.373103: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-11 00:14:14.373460: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:14:14.373468: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:14:14.373473: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:14:14.373479: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-11 00:14:14.373481: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-11 00:14:14.373476: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:14:14.373478: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:14:14.373480: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:14:14.373493: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-11 00:14:14.373496: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-11 00:14:14.373498: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-11 00:14:14.373499: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-11 00:14:14.373529: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:14:14.373544: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-11 00:14:14.373544: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-11 00:14:14.373559: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-11 00:14:14.373794: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:14:14.373860: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:14:14.373821: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:14:14.373801: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:14:14.373800: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:14:14.373863: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:14:14.373860: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:14:14.373825: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:14:14.373881: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:14:14.373896: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-11 00:14:14.373803: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:14:14.373864: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:14:14.373829: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:14:14.373800: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:14:14.373867: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:14:14.373866: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:14:14.373829: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:14:14.373933: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:14:14.373806: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:14:14.373866: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:14:14.373877: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-11 00:14:14.373868: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:14:14.373875: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-11 00:14:14.373908: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:14:14.373823: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:14:14.373838: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-11 00:14:14.373802: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:14:14.373811: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-11 00:14:14.373868: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:14:14.373872: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:14:14.373825: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:14:14.373969: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:14:14.373984: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-11 00:14:14.373934: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:14:14.373809: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:14:14.373816: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-11 00:14:14.373869: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:14:14.373869: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-11 00:14:14.373910: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:14:14.373827: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:14:14.373937: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:14:14.373820: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-11 00:14:14.373822: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-11 00:14:14.373822: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-11 00:14:14.373873: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:14:14.373882: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-11 00:14:14.373869: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-11 00:14:14.373914: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:14:14.373827: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:14:14.373842: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-11 00:14:14.373981: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:14:14.373938: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-11 00:14:14.373821: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-11 00:14:14.373824: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-11 00:14:14.373826: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-11 00:14:14.373882: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-11 00:14:14.373884: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-11 00:14:14.373884: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-11 00:14:14.373877: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-11 00:14:14.373915: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:14:14.373848: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-11 00:14:14.373848: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-11 00:14:14.373851: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-11 00:14:14.373979: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:14:14.373941: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-11 00:14:14.373888: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-11 00:14:14.373889: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-11 00:14:14.373889: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-11 00:14:14.373875: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:14:14.373889: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-11 00:14:14.373915: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-11 00:14:14.373850: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-11 00:14:14.373853: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-11 00:14:14.373853: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-11 00:14:14.373984: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:14:14.373942: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:14:14.373891: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-11 00:14:14.373897: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-11 00:14:14.373898: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-11 00:14:14.373917: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:14:14.373987: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:14:14.373996: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-11 00:14:14.373940: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-11 00:14:14.373898: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-11 00:14:14.373899: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-11 00:14:14.373900: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-11 00:14:14.373919: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:14:14.373988: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:14:14.373942: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:14:14.373951: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-11 00:14:14.373921: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-11 00:14:14.373927: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-11 00:14:14.373986: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-11 00:14:14.373953: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-11 00:14:14.373952: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-11 00:14:14.373954: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-11 00:14:14.373927: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-11 00:14:14.373930: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-11 00:14:14.373933: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-11 00:14:14.373989: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-11 00:14:14.374003: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-11 00:14:14.373958: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-11 00:14:14.373958: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-11 00:14:14.373960: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-11 00:14:14.373935: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-11 00:14:14.373935: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-11 00:14:14.373936: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-11 00:14:14.374006: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-11 00:14:14.374007: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-11 00:14:14.374009: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-11 00:14:14.373960: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-11 00:14:14.373936: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-11 00:14:14.374010: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-11 00:14:14.374010: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-11 00:14:14.374558: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:14:14.374567: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:14:14.374569: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:14:14.374571: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:14:14.374568: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:14:14.374572: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:14:14.374572: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:14:14.374585: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-11 00:14:14.374578: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-11 00:14:14.374596: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-11 00:14:14.374595: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-11 00:14:14.374600: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-11 00:14:14.374601: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-11 00:14:14.374602: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-11 00:14:14.374604: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-11 00:14:14.374605: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-11 00:14:14.374825: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:14:14.374830: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:14:14.374829: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:14:14.374831: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:14:14.374831: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:14:14.374834: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:14:14.374831: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:14:14.374831: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-11 00:14:14.374851: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-11 00:14:14.374854: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-11 00:14:14.374859: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-11 00:14:14.374859: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-11 00:14:14.374862: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-11 00:14:14.374861: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-11 00:14:14.374864: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-11 00:14:14.374864: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-11 00:14:14.375092: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:14:14.375095: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:14:14.375097: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:14:14.375103: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:14:14.375098: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:14:14.375100: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:14:14.375102: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:14:14.375119: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-11 00:14:14.375364: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:14:14.375100: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-11 00:14:14.375119: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-11 00:14:14.375123: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-11 00:14:14.375135: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-11 00:14:14.375137: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-11 00:14:14.375135: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-11 00:14:14.375136: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-11 00:14:14.375141: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-11 00:14:14.375807: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:14:14.375812: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-11 00:14:14.375962: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:14:14.375815: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:14:14.375817: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-11 00:14:14.375965: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:14:14.375819: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-11 00:14:14.375970: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:14:14.375825: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-11 00:14:14.375974: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:14:14.375820: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:14:14.375836: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-11 00:14:14.375979: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-11 00:14:14.375988: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-11 00:14:14.375836: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-11 00:14:14.375836: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-11 00:14:14.375987: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-11 00:14:14.375998: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-11 00:14:14.375996: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-11 00:14:14.375831: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:14:14.375852: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-11 00:14:14.376003: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-11 00:14:14.375851: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-11 00:14:14.375853: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-11 00:14:14.375855: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-11 00:14:14.376133: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-11 00:14:14.375860: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-11 00:14:14.376166: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-11 00:14:14.376157: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-11 00:14:14.376161: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-11 00:14:14.376189: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-11 00:14:14.376192: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-11 00:14:14.376492: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:14:14.376496: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:14:14.376501: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:14:14.376503: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:14:14.376515: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-11 00:14:14.376507: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:14:14.376517: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-11 00:14:14.376596: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:14:14.376513: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:14:14.376519: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-11 00:14:14.376524: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-11 00:14:14.376526: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-11 00:14:14.376596: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-11 00:14:14.376636: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:14:14.376516: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-11 00:14:14.376603: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:14:14.376519: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:14:14.376532: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-11 00:14:14.376607: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-11 00:14:14.376542: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-11 00:14:14.376545: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-11 00:14:14.376609: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-11 00:14:14.376610: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-11 00:14:14.376613: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-11 00:14:14.376619: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-11 00:14:14.376619: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-11 00:14:14.376619: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-11 00:14:14.376632: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-11 00:14:14.376630: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-11 00:14:14.376637: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-11 00:14:14.376636: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-11 00:14:14.376791: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-11 00:14:14.376809: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-11 00:14:14.377016: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:14:14.377043: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-11 00:14:14.377033: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:14:14.377033: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:14:14.377035: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:14:14.377037: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:14:14.377040: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:14:14.377034: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:14:14.377048: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-11 00:14:14.377083: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-11 00:14:14.377084: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-11 00:14:14.377083: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-11 00:14:14.377086: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-11 00:14:14.377087: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-11 00:14:14.377088: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-11 00:14:14.377089: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-11 00:14:14.380271: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:14:14.380308: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:14:14.380333: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:14:14.380347: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:14:14.380352: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:14:14.380355: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:14:14.380549: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:14:14.380595: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:14:14.381807: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-11 00:14:14.398937: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-11 00:14:14.398973: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-11 00:14:14.398991: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-11 00:14:14.399004: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-11 00:14:14.399015: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-11 00:14:14.399018: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-11 00:14:14.399203: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-11 00:14:14.399204: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:14:14.399658: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:14:14.399661: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:14:14.399664: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-11 00:14:14.399723: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:14:14.399666: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:14:14.399666: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-11 00:14:14.399753: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:14:14.399670: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-11 00:14:14.399786: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:14:14.399668: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-11 00:14:14.399797: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:14:14.399666: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:14:14.399686: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-11 00:14:14.399814: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:14:14.399686: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-11 00:14:14.399687: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-11 00:14:14.399688: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-11 00:14:14.399815: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:14:14.399692: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-11 00:14:14.399694: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-11 00:14:14.399693: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-11 00:14:14.399830: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-11 00:14:14.399696: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-11 00:14:14.400030: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-11 00:14:14.400274: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-11 00:14:14.400275: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:14:14.400129: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-11 00:14:14.400277: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-11 00:14:14.400279: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-11 00:14:14.400290: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-11 00:14:14.400281: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:14:14.400145: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-11 00:14:14.400283: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:14:14.400156: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-11 00:14:14.400284: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:14:14.400164: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-11 00:14:14.400285: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-11 00:14:14.400300: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-11 00:14:14.400173: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-11 00:14:14.400302: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-11 00:14:14.400303: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-11 00:14:14.400305: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-11 00:14:14.400177: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-11 00:14:14.400306: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-11 00:14:14.400306: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-11 00:14:14.400307: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-11 00:14:14.400184: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:14:14.400189: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:14:14.400417: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:14:14.400442: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:14:14.400468: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:14:14.400477: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:14:14.400701: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:14:14.400487: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:14:14.400492: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:14:14.400694: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:14:14.400697: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:14:14.400737: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:14:14.400750: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:14:14.400775: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:14:14.400795: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:14:14.400802: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:14:14.400809: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:14:14.400995: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-11 00:14:14.401143: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-11 00:14:14.401146: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-11 00:14:14.401149: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-11 00:14:14.401153: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-11 00:14:14.401158: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-11 00:14:14.401159: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-11 00:14:14.401154: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-11 00:14:14.401200: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-11 00:14:14.401154: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-11 00:14:14.401157: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-11 00:14:14.401158: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-11 00:14:14.401171: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-11 00:14:14.401173: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-11 00:14:14.401175: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-11 00:14:14.401178: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-11 00:14:14.401179: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-11 00:14:14.401180: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-11 00:14:14.401225: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-11 00:14:14.401250: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-11 00:14:14.401261: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-11 00:14:14.401270: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-11 00:14:14.401281: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:14:14.401441: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-11 00:14:14.401448: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:14:14.401444: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:14:14.401443: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:14:14.401444: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:14:14.401445: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:14:14.401447: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:14:14.401462: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-11 00:14:14.401462: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-11 00:14:14.401464: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-11 00:14:14.401451: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:14:14.401465: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-11 00:14:14.401465: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-11 00:14:14.401465: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-11 00:14:14.401535: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:14:14.401553: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-11 00:14:14.401559: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-11 00:14:14.401573: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-11 00:14:14.401523: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:14:14.401566: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:14:14.401590: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:14:14.401608: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:14:14.401623: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:14:14.401747: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:14:14.401629: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:14:14.401649: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:14:14.401670: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:14:14.401757: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:14:14.401766: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-11 00:14:14.401770: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-11 00:14:14.401755: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:14:14.401755: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:14:14.401755: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:14:14.401760: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:14:14.401782: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-11 00:14:14.401798: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:14:14.401784: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-11 00:14:14.401783: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-11 00:14:14.401783: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-11 00:14:14.401841: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:14:14.401856: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-11 00:14:14.401863: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-11 00:14:14.401878: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-11 00:14:14.401835: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-11 00:14:14.401856: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-11 00:14:14.401868: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-11 00:14:14.401882: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-11 00:14:14.401885: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-11 00:14:14.401903: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-11 00:14:14.401906: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:14:14.402421: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:14:14.402445: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-11 00:14:14.402463: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:14:14.402435: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:14:14.402436: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:14:14.402438: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:14:14.402440: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:14:14.402439: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:14:14.402445: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:14:14.402452: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-11 00:14:14.402477: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-11 00:14:14.402478: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-11 00:14:14.402478: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-11 00:14:14.402479: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-11 00:14:14.402480: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-11 00:14:14.402480: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-11 00:14:14.402480: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-11 00:14:14.403200: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:14:14.403215: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-11 00:14:14.403207: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:14:14.403212: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-11 00:14:14.403280: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-11 00:14:14.403295: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-11 00:14:14.403215: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:14:14.403233: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-11 00:14:14.403221: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:14:14.403237: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-11 00:14:14.403224: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:14:14.403224: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-11 00:14:14.403290: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:14:14.403230: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:14:14.403240: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-11 00:14:14.403295: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:14:14.403246: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-11 00:14:14.403252: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-11 00:14:14.403252: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-11 00:14:14.403300: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-11 00:14:14.403253: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-11 00:14:14.403296: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-11 00:14:14.403303: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-11 00:14:14.403298: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-11 00:14:14.403315: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-11 00:14:14.403307: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-11 00:14:14.403318: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-11 00:14:14.403322: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-11 00:14:14.403323: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-11 00:14:14.403323: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-11 00:14:14.403325: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-11 00:14:14.403324: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-11 00:14:14.404827: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:14:14.404855: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:14:14.404872: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:14:14.404881: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:14:14.404898: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:14:14.404906: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:14:14.404925: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:14:14.404927: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:14:14.405885: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:14:14.405888: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:14:14.405891: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:14:14.405900: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-11 00:14:14.405891: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:14:14.405892: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:14:14.405893: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:14:14.405893: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:14:14.405897: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-11 00:14:14.405913: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-11 00:14:14.405912: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-11 00:14:14.405916: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-11 00:14:14.405917: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-11 00:14:14.405918: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-11 00:14:14.405918: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-11 00:14:14.405918: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-11 00:14:14.371343: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-11 00:14:14.375968: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-11 00:14:14.375992: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-11 00:14:14.375983: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-11 00:14:14.375989: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-11 00:14:14.375993: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-11 00:14:14.375994: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-11 00:14:14.375994: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-11 00:14:14.376007: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-11 00:14:14.376003: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-11 00:14:14.376018: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-11 00:14:14.376020: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-11 00:14:14.376011: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-11 00:14:14.376021: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-11 00:14:14.376022: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-11 00:14:14.376025: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-11 00:14:14.376035: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-11 00:14:14.375370: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:14:14.373891: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-11 00:14:14.375371: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:14:14.373896: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-11 00:14:14.375380: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:14:14.373898: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-11 00:14:14.375384: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-11 00:14:14.375395: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-11 00:14:14.373901: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-11 00:14:14.375395: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-11 00:14:14.373904: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-11 00:14:14.375383: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:14:14.373904: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:14:14.373918: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-11 00:14:14.375391: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:14:14.373908: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-11 00:14:14.373922: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-11 00:14:14.375389: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-11 00:14:14.375407: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-11 00:14:14.373922: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-11 00:14:14.373926: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-11 00:14:14.373927: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-11 00:14:14.375413: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-11 00:14:14.375418: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-11 00:14:14.375422: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-11 00:14:14.373929: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-11 00:14:14.373930: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-11 00:14:14.375423: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-11 00:14:14.375422: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-11 00:14:14.376642: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-11 00:14:14.376646: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-11 00:14:14.376650: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-11 00:14:14.376652: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-11 00:14:14.376664: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-11 00:14:14.376653: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-11 00:14:14.376663: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-11 00:14:14.376653: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-11 00:14:14.376660: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-11 00:14:14.376684: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-11 00:14:14.376681: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-11 00:14:14.376685: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-11 00:14:14.376687: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-11 00:14:14.376689: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-11 00:14:14.376690: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-11 00:14:14.381813: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:14:14.381822: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-11 00:14:14.381819: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:14:14.381819: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:14:14.381823: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:14:14.381822: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:14:14.381835: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-11 00:14:14.381842: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-11 00:14:14.381842: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-11 00:14:14.381843: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-11 00:14:14.381843: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-11 00:14:14.381867: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:14:14.381885: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-11 00:14:14.381894: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-11 00:14:14.381904: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-11 00:14:14.402464: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-11 00:14:14.402463: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-11 00:14:14.402465: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-11 00:14:14.402469: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-11 00:14:14.402466: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-11 00:14:14.402471: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-11 00:14:14.402468: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-11 00:14:14.402482: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-11 00:14:14.402481: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-11 00:14:14.402487: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-11 00:14:14.402486: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-11 00:14:14.402487: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-11 00:14:14.402486: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-11 00:14:14.402488: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-11 00:14:14.402489: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: Successfully preprocessed all matching files. + 0: Detected CUDA files, patching ldflags + 0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... + 0: Building extension module scaled_upper_triang_masked_softmax_cuda... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module scaled_upper_triang_masked_softmax_cuda... + 0: Successfully preprocessed all matching files. + 0: Detected CUDA files, patching ldflags + 0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... + 0: Building extension module scaled_masked_softmax_cuda... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module scaled_masked_softmax_cuda... + 0: Successfully preprocessed all matching files. + 0: Detected CUDA files, patching ldflags + 0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... + 0: Building extension module fused_mix_prec_layer_norm_cuda... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module fused_mix_prec_layer_norm_cuda... + 4: Successfully preprocessed all matching files. + 4: Successfully preprocessed all matching files. + 4: Successfully preprocessed all matching files. + 4: Successfully preprocessed all matching files. +23: Successfully preprocessed all matching files. + 6: Successfully preprocessed all matching files. + 6: Successfully preprocessed all matching files. +28: Successfully preprocessed all matching files. +28: Successfully preprocessed all matching files. +21: Successfully preprocessed all matching files. +21: Successfully preprocessed all matching files. +31: Successfully preprocessed all matching files. +31: Successfully preprocessed all matching files. +28: Successfully preprocessed all matching files. +28: Successfully preprocessed all matching files. +28: Successfully preprocessed all matching files. + 1: Successfully preprocessed all matching files. + 1: Successfully preprocessed all matching files. + 1: Successfully preprocessed all matching files. +14: Successfully preprocessed all matching files. + 3: Successfully preprocessed all matching files. + 2: Successfully preprocessed all matching files. + 2: Successfully preprocessed all matching files. + 2: Successfully preprocessed all matching files. +16: Successfully preprocessed all matching files. +18: Successfully preprocessed all matching files. +18: Successfully preprocessed all matching files. +18: Successfully preprocessed all matching files. +13: Successfully preprocessed all matching files. +13: Successfully preprocessed all matching files. +13: Successfully preprocessed all matching files. + 8: Successfully preprocessed all matching files. + 8: Successfully preprocessed all matching files. + 8: Successfully preprocessed all matching files. +11: Successfully preprocessed all matching files. +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: + 5: + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: + 5: + 5: + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: +10: +10: +10: +10: +10: +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: +12: +12: +12: +12: +12: +12: +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: +15: +15: +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: +20: +20: +20: +20: +20: +20: +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: +24: +24: + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: + 7: + 7: + 7: + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: + 9: + 9: + 9: +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: +30: +30: +30: +30: +30: +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: +26: +26: +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: +26: +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: +17: +17: +17: +17: +17: +17: +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: +19: +19: +19: +19: +19: +19: +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: +25: +25: +25: +25: +25: +25: +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: +27: +27: +27: +27: +27: +27: +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: +29: +29: +29: +29: +29: +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: +22: +22: +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: +22: +22: + 0: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... + 0: Building extension module utils... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Loading extension module utils... + 0: Loading extension module utils... + 1: Loading extension module utils... + 0: Loading extension module utils... + 1: Loading extension module utils... + 0: Loading extension module utils... + 1: Loading extension module utils... + 0: Loading extension module utils... + 0: Loading extension module utils... + 1: Loading extension module utils... + 0: Loading extension module utils... + 1: Loading extension module utils... + 1: Loading extension module utils... + 1: Loading extension module utils... + 1: Loading extension module utils... + 3: Loading extension module utils... + 3: Loading extension module utils... + 2: Loading extension module utils... + 3: Loading extension module utils... + 2: Loading extension module utils... + 3: Loading extension module utils... + 2: Loading extension module utils... + 3: Loading extension module utils... + 2: Loading extension module utils... + 4: Loading extension module utils... + 3: Loading extension module utils... + 2: Loading extension module utils... + 4: Loading extension module utils... + 3: Loading extension module utils... + 2: Loading extension module utils... + 3: Loading extension module utils... + 2: Loading extension module utils... + 4: Loading extension module utils... + 2: Loading extension module utils... + 4: Loading extension module utils... + 4: Loading extension module utils... + 4: Loading extension module utils... + 4: Loading extension module utils... + 6: Loading extension module utils... + 4: Loading extension module utils... + 6: Loading extension module utils... + 8: Loading extension module utils... + 6: Loading extension module utils... + 8: Loading extension module utils... + 6: Loading extension module utils... + 8: Loading extension module utils... + 6: Loading extension module utils... + 8: Loading extension module utils... + 6: Loading extension module utils... + 8: Loading extension module utils... + 6: Loading extension module utils... + 8: Loading extension module utils... + 6: Loading extension module utils... + 8: Loading extension module utils... + 8: Loading extension module utils... +11: Loading extension module utils... +13: Loading extension module utils... +11: Loading extension module utils... +13: Loading extension module utils... +11: Loading extension module utils... +14: Loading extension module utils... +16: Loading extension module utils... +13: Loading extension module utils... +11: Loading extension module utils... +14: Loading extension module utils... +13: Loading extension module utils... +16: Loading extension module utils... +13: Loading extension module utils... +11: Loading extension module utils... +14: Loading extension module utils... +14: Loading extension module utils... +11: Loading extension module utils... +16: Loading extension module utils... +13: Loading extension module utils... +14: Loading extension module utils... +16: Loading extension module utils... +11: Loading extension module utils... +13: Loading extension module utils... +14: Loading extension module utils... +13: Loading extension module utils... +11: Loading extension module utils... +14: Loading extension module utils... +16: Loading extension module utils... +14: Loading extension module utils... +16: Loading extension module utils... +16: Loading extension module utils... +18: Loading extension module utils... +16: Loading extension module utils... +18: Loading extension module utils... +18: Loading extension module utils... +18: Loading extension module utils... +18: Loading extension module utils... +18: Loading extension module utils... +21: Loading extension module utils... +18: Loading extension module utils... +21: Loading extension module utils... +18: Loading extension module utils... +21: Loading extension module utils... +21: Loading extension module utils... +21: Loading extension module utils... +21: Loading extension module utils... +21: Loading extension module utils... +21: Loading extension module utils... +23: Loading extension module utils... +23: Loading extension module utils... +23: Loading extension module utils... +23: Loading extension module utils... +23: Loading extension module utils... +23: Loading extension module utils... +28: Loading extension module utils... +23: Loading extension module utils... +23: Loading extension module utils... +28: Loading extension module utils... +28: Loading extension module utils... +28: Loading extension module utils... +28: Loading extension module utils... +28: Loading extension module utils... +28: Loading extension module utils... +28: Loading extension module utils... +31: Loading extension module utils... +31: Loading extension module utils... +31: Loading extension module utils... +31: Loading extension module utils... +31: Loading extension module utils... +31: Loading extension module utils... +31: Loading extension module utils... +31: Loading extension module utils... + 0: Loading extension module utils... +15: Loading extension module utils... + 9: Loading extension module utils... + 5: Loading extension module utils... +10: Loading extension module utils... +24: Loading extension module utils... + 9: Loading extension module utils... +15: Loading extension module utils... +15: Loading extension module utils... + 9: Loading extension module utils... +10: Loading extension module utils... +15: Loading extension module utils... +25: Loading extension module utils... +26: Loading extension module utils... + 5: Loading extension module utils... +17: Loading extension module utils... +12: Loading extension module utils... +27: Loading extension module utils... +24: Loading extension module utils... + 5: Loading extension module utils... +30: Loading extension module utils... +24: Loading extension module utils... +26: Loading extension module utils... +29: Loading extension module utils... +10: Loading extension module utils... +24: Loading extension module utils... +12: Loading extension module utils... +30: Loading extension module utils... +26: Loading extension module utils... + 5: Loading extension module utils... +20: Loading extension module utils... +25: Loading extension module utils... +27: Loading extension module utils... +10: Loading extension module utils... +29: Loading extension module utils... +30: Loading extension module utils... +12: Loading extension module utils... +25: Loading extension module utils... +26: Loading extension module utils... +10: Loading extension module utils... +27: Loading extension module utils... +24: Loading extension module utils... + 9: Loading extension module utils... + 7: Loading extension module utils... +19: Loading extension module utils... +20: Loading extension module utils... +22: Loading extension module utils... +15: Loading extension module utils... + 5: Loading extension module utils... + 9: Loading extension module utils... + 7: Loading extension module utils... +17: Loading extension module utils... +19: Loading extension module utils... +25: Loading extension module utils... +15: Loading extension module utils... + 9: Loading extension module utils... + 7: Loading extension module utils... +17: Loading extension module utils... +12: Loading extension module utils... +26: Loading extension module utils... +29: Loading extension module utils... +15: Loading extension module utils... + 9: Loading extension module utils... +30: Loading extension module utils... +15: Loading extension module utils... +24: Loading extension module utils... +10: Loading extension module utils... + 7: Loading extension module utils... +22: Loading extension module utils... +25: Loading extension module utils... +29: Loading extension module utils... + 5: Loading extension module utils... + 9: Loading extension module utils... +19: Loading extension module utils... +27: Loading extension module utils... + 7: Loading extension module utils... +17: Loading extension module utils... +20: Loading extension module utils... +10: Loading extension module utils... +12: Loading extension module utils... +30: Loading extension module utils... +24: Loading extension module utils... +22: Loading extension module utils... +25: Loading extension module utils... +26: Loading extension module utils... +29: Loading extension module utils... +12: Loading extension module utils... +20: Loading extension module utils... +27: Loading extension module utils... +19: Loading extension module utils... +17: Loading extension module utils... +25: Loading extension module utils... + 7: Loading extension module utils... +30: Loading extension module utils... +27: Loading extension module utils... +10: Loading extension module utils... +22: Loading extension module utils... +24: Loading extension module utils... +29: Loading extension module utils... + 5: Loading extension module utils... +20: Loading extension module utils... +26: Loading extension module utils... +17: Loading extension module utils... +12: Loading extension module utils... +19: Loading extension module utils... +22: Loading extension module utils... +25: Loading extension module utils... +30: Loading extension module utils... + 7: Loading extension module utils... +27: Loading extension module utils... +29: Loading extension module utils... + 5: Loading extension module utils... +12: Loading extension module utils... +20: Loading extension module utils... +26: Loading extension module utils... +19: Loading extension module utils... + 7: Loading extension module utils... +17: Loading extension module utils... +27: Loading extension module utils... +30: Loading extension module utils... +22: Loading extension module utils... +29: Loading extension module utils... +20: Loading extension module utils... +19: Loading extension module utils... +22: Loading extension module utils... +17: Loading extension module utils... +20: Loading extension module utils... +19: Loading extension module utils... +22: Loading extension module utils... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: +28: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +28: +28: Loading extension module utils...Loading extension module utils... +28: +28: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +28: +28: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...Loading extension module utils... +28: +28: +28: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +28: +28: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... + 8: +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: +11: +11: +11: +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: +18: +18: +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +18: +18: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +18: +18: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +18: +18: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +11: +11: Loading extension module utils... +11: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +11: +11: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: + 1: + 1: + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 1: + 1: Loading extension module utils...Loading extension module utils... + 1: +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +14: +14: +14: +14: Loading extension module utils...Loading extension module utils... +14: Loading extension module utils... +14: +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... + 1: + 1: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... + 1: + 1: + 1: Loading extension module utils...Loading extension module utils... + 1: +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +24: +24: Loading extension module utils...Loading extension module utils... +24: +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +24: +24: Loading extension module utils... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... + 4: + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: + 0: + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... + 0: + 0: Loading extension module utils... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +23: +23: Loading extension module utils...Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +23: +23: +23: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +23: +23: Loading extension module utils... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: + 2: + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 2: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils...Loading extension module utils... + 2: + 2: + 2: + 2: Loading extension module utils...Loading extension module utils... + 2: + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +13: +13: Loading extension module utils... +13: Loading extension module utils... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +13: +13: Loading extension module utils... +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: +21: +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +13: +13: Loading extension module utils...Loading extension module utils... +13: +21: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +21: +21: Loading extension module utils...Loading extension module utils... +21: +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... +21: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +21: +21: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...Loading extension module utils... +21: +21: +21: +21: Loading extension module utils...Loading extension module utils... +21: +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: + 7: + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... + 7: + 7: Loading extension module utils... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: + 5: + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: + 9: +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step... +20: Loading extension module utils... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: +30: +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 9: + 9: Loading extension module utils... + 9: Loading extension module utils... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step... +20: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step... +20: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +20: +20: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +20: +20: Loading extension module utils... +20: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +20: +20: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... + 9: +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +12: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +12: +12: Loading extension module utils...Loading extension module utils... +12: +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +12: +20: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... +12: Loading extension module utils... +20: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... +30: No modifications detected for re-loaded extension module utils, skipping build step... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: +26: +26: +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Loading extension module utils... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +16: +16: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +26: +26: Loading extension module utils... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +16: +16: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +25: +25: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +25: +25: Loading extension module utils...Loading extension module utils... +25: +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +22: +22: Loading extension module utils... +22: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +22: +22: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +10: +10: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...Loading extension module utils... +10: +10: +10: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +10: +10: Loading extension module utils... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: + 3: + 3: +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step... + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils... + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils... +27: Loading extension module utils... + 3: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 3: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 3: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... + 3: + 3: + 3: Loading extension module utils... + 3: Loading extension module utils...Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +27: +27: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +27: +27: Loading extension module utils... + 3: + 3: Loading extension module utils... + 3: +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +15: +15: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: +17: +17: +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 6: + 6: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...Loading extension module utils... + 6: + 6: + 6: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... + 6: + 6: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +19: +19: Loading extension module utils... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +31: +31: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +31: +31: Loading extension module utils...Loading extension module utils... +31: +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +31: +31: Loading extension module utils... +31: Loading extension module utils... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +29: +29: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/utils.py:349: UserWarning: Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings + 0: warnings.warn("Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings") diff --git a/4b284b12bc4opt2/3494406.out b/4b284b12bc4opt2/3494406.out new file mode 100644 index 0000000000000000000000000000000000000000..ef438dd87f6ccff9fadd91d5b49e16d0b1677f3c --- /dev/null +++ b/4b284b12bc4opt2/3494406.out @@ -0,0 +1,42560 @@ +Model parameters: d_model 2816 ffw_size 11264 kv_size 128 n_heads 22 n_layers 36 +Megatron-DeepSpeed/pretrain_gpt.py --tensor-model-parallel-size 2 --pipeline-model-parallel-size 1 --num-layers 36 --hidden-size 2816 --num-attention-heads 22 --kv-channels 128 --ffn-hidden-size 11264 --seq-length 2048 --max-position-embeddings 2048 --micro-batch-size 2 --global-batch-size 1024 --train-samples 1 --vocab-file gpt2/vocab.json --merge-file gpt2/merges.txt --clip-grad 1.0 --kill-switch-path kill-switch-4b284b12bc4opt2val --bf16 --optimizer adam --adam-beta1 0.9 --adam-beta2 0.999 --adam-eps 1e-8 --lr 2e-4 --min-lr 2e-5 --lr-decay-style cosine --lr-decay-samples 1 --lr-warmup-samples 0 --clip-grad 1.0 --weight-decay 1e-1 --override-lr-scheduler --reset-progress --no-load-optim --log-interval 10 --save-interval 1000 --eval-interval 1 --eval-iters 100 --eval-only true --tensorboard-dir tensorboard_4b284b12bc4opt2val --tensorboard-queue-size 5 --log-timers-to-tensorboard --log-batch-size-to-tensorboard --log-validation-ppl-to-tensorboard --save lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2 --load lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2 --train-weighted-split-paths-path train1b5.txt --valid-weighted-split-paths-path val.txt --data-impl mmap --deepspeed --deepspeed_config ds_configs/3494406.json --zero-stage 0 +START 3494406: Thu 11 May 2023 12:12:10 AM EEST + 0: + 0: + 0: ======================= ROCm System Management Interface ======================= + 0: ================================= Concise Info ================================= + 0: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 0: 0 43.0c 97.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 2 45.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 4 49.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 6 44.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: ================================================================================ + 0: ============================= End of ROCm SMI Log ============================== +10: +10: +10: ======================= ROCm System Management Interface ======================= +10: ================================= Concise Info ================================= +10: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +10: 0 45.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 2 41.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 4 44.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 6 38.0c 80.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: ================================================================================ +10: ============================= End of ROCm SMI Log ============================== +12: +12: +12: ======================= ROCm System Management Interface ======================= +12: ================================= Concise Info ================================= +12: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +12: 0 43.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 2 45.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 4 47.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 6 43.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: ================================================================================ +12: ============================= End of ROCm SMI Log ============================== +17: +17: +17: ======================= ROCm System Management Interface ======================= +17: ================================= Concise Info ================================= +17: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +17: 0 43.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 2 38.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 4 43.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 6 47.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: ================================================================================ +17: ============================= End of ROCm SMI Log ============================== +18: +18: +18: ======================= ROCm System Management Interface ======================= +18: ================================= Concise Info ================================= +18: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +18: 0 47.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 1 53.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 2 44.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 4 41.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 6 43.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: ================================================================================ +18: ============================= End of ROCm SMI Log ============================== +20: +20: +20: ======================= ROCm System Management Interface ======================= +20: ================================= Concise Info ================================= +20: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +20: 0 40.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 1 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 2 39.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 4 44.0c 79.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 5 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 6 40.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: ================================================================================ +20: ============================= End of ROCm SMI Log ============================== +25: +25: +25: ======================= ROCm System Management Interface ======================= +25: ================================= Concise Info ================================= +25: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +25: 0 45.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 2 42.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 4 41.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 6 44.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: ================================================================================ +25: ============================= End of ROCm SMI Log ============================== +30: +30: +30: ======================= ROCm System Management Interface ======================= +30: ================================= Concise Info ================================= +30: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +30: 0 50.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 2 44.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 4 44.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 6 43.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: ================================================================================ +30: ============================= End of ROCm SMI Log ============================== +27: +27: +27: ======================= ROCm System Management Interface ======================= +27: ================================= Concise Info ================================= +27: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +27: 0 43.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 1 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 2 43.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 4 41.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 6 45.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: ================================================================================ +27: ============================= End of ROCm SMI Log ============================== +23: +23: +23: ======================= ROCm System Management Interface ======================= +23: ================================= Concise Info ================================= +23: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +23: 0 43.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 2 43.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 4 43.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 6 42.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: ================================================================================ +23: ============================= End of ROCm SMI Log ============================== + 2: + 2: + 2: ======================= ROCm System Management Interface ======================= + 2: ================================= Concise Info ================================= + 2: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 2: 0 48.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 2 39.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 4 40.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 6 42.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 7 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: ================================================================================ + 2: ============================= End of ROCm SMI Log ============================== + 6: + 6: + 6: ======================= ROCm System Management Interface ======================= + 6: ================================= Concise Info ================================= + 6: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 6: 0 45.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 2 34.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 3 39.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 4 44.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 6 38.0c 97.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: ================================================================================ + 6: ============================= End of ROCm SMI Log ============================== + 5: + 5: + 5: ======================= ROCm System Management Interface ======================= + 5: ================================= Concise Info ================================= + 5: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 5: 0 45.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 2 40.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 4 40.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 6 50.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: ================================================================================ + 5: ============================= End of ROCm SMI Log ============================== + 7: + 7: + 7: ======================= ROCm System Management Interface ======================= + 7: ================================= Concise Info ================================= + 7: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 7: 0 45.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 2 39.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 3 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 4 42.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 5 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 6 39.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 7 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: ================================================================================ + 7: ============================= End of ROCm SMI Log ============================== +14: +14: +14: ======================= ROCm System Management Interface ======================= +14: ================================= Concise Info ================================= +14: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +14: 0 47.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 2 42.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 4 37.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 6 45.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 7 39.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: ================================================================================ +14: ============================= End of ROCm SMI Log ============================== +28: +28: +28: ======================= ROCm System Management Interface ======================= +28: ================================= Concise Info ================================= +28: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +28: 0 43.0c 97.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 2 48.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 4 49.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 6 40.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: ================================================================================ +28: ============================= End of ROCm SMI Log ============================== +22: +22: +22: ======================= ROCm System Management Interface ======================= +22: ================================= Concise Info ================================= +22: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +22: 0 46.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 2 39.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 4 41.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 5 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 6 42.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 7 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: ================================================================================ +22: ============================= End of ROCm SMI Log ============================== +26: +26: +26: ======================= ROCm System Management Interface ======================= +26: ================================= Concise Info ================================= +26: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +26: 0 43.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 2 40.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 3 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 4 46.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 6 40.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 7 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: ================================================================================ +26: ============================= End of ROCm SMI Log ============================== +24: +24: +24: ======================= ROCm System Management Interface ======================= +24: ================================= Concise Info ================================= +24: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +24: 0 48.0c 98.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 2 41.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 4 41.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 6 41.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: ================================================================================ +24: ============================= End of ROCm SMI Log ============================== +21: +21: +21: ======================= ROCm System Management Interface ======================= +21: ================================= Concise Info ================================= +21: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +21: 0 37.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 2 43.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 4 45.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 5 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 6 39.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: ================================================================================ +21: ============================= End of ROCm SMI Log ============================== +29: +29: +29: ======================= ROCm System Management Interface ======================= +29: ================================= Concise Info ================================= +29: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +29: 0 44.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 2 41.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 4 43.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 6 36.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: ================================================================================ +29: ============================= End of ROCm SMI Log ============================== + 8: + 8: + 8: ======================= ROCm System Management Interface ======================= + 8: ================================= Concise Info ================================= + 8: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 8: 0 45.0c 101.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 2 45.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 4 50.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 6 38.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: ================================================================================ + 8: ============================= End of ROCm SMI Log ============================== +16: +16: +16: ======================= ROCm System Management Interface ======================= +16: ================================= Concise Info ================================= +16: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +16: 0 51.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 2 39.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 4 46.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 6 44.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: ================================================================================ +16: ============================= End of ROCm SMI Log ============================== +31: +31: +31: ======================= ROCm System Management Interface ======================= +31: ================================= Concise Info ================================= +31: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +31: 0 46.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 1 39.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 2 40.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 4 38.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 5 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 6 43.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: ================================================================================ +31: ============================= End of ROCm SMI Log ============================== + 4: + 4: + 4: ======================= ROCm System Management Interface ======================= + 4: ================================= Concise Info ================================= + 4: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 4: 0 47.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 2 50.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 4 36.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 6 37.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: ================================================================================ + 4: ============================= End of ROCm SMI Log ============================== + 9: + 9: + 9: ======================= ROCm System Management Interface ======================= + 9: ================================= Concise Info ================================= + 9: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 9: 0 43.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 2 43.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 3 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 4 45.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 6 39.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: ================================================================================ + 9: ============================= End of ROCm SMI Log ============================== +13: +13: +13: ======================= ROCm System Management Interface ======================= +13: ================================= Concise Info ================================= +13: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +13: 0 47.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 2 43.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 4 41.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 6 37.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: ================================================================================ +13: ============================= End of ROCm SMI Log ============================== + 1: + 1: + 1: ======================= ROCm System Management Interface ======================= + 1: ================================= Concise Info ================================= + 1: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 1: 0 45.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 2 43.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 4 42.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 6 41.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: ================================================================================ + 1: ============================= End of ROCm SMI Log ============================== +11: +11: +11: ======================= ROCm System Management Interface ======================= +11: ================================= Concise Info ================================= +11: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +11: 0 45.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 2 37.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 4 48.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 6 43.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: ================================================================================ +11: ============================= End of ROCm SMI Log ============================== +19: +19: +19: ======================= ROCm System Management Interface ======================= +19: ================================= Concise Info ================================= +19: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +19: 0 41.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 2 43.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 4 40.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 6 43.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: ================================================================================ +19: ============================= End of ROCm SMI Log ============================== + 3: + 3: + 3: ======================= ROCm System Management Interface ======================= + 3: ================================= Concise Info ================================= + 3: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 3: 0 40.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 2 36.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 3 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 4 42.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 6 47.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: ================================================================================ + 3: ============================= End of ROCm SMI Log ============================== +15: +15: +15: ======================= ROCm System Management Interface ======================= +15: ================================= Concise Info ================================= +15: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +15: 0 44.0c 99.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 2 41.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 4 39.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 6 42.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: ================================================================================ +15: ============================= End of ROCm SMI Log ============================== +25: Launching on nid006512 (25/32), master nid006487 port 9999, GPUs 8, CUDA: True +23: Launching on nid006510 (23/32), master nid006487 port 9999, GPUs 8, CUDA: True +24: Launching on nid006511 (24/32), master nid006487 port 9999, GPUs 8, CUDA: True +30: Launching on nid006517 (30/32), master nid006487 port 9999, GPUs 8, CUDA: True +12: Launching on nid006499 (12/32), master nid006487 port 9999, GPUs 8, CUDA: True + 0: Launching on nid006487 (0/32), master nid006487 port 9999, GPUs 8, CUDA: True + 7: Launching on nid006494 (7/32), master nid006487 port 9999, GPUs 8, CUDA: True +17: Launching on nid006504 (17/32), master nid006487 port 9999, GPUs 8, CUDA: True +20: Launching on nid006507 (20/32), master nid006487 port 9999, GPUs 8, CUDA: True +26: Launching on nid006513 (26/32), master nid006487 port 9999, GPUs 8, CUDA: True +14: Launching on nid006501 (14/32), master nid006487 port 9999, GPUs 8, CUDA: True +29: Launching on nid006516 (29/32), master nid006487 port 9999, GPUs 8, CUDA: True +28: Launching on nid006515 (28/32), master nid006487 port 9999, GPUs 8, CUDA: True + 4: Launching on nid006491 (4/32), master nid006487 port 9999, GPUs 8, CUDA: True +16: Launching on nid006503 (16/32), master nid006487 port 9999, GPUs 8, CUDA: True + 6: Launching on nid006493 (6/32), master nid006487 port 9999, GPUs 8, CUDA: True +31: Launching on nid006518 (31/32), master nid006487 port 9999, GPUs 8, CUDA: True + 8: Launching on nid006495 (8/32), master nid006487 port 9999, GPUs 8, CUDA: True +10: Launching on nid006497 (10/32), master nid006487 port 9999, GPUs 8, CUDA: True + 2: Launching on nid006489 (2/32), master nid006487 port 9999, GPUs 8, CUDA: True +18: Launching on nid006505 (18/32), master nid006487 port 9999, GPUs 8, CUDA: True +27: Launching on nid006514 (27/32), master nid006487 port 9999, GPUs 8, CUDA: True + 5: Launching on nid006492 (5/32), master nid006487 port 9999, GPUs 8, CUDA: True +21: Launching on nid006508 (21/32), master nid006487 port 9999, GPUs 8, CUDA: True +22: Launching on nid006509 (22/32), master nid006487 port 9999, GPUs 8, CUDA: True +15: Launching on nid006502 (15/32), master nid006487 port 9999, GPUs 8, CUDA: True +19: Launching on nid006506 (19/32), master nid006487 port 9999, GPUs 8, CUDA: True +11: Launching on nid006498 (11/32), master nid006487 port 9999, GPUs 8, CUDA: True + 9: Launching on nid006496 (9/32), master nid006487 port 9999, GPUs 8, CUDA: True + 1: Launching on nid006488 (1/32), master nid006487 port 9999, GPUs 8, CUDA: True +13: Launching on nid006500 (13/32), master nid006487 port 9999, GPUs 8, CUDA: True + 3: Launching on nid006490 (3/32), master nid006487 port 9999, GPUs 8, CUDA: True + 0: using world size: 256, data-parallel-size: 128, tensor-model-parallel size: 2, pipeline-model-parallel size: 1 + 0: accumulate and all-reduce gradients in fp32 for bfloat16 data type. + 0: using torch.bfloat16 for parameters ... + 0: ------------------------ arguments ------------------------ + 0: abort_on_unmet_fused_kernel_constraints ......... False + 0: accumulate_allreduce_grads_in_fp32 .............. True + 0: adam_beta1 ...................................... 0.9 + 0: adam_beta2 ...................................... 0.999 + 0: adam_eps ........................................ 1e-08 + 0: adlr_autoresume ................................. False + 0: adlr_autoresume_interval ........................ 1000 + 0: apply_query_key_layer_scaling ................... True + 0: apply_residual_connection_post_layernorm ........ False + 0: attention_dropout ............................... 0.1 + 0: attention_softmax_in_fp32 ....................... False + 0: bert_binary_head ................................ True + 0: bert_load ....................................... None + 0: bf16 ............................................ True + 0: bias_dropout_fusion ............................. True + 0: bias_gelu_fusion ................................ True + 0: biencoder_projection_dim ........................ 0 + 0: biencoder_shared_query_context_model ............ False + 0: block_data_path ................................. None + 0: checkpoint_activations .......................... False + 0: checkpoint_in_cpu ............................... False + 0: checkpoint_num_layers ........................... 1 + 0: clip_grad ....................................... 1.0 + 0: codecarbon_dir .................................. None + 0: consumed_train_samples .......................... 0 + 0: consumed_train_tokens ........................... 0 + 0: consumed_valid_samples .......................... 0 + 0: contigious_checkpointing ........................ False + 0: cpu_optimizer ................................... False + 0: cpu_torch_adam .................................. False + 0: curriculum_learning ............................. False + 0: data_impl ....................................... mmap + 0: data_parallel_size .............................. 128 + 0: data_path ....................................... None + 0: dataloader_type ................................. single + 0: DDP_impl ........................................ local + 0: decoder_seq_length .............................. None + 0: deepscale ....................................... False + 0: deepscale_config ................................ None + 0: deepspeed ....................................... True + 0: deepspeed_activation_checkpointing .............. False + 0: deepspeed_config ................................ ds_configs/3494406.json + 0: deepspeed_mpi ................................... False + 0: distribute_checkpointed_activations ............. False + 0: distributed_backend ............................. nccl + 0: embed_layernorm ................................. False + 0: embedding_path .................................. None + 0: encoder_seq_length .............................. 2048 + 0: eod_mask_loss ................................... False + 0: eval_interval ................................... 1 + 0: eval_iters ...................................... 100 + 0: eval_only ....................................... True + 0: evidence_data_path .............................. None + 0: exit_duration_in_mins ........................... None + 0: exit_interval ................................... None + 0: ffn_hidden_size ................................. 11264 + 0: finetune ........................................ False + 0: fp16 ............................................ False + 0: fp16_lm_cross_entropy ........................... False + 0: fp32_residual_connection ........................ False + 0: gigaflos_no_embeds .............................. 0 + 0: global_batch_size ............................... 1024 + 0: glu_activation .................................. None + 0: hidden_dropout .................................. 0.1 + 0: hidden_size ..................................... 2816 + 0: hysteresis ...................................... 2 + 0: ict_head_size ................................... None + 0: ict_load ........................................ None + 0: img_dim ......................................... 224 + 0: indexer_batch_size .............................. 128 + 0: indexer_log_interval ............................ 1000 + 0: inference ....................................... False + 0: init_method_std ................................. 0.02 + 0: init_method_xavier_uniform ...................... False + 0: initial_loss_scale .............................. 4294967296 + 0: kill_switch_path ................................ kill-switch-4b284b12bc4opt2val + 0: kv_channels ..................................... 128 + 0: layer_norm_fusion ............................... True + 0: layernorm_epsilon ............................... 1e-05 + 0: lazy_mpu_init ................................... None + 0: load ............................................ lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2 + 0: local_rank ...................................... None + 0: log_batch_size_to_tensorboard ................... True + 0: log_interval .................................... 10 + 0: log_learning_rate_to_tensorboard ................ True + 0: log_level ....................................... None + 0: log_level_replica ............................... None + 0: log_loss_scale_to_tensorboard ................... True + 0: log_num_zeros_in_grad ........................... False + 0: log_params_norm ................................. False + 0: log_path ........................................ None + 0: log_timers_to_tensorboard ....................... True + 0: log_validation_ppl_to_tensorboard ............... True + 0: loss_on_targets_only ............................ False + 0: loss_scale ...................................... None + 0: loss_scale_window ............................... 1000 + 0: lr .............................................. 0.0002 + 0: lr_decay_iters .................................. None + 0: lr_decay_samples ................................ 1 + 0: lr_decay_style .................................. cosine + 0: lr_decay_tokens ................................. None + 0: lr_warmup_fraction .............................. None + 0: lr_warmup_iters ................................. 0 + 0: lr_warmup_samples ............................... 0 + 0: make_vocab_size_divisible_by .................... 128 + 0: mask_prob ....................................... 0.15 + 0: masked_softmax_fusion ........................... True + 0: max_position_embeddings ......................... 2048 + 0: mean_noise_span_length .......................... None + 0: memory_centric_tiled_linear ..................... False + 0: merge_file ...................................... gpt2/merges.txt + 0: micro_batch_size ................................ 2 + 0: min_loss_scale .................................. 1.0 + 0: min_lr .......................................... 2e-05 + 0: mmap_warmup ..................................... False + 0: no_load_optim ................................... True + 0: no_load_rng ..................................... None + 0: no_save_optim ................................... None + 0: no_save_rng ..................................... None + 0: noise_density ................................... None + 0: num_attention_heads ............................. 22 + 0: num_channels .................................... 3 + 0: num_classes ..................................... 1000 + 0: num_layers ...................................... 36 + 0: num_layers_per_virtual_pipeline_stage ........... None + 0: num_workers ..................................... 2 + 0: onnx_safe ....................................... None + 0: openai_gelu ..................................... False + 0: optimizer ....................................... adam + 0: optimizer_fusion ................................ True + 0: override_lr_scheduler ........................... True + 0: pad_vocab_size_to ............................... None + 0: params_dtype .................................... torch.bfloat16 + 0: partition_activations ........................... False + 0: patch_dim ....................................... 16 + 0: pipeline_model_parallel_size .................... 1 + 0: position_embedding_type ......................... PositionEmbeddingType.absolute + 0: pp_partition_method ............................. None + 0: profile_backward ................................ False + 0: query_in_block_prob ............................. 0.1 + 0: rampup_batch_size ............................... None + 0: rank ............................................ 0 + 0: remote_device ................................... none + 0: reset_attention_mask ............................ False + 0: reset_position_ids .............................. False + 0: reset_progress .................................. True + 0: retriever_report_topk_accuracies ................ [] + 0: retriever_score_scaling ......................... False + 0: retriever_seq_length ............................ 256 + 0: reweight_loss_based_on_position_frequency ....... False + 0: sample_rate ..................................... 1.0 + 0: save ............................................ lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2 + 0: save_interval ................................... 1000 + 0: scatter_gather_tensors_in_pipeline .............. True + 0: scattered_embeddings ............................ False + 0: seed ............................................ 1234 + 0: seq_length ...................................... 2048 + 0: sgd_momentum .................................... 0.9 + 0: short_seq_prob .................................. 0.1 + 0: skip_train_iteration_range ...................... None + 0: split ........................................... None + 0: split_transformers .............................. False + 0: sync_tp_duplicated_parameters ................... False + 0: synchronize_each_layer .......................... False + 0: tensor_model_parallel_size ...................... 2 + 0: tensorboard_dir ................................. tensorboard_4b284b12bc4opt2val + 0: tensorboard_log_interval ........................ 1 + 0: tensorboard_queue_size .......................... 5 + 0: test_weighted_split_paths ....................... None + 0: test_weighted_split_paths_path .................. None + 0: tile_factor ..................................... 1 + 0: titles_data_path ................................ None + 0: tokenizer_name_or_path .......................... None + 0: tokenizer_type .................................. GPT2BPETokenizer + 0: train_iters ..................................... None + 0: train_samples ................................... 1 + 0: train_tokens .................................... None + 0: train_weighted_split_names ...................... ['train'] + 0: train_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document']] + 0: train_weighted_split_paths_path ................. None + 0: train_weighted_split_splits ..................... [['0:1']] + 0: train_weighted_split_weights .................... [['1.0']] + 0: universal_checkpoint ............................ False + 0: use_bnb_optimizer ............................... False + 0: use_checkpoint_lr_scheduler ..................... False + 0: use_contiguous_buffers_in_ddp ................... True + 0: use_cpu_initialization .......................... None + 0: use_one_sent_docs ............................... False + 0: use_pin_memory .................................. False + 0: valid_num_workers ............................... 2 + 0: valid_weighted_split_names ...................... ['validation'] + 0: valid_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document']] + 0: valid_weighted_split_paths_path ................. None + 0: valid_weighted_split_splits ..................... [['0:1']] + 0: valid_weighted_split_weights .................... [['1.0']] + 0: virtual_pipeline_model_parallel_size ............ None + 0: vocab_extra_ids ................................. 0 + 0: vocab_file ...................................... gpt2/vocab.json + 0: weight_decay .................................... 0.1 + 0: world_size ...................................... 256 + 0: zero_allgather_bucket_size ...................... 0.0 + 0: zero_contigious_gradients ....................... False + 0: zero_reduce_bucket_size ......................... 0.0 + 0: zero_reduce_scatter ............................. False + 0: zero_stage ...................................... 0 + 0: -------------------- end of arguments --------------------- + 0: setting number of micro-batches to constant 4 + 0: > building GPT2BPETokenizer tokenizer ... + 0: > padded vocab (size: 50257) with 175 dummy tokens (new size: 50432) + 0: DeepSpeed general environment info: + 0: torch install path ............... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch'] + 0: torch version .................... 1.13.0+rocm5.2 + 0: torch cuda version ............... None + 0: torch hip version ................ 5.2.21151-afdc89f8 + 0: nvcc version ..................... None + 0: deepspeed install path ........... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/deepspeed'] + 0: deepspeed info ................... 0.7.5, unknown, unknown + 0: deepspeed wheel compiled w. ...... torch 1.13, hip 5.1 + 0: **** Git info for Megatron: git_hash=unknown git_branch=unknown **** + 0: > initializing torch distributed ... + 0: [2023-05-11 00:15:40,971] [INFO] [comm.py:633:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +31: > setting tensorboard ... + 0: > initializing tensor model parallel with size 2 + 0: > initializing pipeline model parallel with size 1 + 0: > setting random seeds to 1234 ... + 0: > initializing model parallel cuda seeds on global rank 0, model parallel rank 0, and data parallel rank 0 with model parallel seed: 3952 and data parallel seed: 1234 + 0: > compiling dataset index builder ... + 0: make: Entering directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' + 0: make: Nothing to be done for 'default'. + 0: make: Leaving directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' + 0: >>> done with dataset index builder. Compilation time: 0.112 seconds + 0: WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. + 0: > compiling and loading fused kernels ... + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.cpp [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.hip [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] + 0: Total number of unsupported CUDA function calls: 0 + 0: + 0: + 0: Total number of replaced kernel launches: 87 + 0: ninja: no work to do. + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.cpp [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.hip [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] + 0: Total number of unsupported CUDA function calls: 0 + 0: + 0: + 0: Total number of replaced kernel launches: 63 + 0: ninja: no work to do. + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda_kernel.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_hip_kernel.hip [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] + 0: Total number of unsupported CUDA function calls: 0 + 0: + 0: + 0: Total number of replaced kernel launches: 67 + 0: [1/1] c++ layer_norm_hip_kernel.cuda.o layer_norm_cuda.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/opt/rocm/lib -lamdhip64 -o fused_mix_prec_layer_norm_cuda.so + 0: >>> done with compiling and loading fused kernels. Compilation time: 19.492 seconds + 0: time to initialize megatron (seconds): 24.881 + 0: [after megatron is initialized] datetime: 2023-05-11 00:16:08 + 0: building GPT model ... + 0: [2023-05-11 00:16:09,047] [INFO] [utils.py:827:see_memory_usage] Before Building Model + 0: [2023-05-11 00:16:09,049] [INFO] [utils.py:828:see_memory_usage] MA 0.0 GB Max_MA 0.0 GB CA 0.0 GB Max_CA 0 GB + 0: [2023-05-11 00:16:09,049] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 38.22 GB, percent = 7.6% + 0: SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None + 0: Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=0, model=1): 1, ProcessCoord(pipe=0, data=1, model=0): 2, ProcessCoord(pipe=0, data=1, model=1): 3, ProcessCoord(pipe=0, data=2, model=0): 4, ProcessCoord(pipe=0, data=2, model=1): 5, ProcessCoord(pipe=0, data=3, model=0): 6, ProcessCoord(pipe=0, data=3, model=1): 7, ProcessCoord(pipe=0, data=4, model=0): 8, ProcessCoord(pipe=0, data=4, model=1): 9, ProcessCoord(pipe=0, data=5, model=0): 10, ProcessCoord(pipe=0, data=5, model=1): 11, ProcessCoord(pipe=0, data=6, model=0): 12, ProcessCoord(pipe=0, data=6, model=1): 13, ProcessCoord(pipe=0, data=7, model=0): 14, ProcessCoord(pipe=0, data=7, model=1): 15, ProcessCoord(pipe=0, data=8, model=0): 16, ProcessCoord(pipe=0, data=8, model=1): 17, ProcessCoord(pipe=0, data=9, model=0): 18, ProcessCoord(pipe=0, data=9, model=1): 19, ProcessCoord(pipe=0, data=10, model=0): 20, ProcessCoord(pipe=0, data=10, model=1): 21, ProcessCoord(pipe=0, data=11, model=0): 22, ProcessCoord(pipe=0, data + 0: =11, model=1): 23, ProcessCoord(pipe=0, data=12, model=0): 24, ProcessCoord(pipe=0, data=12, model=1): 25, ProcessCoord(pipe=0, data=13, model=0): 26, ProcessCoord(pipe=0, data=13, model=1): 27, ProcessCoord(pipe=0, data=14, model=0): 28, ProcessCoord(pipe=0, data=14, model=1): 29, ProcessCoord(pipe=0, data=15, model=0): 30, ProcessCoord(pipe=0, data=15, model=1): 31, ProcessCoord(pipe=0, data=16, model=0): 32, ProcessCoord(pipe=0, data=16, model=1): 33, ProcessCoord(pipe=0, data=17, model=0): 34, ProcessCoord(pipe=0, data=17, model=1): 35, ProcessCoord(pipe=0, data=18, model=0): 36, ProcessCoord(pipe=0, data=18, model=1): 37, ProcessCoord(pipe=0, data=19, model=0): 38, ProcessCoord(pipe=0, data=19, model=1): 39, ProcessCoord(pipe=0, data=20, model=0): 40, ProcessCoord(pipe=0, data=20, model=1): 41, ProcessCoord(pipe=0, data=21, model=0): 42, ProcessCoord(pipe=0, data=21, model=1): 43, ProcessCoord(pipe=0, data=22, model=0): 44, ProcessCoord(pipe=0, data=22, model=1): 45, ProcessCoord(pipe=0, data=23, model=0 + 0: ): 46, ProcessCoord(pipe=0, data=23, model=1): 47, ProcessCoord(pipe=0, data=24, model=0): 48, ProcessCoord(pipe=0, data=24, model=1): 49, ProcessCoord(pipe=0, data=25, model=0): 50, ProcessCoord(pipe=0, data=25, model=1): 51, ProcessCoord(pipe=0, data=26, model=0): 52, ProcessCoord(pipe=0, data=26, model=1): 53, ProcessCoord(pipe=0, data=27, model=0): 54, ProcessCoord(pipe=0, data=27, model=1): 55, ProcessCoord(pipe=0, data=28, model=0): 56, ProcessCoord(pipe=0, data=28, model=1): 57, ProcessCoord(pipe=0, data=29, model=0): 58, ProcessCoord(pipe=0, data=29, model=1): 59, ProcessCoord(pipe=0, data=30, model=0): 60, ProcessCoord(pipe=0, data=30, model=1): 61, ProcessCoord(pipe=0, data=31, model=0): 62, ProcessCoord(pipe=0, data=31, model=1): 63, ProcessCoord(pipe=0, data=32, model=0): 64, ProcessCoord(pipe=0, data=32, model=1): 65, ProcessCoord(pipe=0, data=33, model=0): 66, ProcessCoord(pipe=0, data=33, model=1): 67, ProcessCoord(pipe=0, data=34, model=0): 68, ProcessCoord(pipe=0, data=34, model=1): 69, Proce + 0: ssCoord(pipe=0, data=35, model=0): 70, ProcessCoord(pipe=0, data=35, model=1): 71, ProcessCoord(pipe=0, data=36, model=0): 72, ProcessCoord(pipe=0, data=36, model=1): 73, ProcessCoord(pipe=0, data=37, model=0): 74, ProcessCoord(pipe=0, data=37, model=1): 75, ProcessCoord(pipe=0, data=38, model=0): 76, ProcessCoord(pipe=0, data=38, model=1): 77, ProcessCoord(pipe=0, data=39, model=0): 78, ProcessCoord(pipe=0, data=39, model=1): 79, ProcessCoord(pipe=0, data=40, model=0): 80, ProcessCoord(pipe=0, data=40, model=1): 81, ProcessCoord(pipe=0, data=41, model=0): 82, ProcessCoord(pipe=0, data=41, model=1): 83, ProcessCoord(pipe=0, data=42, model=0): 84, ProcessCoord(pipe=0, data=42, model=1): 85, ProcessCoord(pipe=0, data=43, model=0): 86, ProcessCoord(pipe=0, data=43, model=1): 87, ProcessCoord(pipe=0, data=44, model=0): 88, ProcessCoord(pipe=0, data=44, model=1): 89, ProcessCoord(pipe=0, data=45, model=0): 90, ProcessCoord(pipe=0, data=45, model=1): 91, ProcessCoord(pipe=0, data=46, model=0): 92, ProcessCoord(pipe + 0: =0, data=46, model=1): 93, ProcessCoord(pipe=0, data=47, model=0): 94, ProcessCoord(pipe=0, data=47, model=1): 95, ProcessCoord(pipe=0, data=48, model=0): 96, ProcessCoord(pipe=0, data=48, model=1): 97, ProcessCoord(pipe=0, data=49, model=0): 98, ProcessCoord(pipe=0, data=49, model=1): 99, ProcessCoord(pipe=0, data=50, model=0): 100, ProcessCoord(pipe=0, data=50, model=1): 101, ProcessCoord(pipe=0, data=51, model=0): 102, ProcessCoord(pipe=0, data=51, model=1): 103, ProcessCoord(pipe=0, data=52, model=0): 104, ProcessCoord(pipe=0, data=52, model=1): 105, ProcessCoord(pipe=0, data=53, model=0): 106, ProcessCoord(pipe=0, data=53, model=1): 107, ProcessCoord(pipe=0, data=54, model=0): 108, ProcessCoord(pipe=0, data=54, model=1): 109, ProcessCoord(pipe=0, data=55, model=0): 110, ProcessCoord(pipe=0, data=55, model=1): 111, ProcessCoord(pipe=0, data=56, model=0): 112, ProcessCoord(pipe=0, data=56, model=1): 113, ProcessCoord(pipe=0, data=57, model=0): 114, ProcessCoord(pipe=0, data=57, model=1): 115, ProcessCoord( + 0: pipe=0, data=58, model=0): 116, ProcessCoord(pipe=0, data=58, model=1): 117, ProcessCoord(pipe=0, data=59, model=0): 118, ProcessCoord(pipe=0, data=59, model=1): 119, ProcessCoord(pipe=0, data=60, model=0): 120, ProcessCoord(pipe=0, data=60, model=1): 121, ProcessCoord(pipe=0, data=61, model=0): 122, ProcessCoord(pipe=0, data=61, model=1): 123, ProcessCoord(pipe=0, data=62, model=0): 124, ProcessCoord(pipe=0, data=62, model=1): 125, ProcessCoord(pipe=0, data=63, model=0): 126, ProcessCoord(pipe=0, data=63, model=1): 127, ProcessCoord(pipe=0, data=64, model=0): 128, ProcessCoord(pipe=0, data=64, model=1): 129, ProcessCoord(pipe=0, data=65, model=0): 130, ProcessCoord(pipe=0, data=65, model=1): 131, ProcessCoord(pipe=0, data=66, model=0): 132, ProcessCoord(pipe=0, data=66, model=1): 133, ProcessCoord(pipe=0, data=67, model=0): 134, ProcessCoord(pipe=0, data=67, model=1): 135, ProcessCoord(pipe=0, data=68, model=0): 136, ProcessCoord(pipe=0, data=68, model=1): 137, ProcessCoord(pipe=0, data=69, model=0): 138, Pr + 0: ocessCoord(pipe=0, data=69, model=1): 139, ProcessCoord(pipe=0, data=70, model=0): 140, ProcessCoord(pipe=0, data=70, model=1): 141, ProcessCoord(pipe=0, data=71, model=0): 142, ProcessCoord(pipe=0, data=71, model=1): 143, ProcessCoord(pipe=0, data=72, model=0): 144, ProcessCoord(pipe=0, data=72, model=1): 145, ProcessCoord(pipe=0, data=73, model=0): 146, ProcessCoord(pipe=0, data=73, model=1): 147, ProcessCoord(pipe=0, data=74, model=0): 148, ProcessCoord(pipe=0, data=74, model=1): 149, ProcessCoord(pipe=0, data=75, model=0): 150, ProcessCoord(pipe=0, data=75, model=1): 151, ProcessCoord(pipe=0, data=76, model=0): 152, ProcessCoord(pipe=0, data=76, model=1): 153, ProcessCoord(pipe=0, data=77, model=0): 154, ProcessCoord(pipe=0, data=77, model=1): 155, ProcessCoord(pipe=0, data=78, model=0): 156, ProcessCoord(pipe=0, data=78, model=1): 157, ProcessCoord(pipe=0, data=79, model=0): 158, ProcessCoord(pipe=0, data=79, model=1): 159, ProcessCoord(pipe=0, data=80, model=0): 160, ProcessCoord(pipe=0, data=80, model= + 0: 1): 161, ProcessCoord(pipe=0, data=81, model=0): 162, ProcessCoord(pipe=0, data=81, model=1): 163, ProcessCoord(pipe=0, data=82, model=0): 164, ProcessCoord(pipe=0, data=82, model=1): 165, ProcessCoord(pipe=0, data=83, model=0): 166, ProcessCoord(pipe=0, data=83, model=1): 167, ProcessCoord(pipe=0, data=84, model=0): 168, ProcessCoord(pipe=0, data=84, model=1): 169, ProcessCoord(pipe=0, data=85, model=0): 170, ProcessCoord(pipe=0, data=85, model=1): 171, ProcessCoord(pipe=0, data=86, model=0): 172, ProcessCoord(pipe=0, data=86, model=1): 173, ProcessCoord(pipe=0, data=87, model=0): 174, ProcessCoord(pipe=0, data=87, model=1): 175, ProcessCoord(pipe=0, data=88, model=0): 176, ProcessCoord(pipe=0, data=88, model=1): 177, ProcessCoord(pipe=0, data=89, model=0): 178, ProcessCoord(pipe=0, data=89, model=1): 179, ProcessCoord(pipe=0, data=90, model=0): 180, ProcessCoord(pipe=0, data=90, model=1): 181, ProcessCoord(pipe=0, data=91, model=0): 182, ProcessCoord(pipe=0, data=91, model=1): 183, ProcessCoord(pipe=0, data + 0: =92, model=0): 184, ProcessCoord(pipe=0, data=92, model=1): 185, ProcessCoord(pipe=0, data=93, model=0): 186, ProcessCoord(pipe=0, data=93, model=1): 187, ProcessCoord(pipe=0, data=94, model=0): 188, ProcessCoord(pipe=0, data=94, model=1): 189, ProcessCoord(pipe=0, data=95, model=0): 190, ProcessCoord(pipe=0, data=95, model=1): 191, ProcessCoord(pipe=0, data=96, model=0): 192, ProcessCoord(pipe=0, data=96, model=1): 193, ProcessCoord(pipe=0, data=97, model=0): 194, ProcessCoord(pipe=0, data=97, model=1): 195, ProcessCoord(pipe=0, data=98, model=0): 196, ProcessCoord(pipe=0, data=98, model=1): 197, ProcessCoord(pipe=0, data=99, model=0): 198, ProcessCoord(pipe=0, data=99, model=1): 199, ProcessCoord(pipe=0, data=100, model=0): 200, ProcessCoord(pipe=0, data=100, model=1): 201, ProcessCoord(pipe=0, data=101, model=0): 202, ProcessCoord(pipe=0, data=101, model=1): 203, ProcessCoord(pipe=0, data=102, model=0): 204, ProcessCoord(pipe=0, data=102, model=1): 205, ProcessCoord(pipe=0, data=103, model=0): 206, Process + 0: Coord(pipe=0, data=103, model=1): 207, ProcessCoord(pipe=0, data=104, model=0): 208, ProcessCoord(pipe=0, data=104, model=1): 209, ProcessCoord(pipe=0, data=105, model=0): 210, ProcessCoord(pipe=0, data=105, model=1): 211, ProcessCoord(pipe=0, data=106, model=0): 212, ProcessCoord(pipe=0, data=106, model=1): 213, ProcessCoord(pipe=0, data=107, model=0): 214, ProcessCoord(pipe=0, data=107, model=1): 215, ProcessCoord(pipe=0, data=108, model=0): 216, ProcessCoord(pipe=0, data=108, model=1): 217, ProcessCoord(pipe=0, data=109, model=0): 218, ProcessCoord(pipe=0, data=109, model=1): 219, ProcessCoord(pipe=0, data=110, model=0): 220, ProcessCoord(pipe=0, data=110, model=1): 221, ProcessCoord(pipe=0, data=111, model=0): 222, ProcessCoord(pipe=0, data=111, model=1): 223, ProcessCoord(pipe=0, data=112, model=0): 224, ProcessCoord(pipe=0, data=112, model=1): 225, ProcessCoord(pipe=0, data=113, model=0): 226, ProcessCoord(pipe=0, data=113, model=1): 227, ProcessCoord(pipe=0, data=114, model=0): 228, ProcessCoord(pipe=0 + 0: , data=114, model=1): 229, ProcessCoord(pipe=0, data=115, model=0): 230, ProcessCoord(pipe=0, data=115, model=1): 231, ProcessCoord(pipe=0, data=116, model=0): 232, ProcessCoord(pipe=0, data=116, model=1): 233, ProcessCoord(pipe=0, data=117, model=0): 234, ProcessCoord(pipe=0, data=117, model=1): 235, ProcessCoord(pipe=0, data=118, model=0): 236, ProcessCoord(pipe=0, data=118, model=1): 237, ProcessCoord(pipe=0, data=119, model=0): 238, ProcessCoord(pipe=0, data=119, model=1): 239, ProcessCoord(pipe=0, data=120, model=0): 240, ProcessCoord(pipe=0, data=120, model=1): 241, ProcessCoord(pipe=0, data=121, model=0): 242, ProcessCoord(pipe=0, data=121, model=1): 243, ProcessCoord(pipe=0, data=122, model=0): 244, ProcessCoord(pipe=0, data=122, model=1): 245, ProcessCoord(pipe=0, data=123, model=0): 246, ProcessCoord(pipe=0, data=123, model=1): 247, ProcessCoord(pipe=0, data=124, model=0): 248, ProcessCoord(pipe=0, data=124, model=1): 249, ProcessCoord(pipe=0, data=125, model=0): 250, ProcessCoord(pipe=0, data=125, + 0: model=1): 251, ProcessCoord(pipe=0, data=126, model=0): 252, ProcessCoord(pipe=0, data=126, model=1): 253, ProcessCoord(pipe=0, data=127, model=0): 254, ProcessCoord(pipe=0, data=127, model=1): 255} + 0: [2023-05-11 00:16:14,560] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer + 0: stage=0 layers=43 + 0: 0: _to_float16 + 0: 1: EmbeddingPipe + 0: 2: + 0: 3: ParallelTransformerLayerPipe + 0: 4: ParallelTransformerLayerPipe + 0: 5: ParallelTransformerLayerPipe + 0: 6: ParallelTransformerLayerPipe + 0: 7: ParallelTransformerLayerPipe + 0: 8: ParallelTransformerLayerPipe + 0: 9: ParallelTransformerLayerPipe + 0: 10: ParallelTransformerLayerPipe + 0: 11: ParallelTransformerLayerPipe + 0: 12: ParallelTransformerLayerPipe + 0: 13: ParallelTransformerLayerPipe + 0: 14: ParallelTransformerLayerPipe + 0: 15: ParallelTransformerLayerPipe + 0: 16: ParallelTransformerLayerPipe + 0: 17: ParallelTransformerLayerPipe + 0: 18: ParallelTransformerLayerPipe + 0: 19: ParallelTransformerLayerPipe + 0: 20: ParallelTransformerLayerPipe + 0: 21: ParallelTransformerLayerPipe + 0: 22: ParallelTransformerLayerPipe + 0: 23: ParallelTransformerLayerPipe + 0: 24: ParallelTransformerLayerPipe + 0: 25: ParallelTransformerLayerPipe + 0: 26: ParallelTransformerLayerPipe + 0: 27: ParallelTransformerLayerPipe + 0: 28: ParallelTransformerLayerPipe + 0: 29: ParallelTransformerLayerPipe + 0: 30: ParallelTransformerLayerPipe + 0: 31: ParallelTransformerLayerPipe + 0: 32: ParallelTransformerLayerPipe + 0: 33: ParallelTransformerLayerPipe + 0: 34: ParallelTransformerLayerPipe + 0: 35: ParallelTransformerLayerPipe + 0: 36: ParallelTransformerLayerPipe + 0: 37: ParallelTransformerLayerPipe + 0: 38: ParallelTransformerLayerPipe + 0: 39: undo + 0: 40: MixedFusedLayerNorm + 0: 41: EmbeddingPipe + 0: 42: float16_to_fp32 + 0: loss: CrossEntropy + 0: [2023-05-11 00:16:15,178] [INFO] [utils.py:827:see_memory_usage] After Building Model + 0: [2023-05-11 00:16:15,179] [INFO] [utils.py:828:see_memory_usage] MA 3.35 GB Max_MA 3.35 GB CA 3.6 GB Max_CA 4 GB + 0: [2023-05-11 00:16:15,179] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 38.43 GB, percent = 7.6% + 0: setting training iterations to 0 + 0: > learning rate decay style: cosine + 0: DeepSpeed is enabled. + 0: [2023-05-11 00:16:15,182] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.5, git-hash=unknown, git-branch=unknown + 0: [2023-05-11 00:16:20,406] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False + 0: [2023-05-11 00:16:20,407] [INFO] [logging.py:68:log_dist] [Rank 0] Removing param_group that has no 'params' in the client Optimizer + 0: [2023-05-11 00:16:20,407] [INFO] [logging.py:68:log_dist] [Rank 0] Using client Optimizer as basic optimizer + 0: [2023-05-11 00:16:20,428] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam + 0: [2023-05-11 00:16:20,428] [INFO] [logging.py:68:log_dist] [Rank 0] Creating BF16 optimizer + 0: [2023-05-11 00:16:20,567] [INFO] [utils.py:827:see_memory_usage] begin bf16_optimizer + 0: [2023-05-11 00:16:20,568] [INFO] [utils.py:828:see_memory_usage] MA 3.34 GB Max_MA 3.36 GB CA 3.62 GB Max_CA 4 GB + 0: [2023-05-11 00:16:20,568] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.08 GB, percent = 7.8% + 0: ninja: no work to do. + 0: Time to load utils op: 0.39215803146362305 seconds + 0: Time to load utils op: 0.2694845199584961 secondsTime to load utils op: 0.39112377166748047 seconds + 0: + 0: Time to load utils op: 0.3909189701080322 seconds + 0: Time to load utils op: 0.39229750633239746 seconds + 6: Time to load utils op: 0.38757801055908203 secondsTime to load utils op: 0.3871760368347168 seconds + 6: + 6: Time to load utils op: 0.38811397552490234 seconds + 6: Time to load utils op: 0.38872766494750977 seconds + 0: Time to load utils op: 0.39089345932006836 seconds + 0: Time to load utils op: 0.4082362651824951 seconds + 0: Time to load utils op: 0.391176700592041 seconds + 6: Time to load utils op: 0.38800811767578125 seconds + 6: Time to load utils op: 0.38882017135620117 seconds + 6: Time to load utils op: 0.3878355026245117 seconds + 6: Time to load utils op: 0.38849592208862305 seconds + 1: Time to load utils op: 0.3943188190460205 seconds + 1: Time to load utils op: 0.39334654808044434 seconds + 1: Time to load utils op: 0.39268970489501953 seconds + 1: Time to load utils op: 0.39553213119506836 seconds + 1: Time to load utils op: 0.39467525482177734 seconds + 1: Time to load utils op: 0.3949096202850342 seconds + 1: Time to load utils op: 0.39281392097473145 seconds + 1: Time to load utils op: 0.39351606369018555 seconds + 4: Time to load utils op: 0.3916614055633545 secondsTime to load utils op: 0.39153075218200684 seconds + 4: + 4: Time to load utils op: 0.3934767246246338 seconds + 8: Time to load utils op: 0.3896315097808838 seconds + 8: Time to load utils op: 0.39116978645324707 seconds + 4: Time to load utils op: 0.3934938907623291 seconds + 4: Time to load utils op: 0.3927013874053955 secondsTime to load utils op: 0.39207983016967773 secondsTime to load utils op: 0.3924276828765869 seconds + 4: + 4: + 4: Time to load utils op: 0.3928947448730469 seconds + 8: Time to load utils op: 0.388716459274292 seconds + 8: Time to load utils op: 0.3899531364440918 seconds + 8: Time to load utils op: 0.3905982971191406 seconds + 8: Time to load utils op: 0.3898813724517822 seconds + 8: Time to load utils op: 0.3913114070892334 secondsTime to load utils op: 0.3907749652862549 seconds + 8: + 5: Time to load utils op: 0.3690040111541748 secondsTime to load utils op: 0.36900973320007324 seconds + 5: + 5: Time to load utils op: 0.3690149784088135 seconds + 5: Time to load utils op: 0.3690469264984131 seconds + 5: Time to load utils op: 0.369060754776001 seconds + 5: Time to load utils op: 0.36907362937927246 seconds + 5: Time to load utils op: 0.3690755367279053 secondsTime to load utils op: 0.3690791130065918 seconds + 5: +31: Time to load utils op: 0.37844181060791016 seconds +31: Time to load utils op: 0.37868261337280273 seconds +31: Time to load utils op: 0.379610538482666 seconds +12: Time to load utils op: 0.36869072914123535 seconds +31: Time to load utils op: 0.3796846866607666 seconds +18: Time to load utils op: 0.3855400085449219 seconds +18: Time to load utils op: 0.38752102851867676 seconds +31: Time to load utils op: 0.38053154945373535 seconds +31: Time to load utils op: 0.37850451469421387 secondsTime to load utils op: 0.3803074359893799 seconds +31: Time to load utils op: 0.380033016204834 seconds +31: +12: Time to load utils op: 0.36873817443847656 seconds +18: Time to load utils op: 0.3875408172607422 seconds +12: Time to load utils op: 0.3687422275543213 seconds +12: Time to load utils op: 0.36873912811279297 seconds +12: Time to load utils op: 0.3687765598297119 seconds +18: Time to load utils op: 0.3860359191894531 secondsTime to load utils op: 0.3873634338378906 seconds +18: +18: Time to load utils op: 0.3881213665008545 seconds +18: Time to load utils op: 0.3866109848022461 secondsTime to load utils op: 0.3875002861022949 seconds +18: +12: Time to load utils op: 0.3687870502471924 seconds +12: Time to load utils op: 0.368788480758667 secondsTime to load utils op: 0.36878514289855957 seconds +12: +11: Time to load utils op: 0.40865039825439453 secondsTime to load utils op: 0.4095797538757324 seconds +11: +11: Time to load utils op: 0.4083101749420166 seconds +11: Time to load utils op: 0.40906500816345215 seconds +11: Time to load utils op: 0.40918588638305664 seconds +11: Time to load utils op: 0.40960097312927246 seconds +11: Time to load utils op: 0.40966200828552246 seconds +11: Time to load utils op: 0.4083676338195801 seconds + 2: Time to load utils op: 0.4157567024230957 seconds + 2: Time to load utils op: 0.4147632122039795 seconds + 2: Time to load utils op: 0.4155151844024658 seconds + 2: Time to load utils op: 0.41643643379211426 seconds + 2: Time to load utils op: 0.4160585403442383 seconds + 2: Time to load utils op: 0.4160778522491455 seconds + 2: Time to load utils op: 0.41667604446411133 seconds + 2: Time to load utils op: 0.41449618339538574 seconds +14: Time to load utils op: 0.40833616256713867 seconds +14: Time to load utils op: 0.40886831283569336 seconds +14: Time to load utils op: 0.40810394287109375 seconds +14: Time to load utils op: 0.4087798595428467 secondsTime to load utils op: 0.4096076488494873 seconds +14: +14: Time to load utils op: 0.4094705581665039 seconds +14: Time to load utils op: 0.4076564311981201 secondsTime to load utils op: 0.4077463150024414 seconds +14: +21: Time to load utils op: 0.40758538246154785 seconds +21: Time to load utils op: 0.40488123893737793 seconds +21: Time to load utils op: 0.406994104385376 seconds +21: Time to load utils op: 0.40572643280029297 secondsTime to load utils op: 0.40831518173217773 seconds +21: +21: Time to load utils op: 0.40688610076904297 secondsTime to load utils op: 0.40605807304382324 seconds +21: +21: Time to load utils op: 0.40686511993408203 seconds +16: Time to load utils op: 0.41152000427246094 secondsTime to load utils op: 0.4092998504638672 seconds +16: +16: Time to load utils op: 0.4091637134552002 seconds +16: Time to load utils op: 0.40980100631713867 secondsTime to load utils op: 0.4120657444000244 seconds +16: +16: Time to load utils op: 0.41150856018066406 secondsTime to load utils op: 0.4099438190460205 seconds +16: +16: Time to load utils op: 0.4105710983276367 seconds +13: Time to load utils op: 0.41356515884399414 secondsTime to load utils op: 0.4127645492553711 seconds +13: +13: Time to load utils op: 0.41159582138061523 secondsTime to load utils op: 0.413372278213501 seconds +13: Time to load utils op: 0.41225528717041016 seconds +13: +13: Time to load utils op: 0.4118204116821289 seconds +13: Time to load utils op: 0.41378235816955566 seconds +13: Time to load utils op: 0.41330933570861816 seconds +24: Time to load utils op: 0.3921225070953369 secondsTime to load utils op: 0.3921177387237549 seconds +24: +24: Time to load utils op: 0.39213061332702637 seconds +24: Time to load utils op: 0.3921678066253662 seconds +24: Time to load utils op: 0.39215707778930664 seconds +24: Time to load utils op: 0.3921775817871094 secondsTime to load utils op: 0.3921835422515869 seconds +24: +24: Time to load utils op: 0.3921804428100586 seconds +28: Time to load utils op: 0.4062690734863281 seconds +28: Time to load utils op: 0.4046449661254883 seconds +28: Time to load utils op: 0.4060494899749756 seconds +28: Time to load utils op: 0.405285120010376 seconds +28: Time to load utils op: 0.4057884216308594 seconds +28: Time to load utils op: 0.4044032096862793 seconds +28: Time to load utils op: 0.40342140197753906 secondsTime to load utils op: 0.40432310104370117 seconds +28: +23: Time to load utils op: 0.4080808162689209 secondsTime to load utils op: 0.4079468250274658 seconds +23: +30: Time to load utils op: 0.39302992820739746 secondsTime to load utils op: 0.3930325508117676 seconds +30: +30: Time to load utils op: 0.39304041862487793 secondsTime to load utils op: 0.39304161071777344 secondsTime to load utils op: 0.3930532932281494 seconds +30: +30: +30: Time to load utils op: 0.3930535316467285 seconds +23: Time to load utils op: 0.40781331062316895 seconds +23: Time to load utils op: 0.40880799293518066 seconds +30: Time to load utils op: 0.39307260513305664 seconds +17: Time to load utils op: 0.39298295974731445 seconds +17: Time to load utils op: 0.3929922580718994 secondsTime to load utils op: 0.39299464225769043 seconds +17: +30: Time to load utils op: 0.3930792808532715 seconds +27: Time to load utils op: 0.39282703399658203 seconds + 3: Time to load utils op: 0.419675350189209 seconds + 3: Time to load utils op: 0.4191710948944092 secondsTime to load utils op: 0.4189789295196533 seconds + 3: +17: Time to load utils op: 0.3930180072784424 seconds +19: Time to load utils op: 0.39296483993530273 seconds +19: Time to load utils op: 0.392988920211792 seconds +19: Time to load utils op: 0.3929874897003174 seconds +23: Time to load utils op: 0.40862107276916504 seconds +22: Time to load utils op: 0.39289116859436035 seconds +22: Time to load utils op: 0.3928968906402588 seconds +27: Time to load utils op: 0.3928511142730713 seconds +17: Time to load utils op: 0.3930234909057617 seconds +17: Time to load utils op: 0.3930392265319824 seconds +23: Time to load utils op: 0.40866827964782715 seconds +23: Time to load utils op: 0.40766215324401855 seconds +25: Time to load utils op: 0.3929316997528076 seconds +25: Time to load utils op: 0.39294862747192383 seconds +25: Time to load utils op: 0.39296793937683105 seconds +27: Time to load utils op: 0.3929002285003662 seconds + 3: Time to load utils op: 0.4196279048919678 seconds + 3: Time to load utils op: 0.4207298755645752 seconds + 3: Time to load utils op: 0.4201161861419678 seconds +23: Time to load utils op: 0.4089822769165039 seconds +27: Time to load utils op: 0.39293575286865234 secondsTime to load utils op: 0.39294910430908203 seconds +27: Time to load utils op: 0.3929429054260254 seconds +27: +27: Time to load utils op: 0.3929295539855957 seconds + 3: Time to load utils op: 0.4194507598876953 seconds + 3: Time to load utils op: 0.41980862617492676 seconds +17: Time to load utils op: 0.3930516242980957 seconds +17: Time to load utils op: 0.39307546615600586 seconds +22: Time to load utils op: 0.3929119110107422 seconds +22: Time to load utils op: 0.39290833473205566 secondsTime to load utils op: 0.39291834831237793 secondsTime to load utils op: 0.3929133415222168 seconds +22: Time to load utils op: 0.39292407035827637 seconds +22: +22: +27: Time to load utils op: 0.3929603099822998 seconds +22: Time to load utils op: 0.3929448127746582 seconds +25: Time to load utils op: 0.39299726486206055 seconds +25: Time to load utils op: 0.39301156997680664 seconds +25: Time to load utils op: 0.39303159713745117 seconds +25: Time to load utils op: 0.39304375648498535 seconds +20: Time to load utils op: 0.3935418128967285 seconds +20: Time to load utils op: 0.39356398582458496 seconds +20: Time to load utils op: 0.39357995986938477 seconds +20: Time to load utils op: 0.39360904693603516 seconds +20: Time to load utils op: 0.393613338470459 seconds +25: Time to load utils op: 0.3930513858795166 seconds +20: Time to load utils op: 0.39362359046936035 seconds +26: Time to load utils op: 0.39305567741394043 secondsTime to load utils op: 0.3932607173919678 secondsTime to load utils op: 0.39327335357666016 seconds +26: +26: +26: Time to load utils op: 0.3932914733886719 seconds +26: Time to load utils op: 0.3933103084564209 seconds +26: Time to load utils op: 0.39331936836242676 secondsTime to load utils op: 0.39331889152526855 seconds +19: Time to load utils op: 0.3930320739746094 secondsTime to load utils op: 0.3930368423461914 seconds +19: +19: Time to load utils op: 0.39304351806640625 seconds +19: Time to load utils op: 0.3931002616882324 seconds +19: Time to load utils op: 0.39310145378112793 seconds +20: Time to load utils op: 0.39365673065185547 seconds +20: Time to load utils op: 0.39366841316223145 seconds +26: +26: Time to load utils op: 0.39333152770996094 seconds +29: Time to load utils op: 0.3933131694793701 secondsTime to load utils op: 0.3933134078979492 secondsTime to load utils op: 0.39328598976135254 seconds +29: +29: +29: Time to load utils op: 0.3933374881744385 secondsTime to load utils op: 0.39333200454711914 secondsTime to load utils op: 0.3933243751525879 seconds +29: +29: +29: Time to load utils op: 0.39333367347717285 secondsTime to load utils op: 0.39333152770996094 seconds +29: + 7: Time to load utils op: 0.39348459243774414 seconds + 7: Time to load utils op: 0.39353203773498535 secondsTime to load utils op: 0.39354658126831055 seconds + 7: + 7: Time to load utils op: 0.39356350898742676 seconds + 7: Time to load utils op: 0.39359259605407715 seconds + 7: Time to load utils op: 0.3936140537261963 seconds + 7: Time to load utils op: 0.3936491012573242 seconds + 7: Time to load utils op: 0.39368176460266113 seconds +15: Time to load utils op: 0.3940155506134033 secondsTime to load utils op: 0.39401912689208984 seconds +15: +15: Time to load utils op: 0.3940305709838867 seconds +15: Time to load utils op: 0.3940725326538086 seconds +15: Time to load utils op: 0.39407896995544434 seconds +15: Time to load utils op: 0.394101619720459 secondsTime to load utils op: 0.3941359519958496 seconds +15: +15: Time to load utils op: 0.3941457271575928 seconds + 9: Time to load utils op: 0.39397716522216797 secondsTime to load utils op: 0.3939821720123291 seconds + 9: +10: Time to load utils op: 0.39420366287231445 seconds +10: Time to load utils op: 0.39421868324279785 seconds +10: Time to load utils op: 0.39422059059143066 seconds + 9: Time to load utils op: 0.39401698112487793 seconds + 9: Time to load utils op: 0.39402270317077637 seconds +10: Time to load utils op: 0.39424896240234375 seconds +10: Time to load utils op: 0.39425063133239746 seconds +10: Time to load utils op: 0.39425063133239746 seconds + 9: Time to load utils op: 0.39402031898498535 seconds + 9: Time to load utils op: 0.3940238952636719 seconds + 9: Time to load utils op: 0.39403510093688965 secondsTime to load utils op: 0.39400267601013184 seconds + 9: +10: Time to load utils op: 0.3942601680755615 seconds +10: Time to load utils op: 0.3942747116088867 seconds + 0: [2023-05-11 00:16:20,961] [INFO] [utils.py:827:see_memory_usage] before initializing group 0 + 0: [2023-05-11 00:16:20,962] [INFO] [utils.py:828:see_memory_usage] MA 3.34 GB Max_MA 3.34 GB CA 3.62 GB Max_CA 4 GB + 0: [2023-05-11 00:16:20,962] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.08 GB, percent = 7.8% +28: Time to load utils op: 0.0005433559417724609 secondsTime to load utils op: 0.0005238056182861328 seconds +28: +28: Time to load utils op: 0.0005412101745605469 seconds +28: Time to load utils op: 0.0004534721374511719 secondsTime to load utils op: 0.0004532337188720703 seconds +28: +28: Time to load utils op: 0.0006186962127685547 seconds +28: Time to load utils op: 0.0005977153778076172 seconds +28: Time to load utils op: 0.0006299018859863281 seconds + 8: Time to load utils op: 0.0005414485931396484 seconds + 8: Time to load utils op: 0.00048542022705078125 seconds + 8: Time to load utils op: 0.0004165172576904297 seconds + 8: Time to load utils op: 0.00041222572326660156 seconds + 8: Time to load utils op: 0.0004162788391113281 seconds +18: Time to load utils op: 0.00055694580078125 seconds +18: Time to load utils op: 0.0005717277526855469 seconds + 8: Time to load utils op: 0.0010142326354980469 seconds + 8: Time to load utils op: 0.0010728836059570312 seconds +18: Time to load utils op: 0.0009324550628662109 seconds + 8: Time to load utils op: 0.0009906291961669922 seconds +18: Time to load utils op: 0.0008707046508789062 seconds +18: Time to load utils op: 0.0009493827819824219 seconds +18: Time to load utils op: 0.0009396076202392578 seconds +18: Time to load utils op: 0.0009701251983642578 secondsTime to load utils op: 0.0009028911590576172 seconds +18: +11: Time to load utils op: 0.0008783340454101562 seconds +11: Time to load utils op: 0.0008809566497802734 seconds +11: Time to load utils op: 0.0013432502746582031 seconds +11: Time to load utils op: 0.001203775405883789 secondsTime to load utils op: 0.0012977123260498047 seconds +11: Time to load utils op: 0.001285552978515625 seconds +11: +11: Time to load utils op: 0.0012865066528320312 seconds +11: Time to load utils op: 0.0013513565063476562 seconds +14: Time to load utils op: 0.0004937648773193359 seconds +14: Time to load utils op: 0.00042629241943359375 seconds +14: Time to load utils op: 0.0004546642303466797 seconds +14: Time to load utils op: 0.00046825408935546875 secondsTime to load utils op: 0.00041985511779785156 seconds +14: +14: Time to load utils op: 0.0004069805145263672 seconds + 1: Time to load utils op: 0.0005335807800292969 seconds + 1: Time to load utils op: 0.0004899501800537109 secondsTime to load utils op: 0.00048732757568359375 secondsTime to load utils op: 0.0005526542663574219 seconds + 1: + 1: +14: Time to load utils op: 0.0009813308715820312 seconds + 1: Time to load utils op: 0.0005803108215332031 secondsTime to load utils op: 0.0004761219024658203 seconds + 1: + 1: Time to load utils op: 0.0005154609680175781 seconds +14: Time to load utils op: 0.0010628700256347656 seconds + 1: Time to load utils op: 0.0005645751953125 seconds +24: Time to load utils op: 0.0009527206420898438 seconds +24: Time to load utils op: 0.0009791851043701172 seconds +24: Time to load utils op: 0.0010707378387451172 seconds +24: Time to load utils op: 0.0012521743774414062 secondsTime to load utils op: 0.0012946128845214844 seconds +24: +24: Time to load utils op: 0.0012786388397216797 seconds +24: Time to load utils op: 0.001279592514038086 seconds +24: Time to load utils op: 0.0012807846069335938 seconds + 4: Time to load utils op: 0.0011742115020751953 seconds + 4: Time to load utils op: 0.0011415481567382812 seconds + 4: Time to load utils op: 0.0013473033905029297 secondsTime to load utils op: 0.001276254653930664 seconds + 4: + 4: Time to load utils op: 0.0012271404266357422 seconds + 4: Time to load utils op: 0.0012483596801757812 seconds + 4: Time to load utils op: 0.0012416839599609375 seconds + 4: Time to load utils op: 0.0013549327850341797 seconds + 0: Time to load utils op: 0.0020809173583984375 seconds + 0: Time to load utils op: 0.0020089149475097656 seconds + 0: Time to load utils op: 0.0022008419036865234 seconds + 0: Time to load utils op: 0.002268552780151367 secondsTime to load utils op: 0.0022935867309570312 seconds + 0: + 0: Time to load utils op: 0.002321481704711914 seconds + 0: Time to load utils op: 0.002355337142944336 seconds +23: Time to load utils op: 0.0007977485656738281 seconds +23: Time to load utils op: 0.0009474754333496094 seconds +23: Time to load utils op: 0.000993490219116211 seconds +23: Time to load utils op: 0.00122833251953125 seconds +23: Time to load utils op: 0.001218557357788086 seconds +23: Time to load utils op: 0.0011200904846191406 seconds +23: Time to load utils op: 0.0012638568878173828 secondsTime to load utils op: 0.0012652873992919922 seconds +23: + 2: Time to load utils op: 0.0004951953887939453 seconds + 2: Time to load utils op: 0.0003879070281982422 seconds + 2: Time to load utils op: 0.0005593299865722656 seconds + 2: Time to load utils op: 0.00048089027404785156 secondsTime to load utils op: 0.0004982948303222656 secondsTime to load utils op: 0.00048732757568359375 secondsTime to load utils op: 0.000469207763671875 seconds + 2: + 2: + 2: + 2: Time to load utils op: 0.00042319297790527344 seconds +13: Time to load utils op: 0.0004975795745849609 secondsTime to load utils op: 0.0005373954772949219 seconds +13: +13: Time to load utils op: 0.0005629062652587891 seconds +13: Time to load utils op: 0.0004642009735107422 seconds +13: Time to load utils op: 0.0005404949188232422 seconds +13: Time to load utils op: 0.0004105567932128906 seconds +13: Time to load utils op: 0.0006351470947265625 secondsTime to load utils op: 0.0006194114685058594 seconds +13: +21: Time to load utils op: 0.00043320655822753906 secondsTime to load utils op: 0.00046253204345703125 secondsTime to load utils op: 0.0005373954772949219 seconds +21: +21: +21: Time to load utils op: 0.0004687309265136719 secondsTime to load utils op: 0.0004677772521972656 secondsTime to load utils op: 0.00043392181396484375 secondsTime to load utils op: 0.00048422813415527344 seconds +21: +21: +21: +21: Time to load utils op: 0.0004036426544189453 seconds + 7: Time to load utils op: 0.0006043910980224609 seconds + 7: Time to load utils op: 0.0011699199676513672 seconds + 7: Time to load utils op: 0.0013272762298583984 seconds + 7: Time to load utils op: 0.001583099365234375 seconds + 7: Time to load utils op: 0.0015668869018554688 seconds + 7: Time to load utils op: 0.0014605522155761719 seconds + 7: Time to load utils op: 0.001653909683227539 seconds + 7: Time to load utils op: 0.001605987548828125 seconds + 5: Time to load utils op: 0.0008358955383300781 seconds + 5: Time to load utils op: 0.0011868476867675781 seconds + 5: Time to load utils op: 0.0012493133544921875 seconds + 5: Time to load utils op: 0.0013349056243896484 seconds + 5: Time to load utils op: 0.0012173652648925781 seconds + 5: Time to load utils op: 0.0013701915740966797 seconds + 5: Time to load utils op: 0.0012433528900146484 seconds + 5: Time to load utils op: 0.0013670921325683594 seconds +20: Time to load utils op: 0.0008928775787353516 seconds +12: Time to load utils op: 0.0010843276977539062 seconds + 9: Time to load utils op: 0.0010504722595214844 seconds + 9: Time to load utils op: 0.0010814666748046875 seconds +12: Time to load utils op: 0.0013661384582519531 seconds +20: Time to load utils op: 0.001157522201538086 seconds +20: Time to load utils op: 0.001291513442993164 seconds +20: Time to load utils op: 0.0013098716735839844 seconds +20: Time to load utils op: 0.0013217926025390625 seconds +20: Time to load utils op: 0.0013022422790527344 seconds + 9: Time to load utils op: 0.001390218734741211 seconds + 9: Time to load utils op: 0.0014753341674804688 seconds +12: Time to load utils op: 0.0015566349029541016 seconds +20: Time to load utils op: 0.0013146400451660156 seconds + 9: Time to load utils op: 0.001402139663696289 secondsTime to load utils op: 0.0014138221740722656 secondsTime to load utils op: 0.0014576911926269531 seconds + 9: + 9: +12: Time to load utils op: 0.001577138900756836 seconds +12: Time to load utils op: 0.001554727554321289 seconds +12: Time to load utils op: 0.0015676021575927734 seconds +12: Time to load utils op: 0.0015430450439453125 seconds +20: Time to load utils op: 0.0014507770538330078 seconds +30: Time to load utils op: 0.0009210109710693359 seconds + 9: Time to load utils op: 0.001337289810180664 seconds +12: Time to load utils op: 0.001584768295288086 seconds +30: Time to load utils op: 0.0008995532989501953 seconds +30: Time to load utils op: 0.0013761520385742188 seconds +30: Time to load utils op: 0.0013613700866699219 secondsTime to load utils op: 0.0012545585632324219 seconds +30: +30: Time to load utils op: 0.0013692378997802734 seconds +30: Time to load utils op: 0.0013346672058105469 seconds +30: Time to load utils op: 0.0013840198516845703 seconds +16: Time to load utils op: 0.0005357265472412109 seconds +26: Time to load utils op: 0.0007503032684326172 seconds +26: Time to load utils op: 0.0005981922149658203 secondsTime to load utils op: 0.0007891654968261719 seconds +26: +26: Time to load utils op: 0.000629425048828125 seconds +26: Time to load utils op: 0.0006284713745117188 seconds +26: Time to load utils op: 0.0006606578826904297 seconds +16: Time to load utils op: 0.00055694580078125 seconds +26: Time to load utils op: 0.0007586479187011719 seconds +16: Time to load utils op: 0.00041365623474121094 seconds +16: Time to load utils op: 0.0005593299865722656 seconds +16: Time to load utils op: 0.0005905628204345703 seconds +16: Time to load utils op: 0.0005652904510498047 seconds +26: Time to load utils op: 0.0007576942443847656 seconds +25: Time to load utils op: 0.0009427070617675781 seconds +16: Time to load utils op: 0.0005450248718261719 seconds +16: Time to load utils op: 0.0006275177001953125 seconds +10: Time to load utils op: 0.0008034706115722656 seconds +25: Time to load utils op: 0.0012218952178955078 seconds +22: Time to load utils op: 0.0011813640594482422 seconds +25: Time to load utils op: 0.0013391971588134766 seconds +25: Time to load utils op: 0.001209259033203125 secondsTime to load utils op: 0.0012559890747070312 seconds +25: +22: Time to load utils op: 0.0013918876647949219 seconds +25: Time to load utils op: 0.0013885498046875 seconds +25: Time to load utils op: 0.0013349056243896484 seconds +25: Time to load utils op: 0.0012753009796142578 seconds +22: Time to load utils op: 0.0015044212341308594 seconds +22: Time to load utils op: 0.0014607906341552734 seconds +22: Time to load utils op: 0.0014786720275878906 seconds +22: Time to load utils op: 0.0015368461608886719 seconds +22: Time to load utils op: 0.0014498233795166016 seconds +22: Time to load utils op: 0.0015287399291992188 seconds +10: Time to load utils op: 0.001216888427734375 seconds +27: Time to load utils op: 0.0009176731109619141 seconds +10: Time to load utils op: 0.0013632774353027344 seconds +10: Time to load utils op: 0.0012309551239013672 seconds +10: Time to load utils op: 0.0012691020965576172 seconds +27: Time to load utils op: 0.000942230224609375 seconds +10: Time to load utils op: 0.001241922378540039 seconds +10: Time to load utils op: 0.0012483596801757812 seconds +10: Time to load utils op: 0.00138092041015625 seconds + 3: Time to load utils op: 0.0005152225494384766 seconds + 3: Time to load utils op: 0.0005445480346679688 seconds +27: Time to load utils op: 0.0010650157928466797 secondsTime to load utils op: 0.0011065006256103516 seconds +27: + 3: Time to load utils op: 0.0005719661712646484 seconds +27: Time to load utils op: 0.0011878013610839844 seconds +27: Time to load utils op: 0.0012404918670654297 seconds +27: Time to load utils op: 0.001171112060546875 seconds + 3: Time to load utils op: 0.0005042552947998047 secondsTime to load utils op: 0.0005128383636474609 secondsTime to load utils op: 0.0005049705505371094 secondsTime to load utils op: 0.000453948974609375 seconds + 3: + 3: + 3: + 3: Time to load utils op: 0.0005109310150146484 seconds +27: Time to load utils op: 0.001218557357788086 seconds +15: Time to load utils op: 0.0007839202880859375 seconds +15: Time to load utils op: 0.000736236572265625 seconds +15: Time to load utils op: 0.0008192062377929688 seconds +15: Time to load utils op: 0.0009152889251708984 seconds +15: Time to load utils op: 0.0010268688201904297 secondsTime to load utils op: 0.0009629726409912109 seconds +15: +15: Time to load utils op: 0.0010204315185546875 seconds +15: Time to load utils op: 0.0010623931884765625 seconds +17: Time to load utils op: 0.0008974075317382812 seconds +17: Time to load utils op: 0.001066446304321289 seconds +17: Time to load utils op: 0.0011615753173828125 seconds +17: Time to load utils op: 0.001046895980834961 secondsTime to load utils op: 0.001043558120727539 seconds +17: +17: Time to load utils op: 0.0011019706726074219 seconds +17: Time to load utils op: 0.0010602474212646484 seconds +17: Time to load utils op: 0.0012040138244628906 seconds + 6: Time to load utils op: 0.0014355182647705078 seconds + 6: Time to load utils op: 0.0015561580657958984 seconds + 6: Time to load utils op: 0.0019030570983886719 seconds + 6: Time to load utils op: 0.0019059181213378906 seconds + 6: Time to load utils op: 0.0019249916076660156 secondsTime to load utils op: 0.0018219947814941406 seconds + 6: + 6: Time to load utils op: 0.0018775463104248047 seconds + 6: Time to load utils op: 0.001840829849243164 seconds +19: Time to load utils op: 0.0009410381317138672 seconds +19: Time to load utils op: 0.0010502338409423828 seconds +19: Time to load utils op: 0.0011496543884277344 seconds +19: Time to load utils op: 0.001085042953491211 seconds +19: Time to load utils op: 0.0011167526245117188 seconds +19: Time to load utils op: 0.0011174678802490234 secondsTime to load utils op: 0.0011188983917236328 seconds +19: +19: Time to load utils op: 0.0011124610900878906 seconds +31: Time to load utils op: 0.0005366802215576172 secondsTime to load utils op: 0.0004119873046875 seconds +31: +31: Time to load utils op: 0.0005512237548828125 seconds +31: Time to load utils op: 0.00041604042053222656 seconds +31: Time to load utils op: 0.0005526542663574219 seconds +31: Time to load utils op: 0.0005705356597900391 seconds +31: Time to load utils op: 0.0006184577941894531 secondsTime to load utils op: 0.00058746337890625 seconds +31: +29: Time to load utils op: 0.0011715888977050781 seconds +29: Time to load utils op: 0.0013206005096435547 seconds +29: Time to load utils op: 0.0014574527740478516 seconds +29: Time to load utils op: 0.001644134521484375 seconds +29: Time to load utils op: 0.0017971992492675781 seconds +29: Time to load utils op: 0.0017392635345458984 seconds +29: Time to load utils op: 0.0018510818481445312 seconds +29: Time to load utils op: 0.0018949508666992188 seconds + 0: [2023-05-11 00:16:21,209] [INFO] [utils.py:827:see_memory_usage] after initializing group 0 + 0: [2023-05-11 00:16:21,210] [INFO] [utils.py:828:see_memory_usage] MA 6.78 GB Max_MA 6.78 GB CA 8.74 GB Max_CA 9 GB + 0: [2023-05-11 00:16:21,210] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.16 GB, percent = 7.8% + 0: [2023-05-11 00:16:21,338] [INFO] [utils.py:827:see_memory_usage] before initializing group 1 + 0: [2023-05-11 00:16:21,339] [INFO] [utils.py:828:see_memory_usage] MA 6.78 GB Max_MA 6.78 GB CA 8.74 GB Max_CA 9 GB + 0: [2023-05-11 00:16:21,339] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.13 GB, percent = 7.8% + 0: [2023-05-11 00:16:21,468] [INFO] [utils.py:827:see_memory_usage] after initializing group 1 + 0: [2023-05-11 00:16:21,468] [INFO] [utils.py:828:see_memory_usage] MA 10.06 GB Max_MA 10.06 GB CA 13.62 GB Max_CA 14 GB + 0: [2023-05-11 00:16:21,469] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.15 GB, percent = 7.8% + 0: [2023-05-11 00:16:21,595] [INFO] [utils.py:827:see_memory_usage] before initializing group 2 + 0: [2023-05-11 00:16:21,595] [INFO] [utils.py:828:see_memory_usage] MA 10.06 GB Max_MA 10.06 GB CA 13.62 GB Max_CA 14 GB + 0: [2023-05-11 00:16:21,596] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.15 GB, percent = 7.8% + 0: [2023-05-11 00:16:21,728] [INFO] [utils.py:827:see_memory_usage] after initializing group 2 + 0: [2023-05-11 00:16:21,729] [INFO] [utils.py:828:see_memory_usage] MA 10.06 GB Max_MA 10.06 GB CA 13.62 GB Max_CA 14 GB + 0: [2023-05-11 00:16:21,729] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.17 GB, percent = 7.8% + 0: [2023-05-11 00:16:21,856] [INFO] [utils.py:827:see_memory_usage] before initialize_optimizer + 0: [2023-05-11 00:16:21,856] [INFO] [utils.py:828:see_memory_usage] MA 10.06 GB Max_MA 10.06 GB CA 13.62 GB Max_CA 14 GB + 0: [2023-05-11 00:16:21,856] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.15 GB, percent = 7.8% + 0: [2023-05-11 00:16:21,994] [INFO] [utils.py:827:see_memory_usage] end initialize_optimizer + 0: [2023-05-11 00:16:21,994] [INFO] [utils.py:828:see_memory_usage] MA 10.16 GB Max_MA 10.16 GB CA 13.62 GB Max_CA 14 GB + 0: [2023-05-11 00:16:21,994] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.14 GB, percent = 7.8% + 0: [2023-05-11 00:16:22,122] [INFO] [utils.py:827:see_memory_usage] end bf16_optimizer + 0: [2023-05-11 00:16:22,122] [INFO] [utils.py:828:see_memory_usage] MA 10.16 GB Max_MA 10.16 GB CA 13.62 GB Max_CA 14 GB + 0: [2023-05-11 00:16:22,122] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.17 GB, percent = 7.8% + 0: [2023-05-11 00:16:22,123] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam + 0: [2023-05-11 00:16:22,123] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using client LR scheduler + 0: [2023-05-11 00:16:22,123] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = + 0: [2023-05-11 00:16:22,123] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0002, 0.0002, 0.0002], mom=[(0.9, 0.999), (0.9, 0.999), (0.9, 0.999)] + 0: [2023-05-11 00:16:22,124] [INFO] [config.py:1007:print] DeepSpeedEngine configuration: + 0: [2023-05-11 00:16:22,124] [INFO] [config.py:1011:print] activation_checkpointing_config { + 0: "partition_activations": false, + 0: "contiguous_memory_optimization": false, + 0: "cpu_checkpointing": false, + 0: "number_checkpoints": null, + 0: "synchronize_checkpoint_boundary": false, + 0: "profile": false + 0: } + 0: [2023-05-11 00:16:22,124] [INFO] [config.py:1011:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} + 0: [2023-05-11 00:16:22,124] [INFO] [config.py:1011:print] amp_enabled .................. False + 0: [2023-05-11 00:16:22,124] [INFO] [config.py:1011:print] amp_params ................... False + 0: [2023-05-11 00:16:22,124] [INFO] [config.py:1011:print] autotuning_config ............ { + 0: "enabled": false, + 0: "start_step": null, + 0: "end_step": null, + 0: "metric_path": null, + 0: "arg_mappings": null, + 0: "metric": "throughput", + 0: "model_info": null, + 0: "results_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_results", + 0: "exps_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_exps", + 0: "overwrite": true, + 0: "fast": true, + 0: "start_profile_step": 3, + 0: "end_profile_step": 5, + 0: "tuner_type": "gridsearch", + 0: "tuner_early_stopping": 5, + 0: "tuner_num_trials": 50, + 0: "model_info_path": null, + 0: "mp_size": 1, + 0: "max_train_batch_size": null, + 0: "min_train_batch_size": 1, + 0: "max_train_micro_batch_size_per_gpu": 1.024000e+03, + 0: "min_train_micro_batch_size_per_gpu": 1, + 0: "num_tuning_micro_batch_sizes": 3 + 0: } + 0: [2023-05-11 00:16:22,124] [INFO] [config.py:1011:print] bfloat16_enabled ............. True + 0: [2023-05-11 00:16:22,124] [INFO] [config.py:1011:print] checkpoint_parallel_write_pipeline False + 0: [2023-05-11 00:16:22,124] [INFO] [config.py:1011:print] checkpoint_tag_validation_enabled True + 0: [2023-05-11 00:16:22,124] [INFO] [config.py:1011:print] checkpoint_tag_validation_fail False + 0: [2023-05-11 00:16:22,124] [INFO] [config.py:1011:print] comms_config ................. + 0: [2023-05-11 00:16:22,124] [INFO] [config.py:1011:print] communication_data_type ...... None + 0: [2023-05-11 00:16:22,124] [INFO] [config.py:1011:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_pa + 0: rameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] curriculum_enabled ........... False + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] curriculum_params ............ False + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] dataloader_drop_last ......... False + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] disable_allgather ............ False + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] dump_state ................... False + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] dynamic_loss_scale_args ...... None + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] eigenvalue_enabled ........... False + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] eigenvalue_gas_boundary_resolution 1 + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] eigenvalue_layer_name ........ bert.encoder.layer + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] eigenvalue_layer_num ......... 0 + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] eigenvalue_max_iter .......... 100 + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] eigenvalue_stability ......... 1e-06 + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] eigenvalue_tol ............... 0.01 + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] eigenvalue_verbose ........... False + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] elasticity_enabled ........... False + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] flops_profiler_config ........ { + 0: "enabled": false, + 0: "profile_step": 1, + 0: "module_depth": -1, + 0: "top_modules": 1, + 0: "detailed": true, + 0: "output_file": null + 0: } + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] fp16_auto_cast ............... None + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] fp16_enabled ................. False + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] fp16_master_weights_and_gradients False + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] global_rank .................. 0 + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] gradient_accumulation_steps .. 4 + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] gradient_clipping ............ 1.0 + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] gradient_predivide_factor .... 1.0 + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] initial_dynamic_scale ........ 1 + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] load_universal_checkpoint .... False + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] loss_scale ................... 1.0 + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] memory_breakdown ............. False + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] monitor_config ............... + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] nebula_config ................ { + 0: "enabled": false, + 0: "persistent_storage_path": null, + 0: "persistent_time_interval": 100, + 0: "num_of_version_in_retention": 2, + 0: "enable_nebula_load": true, + 0: "load_path": null + 0: } + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] optimizer_legacy_fusion ...... False + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] optimizer_name ............... None + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] optimizer_params ............. None + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] pld_enabled .................. False + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] pld_params ................... False + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] prescale_gradients ........... False + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] scheduler_name ............... None + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] scheduler_params ............. None + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] sparse_attention ............. None + 0: [2023-05-11 00:16:22,125] [INFO] [config.py:1011:print] sparse_gradients_enabled ..... False + 0: [2023-05-11 00:16:22,126] [INFO] [config.py:1011:print] steps_per_print .............. 2000 + 0: [2023-05-11 00:16:22,126] [INFO] [config.py:1011:print] train_batch_size ............. 1024 + 0: [2023-05-11 00:16:22,126] [INFO] [config.py:1011:print] train_micro_batch_size_per_gpu 2 + 0: [2023-05-11 00:16:22,126] [INFO] [config.py:1011:print] use_node_local_storage ....... False + 0: [2023-05-11 00:16:22,126] [INFO] [config.py:1011:print] wall_clock_breakdown ......... False + 0: [2023-05-11 00:16:22,126] [INFO] [config.py:1011:print] world_size ................... 128 + 0: [2023-05-11 00:16:22,126] [INFO] [config.py:1011:print] zero_allow_untested_optimizer False + 0: [2023-05-11 00:16:22,126] [INFO] [config.py:1011:print] zero_config .................. stage=0 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False + 0: [2023-05-11 00:16:22,126] [INFO] [config.py:1011:print] zero_enabled ................. False + 0: [2023-05-11 00:16:22,126] [INFO] [config.py:1011:print] zero_optimization_stage ...... 0 + 0: [2023-05-11 00:16:22,126] [INFO] [config.py:996:print_user_config] json = { + 0: "train_micro_batch_size_per_gpu": 2, + 0: "train_batch_size": 1.024000e+03, + 0: "gradient_clipping": 1.0, + 0: "zero_optimization": { + 0: "stage": 0 + 0: }, + 0: "bf16": { + 0: "enabled": true + 0: }, + 0: "steps_per_print": 2.000000e+03, + 0: "wall_clock_breakdown": false + 0: } + 0: Time to load utils op: 0.0004515647888183594 seconds + 0: [2023-05-11 00:16:22,126] [INFO] [engine.py:87:__init__] CONFIG: micro_batches=4 micro_batch_size=2 + 0: [2023-05-11 00:16:22,150] [INFO] [engine.py:145:__init__] RANK=0 STAGE=0 LAYERS=43 [0, 43) STAGE_PARAMS=1790593024 (1790.593M) TOTAL_PARAMS=3581186048 (3581.186M) UNIQUE_PARAMS=3581186048 (3581.186M) + 0: [2023-05-11 00:16:22,150] [INFO] [engine.py:145:__init__] RANK=1 STAGE=0 LAYERS=43 [0, 43) STAGE_PARAMS=1790593024 (1790.593M) TOTAL_PARAMS=3581186048 (3581.186M) UNIQUE_PARAMS=3581186048 (3581.186M) +27: [2023-05-11 00:16:24,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +27: [2023-05-11 00:16:24,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +27: [2023-05-11 00:16:24,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +27: [2023-05-11 00:16:24,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +27: [2023-05-11 00:16:24,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +27: [2023-05-11 00:16:24,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +27: [2023-05-11 00:16:24,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +27: [2023-05-11 00:16:24,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +23: [2023-05-11 00:16:24,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +23: [2023-05-11 00:16:24,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +23: [2023-05-11 00:16:24,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +23: [2023-05-11 00:16:24,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +23: [2023-05-11 00:16:24,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +30: [2023-05-11 00:16:24,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +23: [2023-05-11 00:16:24,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +23: [2023-05-11 00:16:24,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +18: [2023-05-11 00:16:24,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +18: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +18: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +18: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +18: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +30: [2023-05-11 00:16:24,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +30: [2023-05-11 00:16:24,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 0: [2023-05-11 00:16:24,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 0: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 0: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +30: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +30: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +30: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 0: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 0: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +16: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +16: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +16: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +16: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +16: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +18: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +18: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 0: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +16: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 0: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +16: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +30: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +28: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +28: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +28: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +28: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +28: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +28: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +28: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +23: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +24: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +24: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +24: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +24: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +24: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +24: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +24: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 7: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +30: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 0: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 8: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 8: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 8: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 7: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 7: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 7: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 7: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 7: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 8: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 8: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 8: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 8: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 7: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +14: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +14: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +14: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +14: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +14: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +16: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +18: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +14: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +14: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +28: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +17: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +17: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +17: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +17: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +17: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +17: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +17: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +10: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +10: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +10: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +10: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +10: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +12: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +12: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +12: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +12: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +12: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +12: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +12: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +10: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +10: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +21: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +14: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 8: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +12: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +15: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +15: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +15: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +15: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 2: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 2: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 2: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 2: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 2: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +22: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +22: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +24: [2023-05-11 00:16:24,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +31: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +31: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +31: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +31: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +31: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 2: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 2: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +21: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +21: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +21: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +21: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +15: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +15: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +15: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +21: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +21: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +22: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +22: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +22: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +22: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +22: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +31: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +31: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 7: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +17: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +10: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +13: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +19: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +13: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +13: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +13: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +13: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 9: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +13: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +13: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 9: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 9: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 9: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 9: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 9: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 9: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +19: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +19: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +19: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 2: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +20: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +20: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +20: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +20: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +20: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +29: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +29: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +29: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +29: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +29: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 5: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 5: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 5: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 5: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 5: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 4: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 4: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 4: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 4: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 4: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +19: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +19: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +20: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +20: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +29: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 3: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 3: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 3: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 3: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 3: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 1: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 5: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 5: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 4: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +13: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +19: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +21: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +29: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 3: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 4: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +15: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 3: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 4: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +20: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +22: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +31: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +29: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 3: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 5: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +19: [2023-05-11 00:16:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 9: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +26: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +26: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +26: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +26: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +26: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +26: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 1: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +11: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +11: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 1: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +26: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +11: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +11: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +11: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +11: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +11: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 1: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 1: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 1: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 6: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 6: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 6: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 6: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +25: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +25: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +25: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +25: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +25: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 1: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 6: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 6: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 6: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +11: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +25: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +25: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +25: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +26: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 1: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 6: [2023-05-11 00:16:24,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +10: [2023-05-11 00:16:24,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +10: [2023-05-11 00:16:24,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +10: [2023-05-11 00:16:24,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +10: [2023-05-11 00:16:24,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +10: [2023-05-11 00:16:24,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +10: [2023-05-11 00:16:24,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +10: [2023-05-11 00:16:24,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +10: [2023-05-11 00:16:24,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +10: [2023-05-11 00:16:24,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +10: [2023-05-11 00:16:24,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +10: [2023-05-11 00:16:24,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +10: [2023-05-11 00:16:24,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +10: [2023-05-11 00:16:24,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +10: [2023-05-11 00:16:24,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +10: [2023-05-11 00:16:24,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +10: [2023-05-11 00:16:24,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +10: [2023-05-11 00:16:24,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +10: [2023-05-11 00:16:24,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +10: [2023-05-11 00:16:24,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +10: [2023-05-11 00:16:24,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +10: [2023-05-11 00:16:24,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +10: [2023-05-11 00:16:24,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +10: [2023-05-11 00:16:24,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +10: [2023-05-11 00:16:24,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +10: [2023-05-11 00:16:24,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +10: [2023-05-11 00:16:24,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +10: [2023-05-11 00:16:24,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +10: [2023-05-11 00:16:24,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +10: [2023-05-11 00:16:24,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +10: [2023-05-11 00:16:24,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +10: [2023-05-11 00:16:24,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +10: [2023-05-11 00:16:24,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +27: [2023-05-11 00:16:24,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +27: [2023-05-11 00:16:24,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +27: [2023-05-11 00:16:24,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +27: [2023-05-11 00:16:24,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +27: [2023-05-11 00:16:24,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +27: [2023-05-11 00:16:24,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +27: [2023-05-11 00:16:24,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +27: [2023-05-11 00:16:24,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +27: [2023-05-11 00:16:24,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +27: [2023-05-11 00:16:24,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +27: [2023-05-11 00:16:24,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +27: [2023-05-11 00:16:24,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +27: [2023-05-11 00:16:24,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +27: [2023-05-11 00:16:24,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +27: [2023-05-11 00:16:24,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +27: [2023-05-11 00:16:24,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +18: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +18: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +18: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +18: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +18: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +27: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +18: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +18: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +18: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +18: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +18: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +27: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +18: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +27: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 8: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 8: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 8: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +27: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +18: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 8: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 8: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 8: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 8: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 8: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 8: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 8: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 8: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 8: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +18: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +13: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +13: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +13: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +13: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +13: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +13: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +13: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +13: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +13: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +13: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +18: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +18: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +13: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +13: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +18: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +14: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +14: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +14: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +14: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +14: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +14: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +14: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +30: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +30: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +30: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +30: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +14: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +30: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +30: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +30: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +30: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +14: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +14: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +14: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +30: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +30: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 8: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +14: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 8: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +30: [2023-05-11 00:16:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +30: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 8: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 8: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +13: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +13: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +27: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +27: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +13: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +13: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +27: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +14: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +14: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +14: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +14: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +27: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +30: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +30: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +30: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +31: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +31: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +31: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +31: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +31: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +30: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +31: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +31: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +22: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +22: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +22: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +22: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +22: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +31: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +31: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +22: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +22: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +31: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +31: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +22: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +18: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 4: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 4: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 4: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 4: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 4: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +22: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +22: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +22: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 4: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 4: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +18: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 4: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 4: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 4: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 4: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 4: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +31: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +18: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +22: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +18: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +23: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +23: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +23: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +23: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +23: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 6: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 6: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 6: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 6: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 6: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +20: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +20: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +19: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +19: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +19: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +19: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +19: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +20: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +20: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +20: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +20: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +20: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +23: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +23: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 6: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 6: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +19: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +19: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +23: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +23: [2023-05-11 00:16:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +23: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +23: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +13: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +19: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +19: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +19: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +19: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +20: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +20: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +22: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +31: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +31: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +31: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +31: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 4: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 4: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 4: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 6: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 6: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 6: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +20: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +20: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +20: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +23: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +22: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +22: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 6: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +19: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +22: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 4: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +14: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +13: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +30: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 6: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +13: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +23: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +23: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +23: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +23: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +19: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +20: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +18: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +30: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +20: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +20: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +14: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +19: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +19: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +20: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +19: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +14: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +18: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +13: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +27: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +27: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +27: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +18: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 9: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 9: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 9: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 9: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 9: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 6: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 6: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 6: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 6: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 9: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 9: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 9: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 9: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 9: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +14: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +30: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +30: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +27: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 9: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +11: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +11: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +11: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +11: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +11: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +11: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +11: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 9: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +11: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +28: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +28: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +28: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +28: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +28: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +11: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +11: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +28: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +28: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +11: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +11: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +28: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +28: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +28: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +18: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 9: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 9: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +28: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 5: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 5: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 5: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 5: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 5: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +28: [2023-05-11 00:16:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 9: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +14: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 4: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +13: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +27: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +27: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 5: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 5: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 9: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +14: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +30: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 5: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 5: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +27: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 5: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +13: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 5: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 5: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +30: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 4: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +27: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +13: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +14: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +30: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 8: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +26: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +26: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +26: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +26: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +26: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +11: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +11: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +23: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +28: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +28: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +28: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 4: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +11: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +11: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +28: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +29: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +29: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +29: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +29: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +29: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +23: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 4: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +29: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +29: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 5: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 5: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 5: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +23: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +13: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +29: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +29: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +29: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 8: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 8: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 5: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +14: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +20: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +29: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +23: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +29: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 8: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +18: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +20: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +30: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +25: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +25: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +25: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +25: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +25: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +19: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 2: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 2: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 2: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 2: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 2: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +25: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +25: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +31: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 2: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 2: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +29: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +29: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +18: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +29: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 2: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 2: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 2: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 9: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +25: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +25: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +25: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +25: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +25: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +29: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +22: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 2: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 2: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +20: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +23: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +31: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +19: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +19: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 9: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 9: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +12: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +12: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +12: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +12: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +12: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +18: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +24: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +24: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 7: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 7: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 7: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 7: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 7: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +13: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +13: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +13: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +23: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +23: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 9: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 7: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 7: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +12: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +12: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +24: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +24: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +24: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +24: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +24: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +14: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +14: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +19: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +20: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 7: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 7: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 7: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 7: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 7: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +14: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +18: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 2: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 2: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 2: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 2: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +12: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +12: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +12: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +12: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +24: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +16: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +16: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +16: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +16: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +16: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 4: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 4: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 4: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +12: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +25: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +25: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +24: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +24: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +24: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +20: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +16: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +16: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +15: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +15: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +15: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +24: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +15: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +15: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +15: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +15: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +17: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +17: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +17: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +17: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +17: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +14: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +16: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +16: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +25: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +25: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 8: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +30: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +31: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +17: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +23: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +16: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +16: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +16: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +15: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +17: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +20: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +30: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 8: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +18: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +30: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +15: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +15: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +15: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 5: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +17: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +22: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +15: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +13: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +17: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +17: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +22: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 8: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 5: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 7: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 7: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 7: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +12: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +12: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +12: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +12: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +31: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 4: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 7: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +17: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +22: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +24: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +24: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +17: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +19: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +18: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +18: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +24: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +24: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 8: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 9: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 5: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +18: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +30: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +29: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 6: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +15: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 1: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 1: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 1: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 1: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 1: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 9: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +16: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +16: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +16: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +31: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +17: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +17: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +17: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +15: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +15: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +15: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 1: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 1: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +16: [2023-05-11 00:16:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 9: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +17: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +20: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +29: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 1: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 5: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +28: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 4: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 6: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +19: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +29: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 1: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 1: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 1: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 4: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +13: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +19: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 1: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 4: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +29: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +28: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +14: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +14: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +20: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +23: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +13: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 6: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 4: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +19: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +13: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +13: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 5: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +23: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 6: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +14: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +14: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +23: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +30: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 5: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +11: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 9: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +31: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +30: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 1: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 5: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 1: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 1: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 1: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +20: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +22: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +30: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +30: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +11: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +28: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +23: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +28: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 5: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +19: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +19: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 7: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +12: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +28: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 7: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +20: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 4: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 4: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +11: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +19: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +17: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +31: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 4: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +29: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +17: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 7: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +29: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 9: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 9: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 9: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 4: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +22: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +12: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +20: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +20: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +29: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +11: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +23: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +19: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +31: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +23: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 2: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +22: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +12: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +20: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 9: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 7: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +17: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +12: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +29: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +28: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +23: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +23: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +22: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 2: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 8: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +24: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 8: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 8: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +17: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +20: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +11: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +19: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +19: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +12: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +15: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 7: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +19: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 6: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 5: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +20: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 6: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +12: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 5: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +11: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +25: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +31: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 5: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 7: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 7: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 8: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +24: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +31: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +31: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 2: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 2: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +17: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +12: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +20: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 6: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +19: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 5: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +17: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +17: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 9: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 9: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +28: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +24: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 6: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +25: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +28: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +22: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +11: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +12: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +16: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +15: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +24: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +15: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 9: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 9: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 7: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +25: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +16: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +15: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 2: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +29: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +16: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 1: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +25: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 2: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +29: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +17: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +11: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +29: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 2: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 5: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +22: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +24: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 5: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +22: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 2: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +31: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +28: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 5: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 5: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +16: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +31: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +15: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +28: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +24: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 8: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +31: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +31: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +22: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 6: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +12: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +29: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +28: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 6: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 6: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 7: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 8: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 8: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 8: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +24: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +22: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +12: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +12: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 1: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +17: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +17: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +15: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 7: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 7: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +17: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +12: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +28: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +16: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +15: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 6: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +15: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +16: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 1: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +17: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 1: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +28: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +29: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 7: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +25: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +28: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +11: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +24: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +29: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +25: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +29: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +12: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +12: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +12: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +29: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +22: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +25: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +22: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +28: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +16: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +12: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +31: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +11: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +11: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 1: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 7: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 2: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 2: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +25: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 2: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +22: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 2: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 7: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 7: [2023-05-11 00:16:24,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 7: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +11: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 6: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 6: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 6: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +16: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 0: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 0: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 0: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 0: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 0: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 0: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 0: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +15: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +28: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +17: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +17: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +15: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 0: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +17: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 0: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 0: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 0: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 6: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 0: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 1: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 1: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +17: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +15: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 1: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +15: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 0: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 0: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 0: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 0: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +24: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +11: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +24: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +24: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +25: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +15: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +11: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +11: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +25: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +15: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +25: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 2: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +25: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 1: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +11: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +24: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 2: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 2: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +15: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 1: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +15: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 2: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +16: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +16: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +16: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 0: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 1: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 0: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 0: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +16: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 0: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +24: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 1: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +25: [2023-05-11 00:16:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +25: [2023-05-11 00:16:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 1: [2023-05-11 00:16:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +24: [2023-05-11 00:16:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +25: [2023-05-11 00:16:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +25: [2023-05-11 00:16:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 1: [2023-05-11 00:16:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +24: [2023-05-11 00:16:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +24: [2023-05-11 00:16:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +16: [2023-05-11 00:16:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +16: [2023-05-11 00:16:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +16: [2023-05-11 00:16:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 0: [2023-05-11 00:16:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 0: [2023-05-11 00:16:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 0: [2023-05-11 00:16:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 1: [2023-05-11 00:16:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +16: [2023-05-11 00:16:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 0: [2023-05-11 00:16:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 1: [2023-05-11 00:16:24,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 0: [2023-05-11 00:16:24,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 0: [2023-05-11 00:16:24,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 0: [2023-05-11 00:16:24,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 0: [2023-05-11 00:16:24,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 0: [2023-05-11 00:16:24,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 0: [2023-05-11 00:16:24,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 0: [2023-05-11 00:16:24,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 0: [2023-05-11 00:16:24,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +26: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +26: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +26: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +26: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +26: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +26: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +26: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +26: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +26: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +26: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +26: [2023-05-11 00:16:24,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +26: [2023-05-11 00:16:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +26: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +26: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +26: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +26: [2023-05-11 00:16:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +26: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +26: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +26: [2023-05-11 00:16:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +26: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +26: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +26: [2023-05-11 00:16:24,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +26: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +26: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +26: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +26: [2023-05-11 00:16:24,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +26: [2023-05-11 00:16:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +21: [2023-05-11 00:16:24,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +21: [2023-05-11 00:16:24,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +21: [2023-05-11 00:16:24,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +21: [2023-05-11 00:16:24,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +21: [2023-05-11 00:16:24,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +21: [2023-05-11 00:16:24,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +21: [2023-05-11 00:16:24,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +21: [2023-05-11 00:16:24,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +21: [2023-05-11 00:16:24,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +21: [2023-05-11 00:16:24,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +21: [2023-05-11 00:16:24,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +21: [2023-05-11 00:16:24,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... +21: [2023-05-11 00:16:24,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +21: [2023-05-11 00:16:24,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +21: [2023-05-11 00:16:24,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +21: [2023-05-11 00:16:24,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 3: [2023-05-11 00:16:24,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 3: [2023-05-11 00:16:24,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 3: [2023-05-11 00:16:24,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 3: [2023-05-11 00:16:24,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 3: [2023-05-11 00:16:24,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 3: [2023-05-11 00:16:24,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 3: [2023-05-11 00:16:24,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 3: [2023-05-11 00:16:24,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 3: [2023-05-11 00:16:24,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 3: [2023-05-11 00:16:24,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 3: [2023-05-11 00:16:24,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt... + 3: [2023-05-11 00:16:24,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 3: [2023-05-11 00:16:24,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 3: [2023-05-11 00:16:24,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 3: [2023-05-11 00:16:24,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... + 3: [2023-05-11 00:16:24,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt... +21: [2023-05-11 00:16:24,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +21: [2023-05-11 00:16:24,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +21: [2023-05-11 00:16:24,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. +21: [2023-05-11 00:16:24,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 3: [2023-05-11 00:16:24,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 3: [2023-05-11 00:16:24,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 3: [2023-05-11 00:16:24,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 3: [2023-05-11 00:16:24,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt. + 3: [2023-05-11 00:16:24,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +21: [2023-05-11 00:16:24,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +21: [2023-05-11 00:16:24,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 3: [2023-05-11 00:16:24,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 3: [2023-05-11 00:16:24,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +21: [2023-05-11 00:16:24,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +21: [2023-05-11 00:16:24,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 3: [2023-05-11 00:16:24,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 3: [2023-05-11 00:16:24,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 3: [2023-05-11 00:16:24,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 3: [2023-05-11 00:16:24,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +21: [2023-05-11 00:16:24,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +21: [2023-05-11 00:16:24,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 3: [2023-05-11 00:16:24,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +21: [2023-05-11 00:16:24,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. +21: [2023-05-11 00:16:24,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt. + 3: [2023-05-11 00:16:24,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 3: [2023-05-11 00:16:24,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 3: [2023-05-11 00:16:24,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 3: [2023-05-11 00:16:24,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +21: [2023-05-11 00:16:24,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +21: [2023-05-11 00:16:24,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +21: [2023-05-11 00:16:24,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +21: [2023-05-11 00:16:24,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +10: [2023-05-11 00:16:24,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +10: [2023-05-11 00:16:24,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +10: [2023-05-11 00:16:24,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +10: [2023-05-11 00:16:24,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +10: [2023-05-11 00:16:24,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +10: [2023-05-11 00:16:24,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +10: [2023-05-11 00:16:24,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +10: [2023-05-11 00:16:24,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +10: [2023-05-11 00:16:24,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +10: [2023-05-11 00:16:24,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +10: [2023-05-11 00:16:24,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +10: [2023-05-11 00:16:24,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +10: [2023-05-11 00:16:24,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +10: [2023-05-11 00:16:24,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +10: [2023-05-11 00:16:24,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +10: [2023-05-11 00:16:24,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +10: [2023-05-11 00:16:24,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 4: [2023-05-11 00:16:24,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 4: [2023-05-11 00:16:24,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 4: [2023-05-11 00:16:24,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 4: [2023-05-11 00:16:24,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 4: [2023-05-11 00:16:24,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 4: [2023-05-11 00:16:24,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 4: [2023-05-11 00:16:24,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +10: [2023-05-11 00:16:24,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 4: [2023-05-11 00:16:24,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +10: [2023-05-11 00:16:24,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 4: [2023-05-11 00:16:24,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 4: [2023-05-11 00:16:24,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +10: [2023-05-11 00:16:24,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 4: [2023-05-11 00:16:24,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 4: [2023-05-11 00:16:24,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 4: [2023-05-11 00:16:24,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 4: [2023-05-11 00:16:24,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 4: [2023-05-11 00:16:24,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 4: [2023-05-11 00:16:24,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +10: [2023-05-11 00:16:24,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +10: [2023-05-11 00:16:24,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +10: [2023-05-11 00:16:24,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +10: [2023-05-11 00:16:24,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +15: [2023-05-11 00:16:24,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +15: [2023-05-11 00:16:24,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +15: [2023-05-11 00:16:24,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +15: [2023-05-11 00:16:24,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +15: [2023-05-11 00:16:24,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +15: [2023-05-11 00:16:24,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +15: [2023-05-11 00:16:24,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 9: [2023-05-11 00:16:24,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 9: [2023-05-11 00:16:24,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 9: [2023-05-11 00:16:24,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +15: [2023-05-11 00:16:24,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 9: [2023-05-11 00:16:24,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 9: [2023-05-11 00:16:24,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 9: [2023-05-11 00:16:24,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 9: [2023-05-11 00:16:24,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 9: [2023-05-11 00:16:24,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 5: [2023-05-11 00:16:24,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 5: [2023-05-11 00:16:24,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 5: [2023-05-11 00:16:24,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 5: [2023-05-11 00:16:24,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 5: [2023-05-11 00:16:24,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 5: [2023-05-11 00:16:24,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 5: [2023-05-11 00:16:24,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +23: [2023-05-11 00:16:24,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +23: [2023-05-11 00:16:24,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +23: [2023-05-11 00:16:24,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +23: [2023-05-11 00:16:24,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +23: [2023-05-11 00:16:24,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +23: [2023-05-11 00:16:24,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +23: [2023-05-11 00:16:24,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 5: [2023-05-11 00:16:24,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +23: [2023-05-11 00:16:24,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +15: [2023-05-11 00:16:24,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +15: [2023-05-11 00:16:24,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +15: [2023-05-11 00:16:24,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +15: [2023-05-11 00:16:24,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 9: [2023-05-11 00:16:24,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 9: [2023-05-11 00:16:24,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 9: [2023-05-11 00:16:24,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 9: [2023-05-11 00:16:24,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 9: [2023-05-11 00:16:24,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 9: [2023-05-11 00:16:24,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 5: [2023-05-11 00:16:24,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 9: [2023-05-11 00:16:24,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 5: [2023-05-11 00:16:24,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 9: [2023-05-11 00:16:24,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 5: [2023-05-11 00:16:24,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 5: [2023-05-11 00:16:24,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 5: [2023-05-11 00:16:24,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 5: [2023-05-11 00:16:24,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 5: [2023-05-11 00:16:24,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 5: [2023-05-11 00:16:24,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +15: [2023-05-11 00:16:24,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +15: [2023-05-11 00:16:24,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +15: [2023-05-11 00:16:24,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +15: [2023-05-11 00:16:24,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 4: [2023-05-11 00:16:24,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 4: [2023-05-11 00:16:24,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +23: [2023-05-11 00:16:24,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +23: [2023-05-11 00:16:24,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +23: [2023-05-11 00:16:24,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +23: [2023-05-11 00:16:24,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +23: [2023-05-11 00:16:24,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +23: [2023-05-11 00:16:24,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +23: [2023-05-11 00:16:24,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +23: [2023-05-11 00:16:24,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 4: [2023-05-11 00:16:24,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 7: [2023-05-11 00:16:24,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 7: [2023-05-11 00:16:24,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 7: [2023-05-11 00:16:24,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 7: [2023-05-11 00:16:24,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 7: [2023-05-11 00:16:24,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 7: [2023-05-11 00:16:24,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 7: [2023-05-11 00:16:24,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 7: [2023-05-11 00:16:24,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 4: [2023-05-11 00:16:24,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +27: [2023-05-11 00:16:24,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +27: [2023-05-11 00:16:24,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +19: [2023-05-11 00:16:24,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +19: [2023-05-11 00:16:24,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +19: [2023-05-11 00:16:24,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +19: [2023-05-11 00:16:24,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +19: [2023-05-11 00:16:24,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +19: [2023-05-11 00:16:24,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +19: [2023-05-11 00:16:24,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +19: [2023-05-11 00:16:24,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +27: [2023-05-11 00:16:24,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +27: [2023-05-11 00:16:24,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +14: [2023-05-11 00:16:24,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +14: [2023-05-11 00:16:24,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +14: [2023-05-11 00:16:24,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +16: [2023-05-11 00:16:24,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +16: [2023-05-11 00:16:24,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +16: [2023-05-11 00:16:24,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +16: [2023-05-11 00:16:24,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +16: [2023-05-11 00:16:24,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +16: [2023-05-11 00:16:24,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +16: [2023-05-11 00:16:24,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +27: [2023-05-11 00:16:24,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +27: [2023-05-11 00:16:24,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +27: [2023-05-11 00:16:24,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +27: [2023-05-11 00:16:24,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +14: [2023-05-11 00:16:24,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +14: [2023-05-11 00:16:24,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +14: [2023-05-11 00:16:24,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +14: [2023-05-11 00:16:24,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +16: [2023-05-11 00:16:24,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 7: [2023-05-11 00:16:24,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 7: [2023-05-11 00:16:24,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 7: [2023-05-11 00:16:24,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 7: [2023-05-11 00:16:24,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +14: [2023-05-11 00:16:24,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 7: [2023-05-11 00:16:24,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 7: [2023-05-11 00:16:24,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 7: [2023-05-11 00:16:24,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 7: [2023-05-11 00:16:24,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +30: [2023-05-11 00:16:24,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +30: [2023-05-11 00:16:24,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +30: [2023-05-11 00:16:24,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +30: [2023-05-11 00:16:24,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +30: [2023-05-11 00:16:24,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +30: [2023-05-11 00:16:24,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +30: [2023-05-11 00:16:24,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +30: [2023-05-11 00:16:24,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +19: [2023-05-11 00:16:24,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +27: [2023-05-11 00:16:24,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +19: [2023-05-11 00:16:24,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +14: [2023-05-11 00:16:24,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +19: [2023-05-11 00:16:24,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +14: [2023-05-11 00:16:24,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +14: [2023-05-11 00:16:24,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +16: [2023-05-11 00:16:24,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +16: [2023-05-11 00:16:24,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +27: [2023-05-11 00:16:24,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +16: [2023-05-11 00:16:24,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +16: [2023-05-11 00:16:24,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +30: [2023-05-11 00:16:24,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +30: [2023-05-11 00:16:24,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +30: [2023-05-11 00:16:24,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +14: [2023-05-11 00:16:24,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +16: [2023-05-11 00:16:24,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +14: [2023-05-11 00:16:24,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +19: [2023-05-11 00:16:24,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +27: [2023-05-11 00:16:24,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +14: [2023-05-11 00:16:24,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +14: [2023-05-11 00:16:24,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +16: [2023-05-11 00:16:24,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +16: [2023-05-11 00:16:24,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +19: [2023-05-11 00:16:24,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +19: [2023-05-11 00:16:24,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +16: [2023-05-11 00:16:24,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +14: [2023-05-11 00:16:24,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +19: [2023-05-11 00:16:24,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +19: [2023-05-11 00:16:24,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +30: [2023-05-11 00:16:24,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +30: [2023-05-11 00:16:24,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +30: [2023-05-11 00:16:24,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +27: [2023-05-11 00:16:24,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +30: [2023-05-11 00:16:24,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +30: [2023-05-11 00:16:24,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 4: [2023-05-11 00:16:24,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 4: [2023-05-11 00:16:24,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +27: [2023-05-11 00:16:24,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +27: [2023-05-11 00:16:24,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +27: [2023-05-11 00:16:24,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +27: [2023-05-11 00:16:24,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 4: [2023-05-11 00:16:24,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 4: [2023-05-11 00:16:24,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +15: [2023-05-11 00:16:24,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +15: [2023-05-11 00:16:24,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 5: [2023-05-11 00:16:24,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 5: [2023-05-11 00:16:24,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 9: [2023-05-11 00:16:24,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 9: [2023-05-11 00:16:24,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 9: [2023-05-11 00:16:24,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +11: [2023-05-11 00:16:24,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +11: [2023-05-11 00:16:24,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +11: [2023-05-11 00:16:24,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 9: [2023-05-11 00:16:24,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +28: [2023-05-11 00:16:24,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +28: [2023-05-11 00:16:24,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +28: [2023-05-11 00:16:24,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +28: [2023-05-11 00:16:24,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +28: [2023-05-11 00:16:24,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +11: [2023-05-11 00:16:24,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +11: [2023-05-11 00:16:24,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +11: [2023-05-11 00:16:24,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +11: [2023-05-11 00:16:24,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +28: [2023-05-11 00:16:24,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +11: [2023-05-11 00:16:24,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +28: [2023-05-11 00:16:24,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 5: [2023-05-11 00:16:24,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 5: [2023-05-11 00:16:24,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +28: [2023-05-11 00:16:24,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +28: [2023-05-11 00:16:24,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +28: [2023-05-11 00:16:24,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +28: [2023-05-11 00:16:24,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +28: [2023-05-11 00:16:24,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +11: [2023-05-11 00:16:24,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +28: [2023-05-11 00:16:24,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +11: [2023-05-11 00:16:24,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +11: [2023-05-11 00:16:24,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +28: [2023-05-11 00:16:24,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +23: [2023-05-11 00:16:24,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +28: [2023-05-11 00:16:24,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +28: [2023-05-11 00:16:24,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +11: [2023-05-11 00:16:24,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +15: [2023-05-11 00:16:24,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +15: [2023-05-11 00:16:24,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +11: [2023-05-11 00:16:24,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +23: [2023-05-11 00:16:24,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +11: [2023-05-11 00:16:24,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +11: [2023-05-11 00:16:24,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 7: [2023-05-11 00:16:24,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +11: [2023-05-11 00:16:24,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +15: [2023-05-11 00:16:24,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +23: [2023-05-11 00:16:24,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +15: [2023-05-11 00:16:24,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +19: [2023-05-11 00:16:24,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 5: [2023-05-11 00:16:24,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +14: [2023-05-11 00:16:24,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 5: [2023-05-11 00:16:24,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +23: [2023-05-11 00:16:24,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +19: [2023-05-11 00:16:24,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 7: [2023-05-11 00:16:24,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 7: [2023-05-11 00:16:24,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 7: [2023-05-11 00:16:24,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 9: [2023-05-11 00:16:24,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 5: [2023-05-11 00:16:24,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +30: [2023-05-11 00:16:24,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 5: [2023-05-11 00:16:24,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 9: [2023-05-11 00:16:24,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +16: [2023-05-11 00:16:24,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 9: [2023-05-11 00:16:24,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 9: [2023-05-11 00:16:24,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +30: [2023-05-11 00:16:24,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +30: [2023-05-11 00:16:24,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +19: [2023-05-11 00:16:24,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +19: [2023-05-11 00:16:24,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +30: [2023-05-11 00:16:24,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +16: [2023-05-11 00:16:24,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +16: [2023-05-11 00:16:24,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +16: [2023-05-11 00:16:24,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +14: [2023-05-11 00:16:24,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +14: [2023-05-11 00:16:24,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 7: [2023-05-11 00:16:24,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +14: [2023-05-11 00:16:24,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +23: [2023-05-11 00:16:24,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +15: [2023-05-11 00:16:24,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +15: [2023-05-11 00:16:24,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +27: [2023-05-11 00:16:24,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +23: [2023-05-11 00:16:24,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 6: [2023-05-11 00:16:24,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 6: [2023-05-11 00:16:24,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 6: [2023-05-11 00:16:24,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 6: [2023-05-11 00:16:24,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 6: [2023-05-11 00:16:24,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 6: [2023-05-11 00:16:24,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 6: [2023-05-11 00:16:24,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 6: [2023-05-11 00:16:24,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +24: [2023-05-11 00:16:24,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +24: [2023-05-11 00:16:24,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +24: [2023-05-11 00:16:24,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +24: [2023-05-11 00:16:24,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +24: [2023-05-11 00:16:24,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +24: [2023-05-11 00:16:24,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +24: [2023-05-11 00:16:24,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +12: [2023-05-11 00:16:24,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +12: [2023-05-11 00:16:24,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +24: [2023-05-11 00:16:24,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +27: [2023-05-11 00:16:24,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +12: [2023-05-11 00:16:24,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +12: [2023-05-11 00:16:24,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +12: [2023-05-11 00:16:24,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +12: [2023-05-11 00:16:24,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +12: [2023-05-11 00:16:24,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +12: [2023-05-11 00:16:24,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +17: [2023-05-11 00:16:24,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +17: [2023-05-11 00:16:24,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +17: [2023-05-11 00:16:24,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +17: [2023-05-11 00:16:24,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +17: [2023-05-11 00:16:24,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +17: [2023-05-11 00:16:24,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +17: [2023-05-11 00:16:24,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +17: [2023-05-11 00:16:24,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +26: [2023-05-11 00:16:24,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +26: [2023-05-11 00:16:24,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +26: [2023-05-11 00:16:24,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +26: [2023-05-11 00:16:24,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +26: [2023-05-11 00:16:24,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +26: [2023-05-11 00:16:24,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +26: [2023-05-11 00:16:24,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +26: [2023-05-11 00:16:24,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 7: [2023-05-11 00:16:24,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 7: [2023-05-11 00:16:24,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +19: [2023-05-11 00:16:24,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +12: [2023-05-11 00:16:24,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +19: [2023-05-11 00:16:24,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +23: [2023-05-11 00:16:24,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +24: [2023-05-11 00:16:24,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +24: [2023-05-11 00:16:24,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +24: [2023-05-11 00:16:24,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +12: [2023-05-11 00:16:24,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +17: [2023-05-11 00:16:24,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +17: [2023-05-11 00:16:24,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +12: [2023-05-11 00:16:24,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +12: [2023-05-11 00:16:24,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +12: [2023-05-11 00:16:24,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +12: [2023-05-11 00:16:24,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +17: [2023-05-11 00:16:24,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +12: [2023-05-11 00:16:24,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +17: [2023-05-11 00:16:24,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +14: [2023-05-11 00:16:24,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +12: [2023-05-11 00:16:24,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +27: [2023-05-11 00:16:24,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +27: [2023-05-11 00:16:24,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 6: [2023-05-11 00:16:24,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 6: [2023-05-11 00:16:24,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 6: [2023-05-11 00:16:24,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +30: [2023-05-11 00:16:24,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 7: [2023-05-11 00:16:24,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +17: [2023-05-11 00:16:24,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +26: [2023-05-11 00:16:24,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +17: [2023-05-11 00:16:24,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +19: [2023-05-11 00:16:24,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +17: [2023-05-11 00:16:24,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +17: [2023-05-11 00:16:24,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +24: [2023-05-11 00:16:24,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +24: [2023-05-11 00:16:24,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 6: [2023-05-11 00:16:24,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +30: [2023-05-11 00:16:24,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +30: [2023-05-11 00:16:24,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +24: [2023-05-11 00:16:24,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +26: [2023-05-11 00:16:24,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 6: [2023-05-11 00:16:24,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +19: [2023-05-11 00:16:24,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +26: [2023-05-11 00:16:24,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 6: [2023-05-11 00:16:24,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 6: [2023-05-11 00:16:24,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 6: [2023-05-11 00:16:24,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +16: [2023-05-11 00:16:24,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +24: [2023-05-11 00:16:24,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +23: [2023-05-11 00:16:24,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +24: [2023-05-11 00:16:24,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +30: [2023-05-11 00:16:24,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +16: [2023-05-11 00:16:24,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +26: [2023-05-11 00:16:24,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +16: [2023-05-11 00:16:24,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +26: [2023-05-11 00:16:24,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +26: [2023-05-11 00:16:24,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +26: [2023-05-11 00:16:24,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +26: [2023-05-11 00:16:24,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +16: [2023-05-11 00:16:24,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 8: [2023-05-11 00:16:24,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 8: [2023-05-11 00:16:24,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 8: [2023-05-11 00:16:24,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 8: [2023-05-11 00:16:24,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 8: [2023-05-11 00:16:24,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 8: [2023-05-11 00:16:24,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 8: [2023-05-11 00:16:24,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 8: [2023-05-11 00:16:24,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +14: [2023-05-11 00:16:24,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +14: [2023-05-11 00:16:24,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +14: [2023-05-11 00:16:24,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +31: [2023-05-11 00:16:24,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +31: [2023-05-11 00:16:24,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +31: [2023-05-11 00:16:24,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +31: [2023-05-11 00:16:24,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +31: [2023-05-11 00:16:24,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +31: [2023-05-11 00:16:24,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +31: [2023-05-11 00:16:24,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +31: [2023-05-11 00:16:24,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +25: [2023-05-11 00:16:24,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +25: [2023-05-11 00:16:24,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +25: [2023-05-11 00:16:24,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +25: [2023-05-11 00:16:24,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +25: [2023-05-11 00:16:24,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +25: [2023-05-11 00:16:24,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +25: [2023-05-11 00:16:24,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +25: [2023-05-11 00:16:24,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 8: [2023-05-11 00:16:24,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 8: [2023-05-11 00:16:24,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +11: [2023-05-11 00:16:24,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 8: [2023-05-11 00:16:24,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 8: [2023-05-11 00:16:24,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 2: [2023-05-11 00:16:24,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 2: [2023-05-11 00:16:24,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 2: [2023-05-11 00:16:24,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 2: [2023-05-11 00:16:24,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 2: [2023-05-11 00:16:24,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 2: [2023-05-11 00:16:24,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 2: [2023-05-11 00:16:24,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 2: [2023-05-11 00:16:24,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +28: [2023-05-11 00:16:24,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 8: [2023-05-11 00:16:24,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 8: [2023-05-11 00:16:24,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +13: [2023-05-11 00:16:24,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +13: [2023-05-11 00:16:24,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +13: [2023-05-11 00:16:24,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +13: [2023-05-11 00:16:24,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +13: [2023-05-11 00:16:24,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +13: [2023-05-11 00:16:24,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +13: [2023-05-11 00:16:24,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 8: [2023-05-11 00:16:24,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 8: [2023-05-11 00:16:24,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +13: [2023-05-11 00:16:24,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +22: [2023-05-11 00:16:24,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +22: [2023-05-11 00:16:24,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +22: [2023-05-11 00:16:24,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +22: [2023-05-11 00:16:24,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +22: [2023-05-11 00:16:24,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +22: [2023-05-11 00:16:24,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +22: [2023-05-11 00:16:24,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +22: [2023-05-11 00:16:24,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +28: [2023-05-11 00:16:24,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +28: [2023-05-11 00:16:24,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +28: [2023-05-11 00:16:24,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +18: [2023-05-11 00:16:24,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +18: [2023-05-11 00:16:24,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +11: [2023-05-11 00:16:24,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +18: [2023-05-11 00:16:24,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +18: [2023-05-11 00:16:24,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +18: [2023-05-11 00:16:24,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +18: [2023-05-11 00:16:24,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +18: [2023-05-11 00:16:24,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 3: [2023-05-11 00:16:24,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +18: [2023-05-11 00:16:24,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 3: [2023-05-11 00:16:24,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 3: [2023-05-11 00:16:24,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 3: [2023-05-11 00:16:24,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 3: [2023-05-11 00:16:24,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 3: [2023-05-11 00:16:24,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 3: [2023-05-11 00:16:24,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 2: [2023-05-11 00:16:24,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 2: [2023-05-11 00:16:24,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 3: [2023-05-11 00:16:24,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +31: [2023-05-11 00:16:24,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 2: [2023-05-11 00:16:24,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +13: [2023-05-11 00:16:24,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +13: [2023-05-11 00:16:24,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +11: [2023-05-11 00:16:24,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +11: [2023-05-11 00:16:24,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +13: [2023-05-11 00:16:24,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +13: [2023-05-11 00:16:24,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +25: [2023-05-11 00:16:24,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +31: [2023-05-11 00:16:24,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +13: [2023-05-11 00:16:24,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +31: [2023-05-11 00:16:24,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +25: [2023-05-11 00:16:24,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +13: [2023-05-11 00:16:24,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +13: [2023-05-11 00:16:24,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +25: [2023-05-11 00:16:24,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +25: [2023-05-11 00:16:24,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +13: [2023-05-11 00:16:24,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 2: [2023-05-11 00:16:24,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 2: [2023-05-11 00:16:24,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 2: [2023-05-11 00:16:24,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 2: [2023-05-11 00:16:24,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 2: [2023-05-11 00:16:24,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 3: [2023-05-11 00:16:24,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +22: [2023-05-11 00:16:24,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +22: [2023-05-11 00:16:24,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +22: [2023-05-11 00:16:24,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 3: [2023-05-11 00:16:24,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 3: [2023-05-11 00:16:24,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +22: [2023-05-11 00:16:24,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 3: [2023-05-11 00:16:24,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +18: [2023-05-11 00:16:24,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 3: [2023-05-11 00:16:24,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 3: [2023-05-11 00:16:24,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +18: [2023-05-11 00:16:24,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +18: [2023-05-11 00:16:24,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +25: [2023-05-11 00:16:24,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +31: [2023-05-11 00:16:24,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +31: [2023-05-11 00:16:24,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +22: [2023-05-11 00:16:24,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +22: [2023-05-11 00:16:24,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 3: [2023-05-11 00:16:24,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 3: [2023-05-11 00:16:24,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +31: [2023-05-11 00:16:24,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +25: [2023-05-11 00:16:24,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +18: [2023-05-11 00:16:24,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +18: [2023-05-11 00:16:24,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +22: [2023-05-11 00:16:24,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +18: [2023-05-11 00:16:24,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +31: [2023-05-11 00:16:24,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +18: [2023-05-11 00:16:24,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +31: [2023-05-11 00:16:24,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +18: [2023-05-11 00:16:24,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +25: [2023-05-11 00:16:24,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 1: [2023-05-11 00:16:24,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +25: [2023-05-11 00:16:24,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 1: [2023-05-11 00:16:24,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 1: [2023-05-11 00:16:24,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 1: [2023-05-11 00:16:24,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 1: [2023-05-11 00:16:24,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 1: [2023-05-11 00:16:24,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 1: [2023-05-11 00:16:24,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 1: [2023-05-11 00:16:24,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +20: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +20: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +20: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +20: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +20: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +22: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +21: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +20: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +20: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +21: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +21: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +21: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +21: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +21: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +21: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +20: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +21: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +29: [2023-05-11 00:16:24,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +29: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +29: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +29: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +29: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +29: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +29: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +29: [2023-05-11 00:16:24,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +20: [2023-05-11 00:16:24,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +20: [2023-05-11 00:16:24,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +20: [2023-05-11 00:16:24,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +29: [2023-05-11 00:16:24,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +29: [2023-05-11 00:16:24,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +29: [2023-05-11 00:16:24,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +29: [2023-05-11 00:16:24,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +29: [2023-05-11 00:16:24,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +29: [2023-05-11 00:16:24,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +29: [2023-05-11 00:16:24,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +20: [2023-05-11 00:16:24,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +29: [2023-05-11 00:16:24,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +21: [2023-05-11 00:16:24,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +20: [2023-05-11 00:16:24,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +20: [2023-05-11 00:16:24,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +20: [2023-05-11 00:16:24,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +20: [2023-05-11 00:16:24,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +21: [2023-05-11 00:16:24,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +21: [2023-05-11 00:16:24,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +21: [2023-05-11 00:16:24,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 1: [2023-05-11 00:16:24,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 1: [2023-05-11 00:16:24,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 1: [2023-05-11 00:16:24,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 1: [2023-05-11 00:16:24,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 1: [2023-05-11 00:16:24,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 1: [2023-05-11 00:16:24,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 1: [2023-05-11 00:16:24,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 1: [2023-05-11 00:16:24,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +21: [2023-05-11 00:16:24,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +21: [2023-05-11 00:16:24,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 0: [2023-05-11 00:16:24,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 0: [2023-05-11 00:16:24,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 0: [2023-05-11 00:16:24,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 0: [2023-05-11 00:16:24,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 0: [2023-05-11 00:16:24,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 0: [2023-05-11 00:16:24,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 0: [2023-05-11 00:16:24,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 0: [2023-05-11 00:16:24,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +21: [2023-05-11 00:16:24,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +28: [2023-05-11 00:16:24,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +21: [2023-05-11 00:16:24,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +28: [2023-05-11 00:16:24,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +11: [2023-05-11 00:16:24,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +11: [2023-05-11 00:16:24,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +11: [2023-05-11 00:16:24,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +24: [2023-05-11 00:16:24,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +12: [2023-05-11 00:16:24,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +28: [2023-05-11 00:16:24,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +28: [2023-05-11 00:16:24,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +17: [2023-05-11 00:16:24,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 0: [2023-05-11 00:16:24,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 0: [2023-05-11 00:16:24,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 0: [2023-05-11 00:16:24,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... +27: [2023-05-11 00:16:24,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +27: [2023-05-11 00:16:24,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 0: [2023-05-11 00:16:24,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 0: [2023-05-11 00:16:24,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +17: [2023-05-11 00:16:24,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 0: [2023-05-11 00:16:24,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... + 0: [2023-05-11 00:16:24,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt... + 0: [2023-05-11 00:16:24,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt... +12: [2023-05-11 00:16:24,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +12: [2023-05-11 00:16:24,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +11: [2023-05-11 00:16:24,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +12: [2023-05-11 00:16:24,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 6: [2023-05-11 00:16:24,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 6: [2023-05-11 00:16:24,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 6: [2023-05-11 00:16:24,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +24: [2023-05-11 00:16:24,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +24: [2023-05-11 00:16:24,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +24: [2023-05-11 00:16:24,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +27: [2023-05-11 00:16:24,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +27: [2023-05-11 00:16:24,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +17: [2023-05-11 00:16:24,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +17: [2023-05-11 00:16:24,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +26: [2023-05-11 00:16:24,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +26: [2023-05-11 00:16:24,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +26: [2023-05-11 00:16:24,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 6: [2023-05-11 00:16:24,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +26: [2023-05-11 00:16:24,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +24: [2023-05-11 00:16:24,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 8: [2023-05-11 00:16:24,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +17: [2023-05-11 00:16:24,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 8: [2023-05-11 00:16:24,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +17: [2023-05-11 00:16:24,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +12: [2023-05-11 00:16:24,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +12: [2023-05-11 00:16:24,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +12: [2023-05-11 00:16:24,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 2: [2023-05-11 00:16:24,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 2: [2023-05-11 00:16:24,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +12: [2023-05-11 00:16:24,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +24: [2023-05-11 00:16:24,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 3: [2023-05-11 00:16:24,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +13: [2023-05-11 00:16:24,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +13: [2023-05-11 00:16:24,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +13: [2023-05-11 00:16:24,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 2: [2023-05-11 00:16:24,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 8: [2023-05-11 00:16:24,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +18: [2023-05-11 00:16:24,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 8: [2023-05-11 00:16:24,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 2: [2023-05-11 00:16:24,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 6: [2023-05-11 00:16:24,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +25: [2023-05-11 00:16:24,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +24: [2023-05-11 00:16:24,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +24: [2023-05-11 00:16:24,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +31: [2023-05-11 00:16:24,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +17: [2023-05-11 00:16:24,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +22: [2023-05-11 00:16:24,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +22: [2023-05-11 00:16:24,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 6: [2023-05-11 00:16:24,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 6: [2023-05-11 00:16:24,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +17: [2023-05-11 00:16:24,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +18: [2023-05-11 00:16:24,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +13: [2023-05-11 00:16:24,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +18: [2023-05-11 00:16:24,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +18: [2023-05-11 00:16:24,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 3: [2023-05-11 00:16:24,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 3: [2023-05-11 00:16:24,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 3: [2023-05-11 00:16:24,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 8: [2023-05-11 00:16:24,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +31: [2023-05-11 00:16:24,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +31: [2023-05-11 00:16:24,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 8: [2023-05-11 00:16:24,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +31: [2023-05-11 00:16:24,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +22: [2023-05-11 00:16:24,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +20: [2023-05-11 00:16:24,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +20: [2023-05-11 00:16:24,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +22: [2023-05-11 00:16:24,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +26: [2023-05-11 00:16:24,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +26: [2023-05-11 00:16:24,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +29: [2023-05-11 00:16:24,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +25: [2023-05-11 00:16:24,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 6: [2023-05-11 00:16:24,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +21: [2023-05-11 00:16:24,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +26: [2023-05-11 00:16:24,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +26: [2023-05-11 00:16:24,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 2: [2023-05-11 00:16:24,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 2: [2023-05-11 00:16:24,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +20: [2023-05-11 00:16:24,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +20: [2023-05-11 00:16:24,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +13: [2023-05-11 00:16:24,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +29: [2023-05-11 00:16:24,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +29: [2023-05-11 00:16:24,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +25: [2023-05-11 00:16:24,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +25: [2023-05-11 00:16:24,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +18: [2023-05-11 00:16:24,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 8: [2023-05-11 00:16:24,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 8: [2023-05-11 00:16:24,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +25: [2023-05-11 00:16:24,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +29: [2023-05-11 00:16:24,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +18: [2023-05-11 00:16:24,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 2: [2023-05-11 00:16:24,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +13: [2023-05-11 00:16:24,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +13: [2023-05-11 00:16:24,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +22: [2023-05-11 00:16:24,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +22: [2023-05-11 00:16:24,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +21: [2023-05-11 00:16:24,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +21: [2023-05-11 00:16:24,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +21: [2023-05-11 00:16:24,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 2: [2023-05-11 00:16:24,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 3: [2023-05-11 00:16:24,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 1: [2023-05-11 00:16:24,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 1: [2023-05-11 00:16:24,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 1: [2023-05-11 00:16:24,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 3: [2023-05-11 00:16:24,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +31: [2023-05-11 00:16:24,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 3: [2023-05-11 00:16:24,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 3: [2023-05-11 00:16:24,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +18: [2023-05-11 00:16:24,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +18: [2023-05-11 00:16:24,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +20: [2023-05-11 00:16:24,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +20: [2023-05-11 00:16:24,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +13: [2023-05-11 00:16:24,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +29: [2023-05-11 00:16:24,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +21: [2023-05-11 00:16:24,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +22: [2023-05-11 00:16:24,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +31: [2023-05-11 00:16:24,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +31: [2023-05-11 00:16:24,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 1: [2023-05-11 00:16:24,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +20: [2023-05-11 00:16:24,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +20: [2023-05-11 00:16:24,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 0: [2023-05-11 00:16:24,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 0: [2023-05-11 00:16:24,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 0: [2023-05-11 00:16:24,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. + 0: [2023-05-11 00:16:24,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt. +25: [2023-05-11 00:16:24,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +29: [2023-05-11 00:16:24,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +29: [2023-05-11 00:16:24,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +22: [2023-05-11 00:16:24,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +29: [2023-05-11 00:16:24,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +31: [2023-05-11 00:16:24,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +21: [2023-05-11 00:16:24,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +21: [2023-05-11 00:16:24,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +21: [2023-05-11 00:16:24,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 1: [2023-05-11 00:16:24,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 1: [2023-05-11 00:16:24,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +25: [2023-05-11 00:16:24,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 1: [2023-05-11 00:16:24,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 1: [2023-05-11 00:16:24,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +25: [2023-05-11 00:16:24,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 0: [2023-05-11 00:16:24,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 0: [2023-05-11 00:16:24,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 0: [2023-05-11 00:16:24,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 0: [2023-05-11 00:16:24,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +15: [2023-05-11 00:16:25,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +15: [2023-05-11 00:16:25,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +28: [2023-05-11 00:16:25,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +28: [2023-05-11 00:16:25,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +28: [2023-05-11 00:16:25,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +28: [2023-05-11 00:16:25,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +15: [2023-05-11 00:16:25,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +28: [2023-05-11 00:16:25,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +15: [2023-05-11 00:16:25,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +28: [2023-05-11 00:16:25,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +28: [2023-05-11 00:16:25,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +28: [2023-05-11 00:16:25,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +14: [2023-05-11 00:16:25,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +14: [2023-05-11 00:16:25,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +14: [2023-05-11 00:16:25,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +14: [2023-05-11 00:16:25,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 9: [2023-05-11 00:16:25,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 9: [2023-05-11 00:16:25,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 9: [2023-05-11 00:16:25,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 9: [2023-05-11 00:16:25,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +11: [2023-05-11 00:16:25,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +11: [2023-05-11 00:16:25,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 7: [2023-05-11 00:16:25,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 7: [2023-05-11 00:16:25,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +30: [2023-05-11 00:16:25,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 7: [2023-05-11 00:16:25,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +30: [2023-05-11 00:16:25,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +30: [2023-05-11 00:16:25,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 7: [2023-05-11 00:16:25,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +30: [2023-05-11 00:16:25,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +11: [2023-05-11 00:16:25,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +11: [2023-05-11 00:16:25,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +15: [2023-05-11 00:16:25,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +15: [2023-05-11 00:16:25,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +15: [2023-05-11 00:16:25,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +15: [2023-05-11 00:16:25,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +14: [2023-05-11 00:16:25,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +14: [2023-05-11 00:16:25,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +11: [2023-05-11 00:16:25,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +11: [2023-05-11 00:16:25,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +14: [2023-05-11 00:16:25,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +30: [2023-05-11 00:16:25,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +11: [2023-05-11 00:16:25,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +30: [2023-05-11 00:16:25,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +30: [2023-05-11 00:16:25,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +30: [2023-05-11 00:16:25,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 7: [2023-05-11 00:16:25,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +28: [2023-05-11 00:16:25,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +28: [2023-05-11 00:16:25,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 7: [2023-05-11 00:16:25,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +11: [2023-05-11 00:16:25,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +28: [2023-05-11 00:16:25,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +28: [2023-05-11 00:16:25,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +23: [2023-05-11 00:16:25,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +23: [2023-05-11 00:16:25,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +23: [2023-05-11 00:16:25,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +14: [2023-05-11 00:16:25,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +23: [2023-05-11 00:16:25,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 7: [2023-05-11 00:16:25,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +11: [2023-05-11 00:16:25,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +11: [2023-05-11 00:16:25,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +11: [2023-05-11 00:16:25,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +11: [2023-05-11 00:16:25,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +28: [2023-05-11 00:16:25,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +23: [2023-05-11 00:16:25,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +10: [2023-05-11 00:16:25,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +10: [2023-05-11 00:16:25,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +10: [2023-05-11 00:16:25,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +10: [2023-05-11 00:16:25,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +23: [2023-05-11 00:16:25,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 4: [2023-05-11 00:16:25,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 4: [2023-05-11 00:16:25,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 4: [2023-05-11 00:16:25,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +23: [2023-05-11 00:16:25,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 4: [2023-05-11 00:16:25,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +11: [2023-05-11 00:16:25,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +11: [2023-05-11 00:16:25,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +11: [2023-05-11 00:16:25,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +28: [2023-05-11 00:16:25,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +11: [2023-05-11 00:16:25,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +10: [2023-05-11 00:16:25,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +23: [2023-05-11 00:16:25,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +23: [2023-05-11 00:16:25,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +10: [2023-05-11 00:16:25,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +10: [2023-05-11 00:16:25,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +10: [2023-05-11 00:16:25,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +28: [2023-05-11 00:16:25,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +28: [2023-05-11 00:16:25,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 6: [2023-05-11 00:16:25,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 6: [2023-05-11 00:16:25,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +23: [2023-05-11 00:16:25,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +19: [2023-05-11 00:16:25,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 6: [2023-05-11 00:16:25,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +19: [2023-05-11 00:16:25,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +29: [2023-05-11 00:16:25,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +29: [2023-05-11 00:16:25,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +29: [2023-05-11 00:16:25,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 0: [2023-05-11 00:16:25,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 0: [2023-05-11 00:16:25,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 0: [2023-05-11 00:16:25,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +19: [2023-05-11 00:16:25,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 0: [2023-05-11 00:16:25,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +19: [2023-05-11 00:16:25,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +17: [2023-05-11 00:16:25,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +17: [2023-05-11 00:16:25,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +17: [2023-05-11 00:16:25,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 5: [2023-05-11 00:16:25,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 5: [2023-05-11 00:16:25,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +23: [2023-05-11 00:16:25,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 5: [2023-05-11 00:16:25,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +25: [2023-05-11 00:16:25,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +25: [2023-05-11 00:16:25,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 5: [2023-05-11 00:16:25,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +29: [2023-05-11 00:16:25,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +21: [2023-05-11 00:16:25,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +27: [2023-05-11 00:16:25,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +21: [2023-05-11 00:16:25,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +21: [2023-05-11 00:16:25,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +13: [2023-05-11 00:16:25,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +13: [2023-05-11 00:16:25,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +13: [2023-05-11 00:16:25,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +13: [2023-05-11 00:16:25,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +10: [2023-05-11 00:16:25,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +17: [2023-05-11 00:16:25,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +27: [2023-05-11 00:16:25,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +10: [2023-05-11 00:16:25,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +25: [2023-05-11 00:16:25,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +26: [2023-05-11 00:16:25,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +26: [2023-05-11 00:16:25,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +26: [2023-05-11 00:16:25,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +10: [2023-05-11 00:16:25,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +25: [2023-05-11 00:16:25,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +26: [2023-05-11 00:16:25,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 2: [2023-05-11 00:16:25,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 2: [2023-05-11 00:16:25,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 2: [2023-05-11 00:16:25,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +10: [2023-05-11 00:16:25,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +27: [2023-05-11 00:16:25,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 2: [2023-05-11 00:16:25,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +21: [2023-05-11 00:16:25,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +24: [2023-05-11 00:16:25,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +27: [2023-05-11 00:16:25,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +24: [2023-05-11 00:16:25,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +24: [2023-05-11 00:16:25,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +28: [2023-05-11 00:16:25,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +28: [2023-05-11 00:16:25,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +28: [2023-05-11 00:16:25,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +28: [2023-05-11 00:16:25,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 3: [2023-05-11 00:16:25,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 3: [2023-05-11 00:16:25,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +24: [2023-05-11 00:16:25,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +31: [2023-05-11 00:16:25,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +23: [2023-05-11 00:16:25,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 3: [2023-05-11 00:16:25,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 8: [2023-05-11 00:16:25,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 8: [2023-05-11 00:16:25,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 8: [2023-05-11 00:16:25,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +31: [2023-05-11 00:16:25,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 8: [2023-05-11 00:16:25,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +31: [2023-05-11 00:16:25,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +31: [2023-05-11 00:16:25,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +20: [2023-05-11 00:16:25,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +20: [2023-05-11 00:16:25,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +20: [2023-05-11 00:16:25,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +20: [2023-05-11 00:16:25,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +24: [2023-05-11 00:16:25,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +24: [2023-05-11 00:16:25,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +24: [2023-05-11 00:16:25,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +24: [2023-05-11 00:16:25,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +16: [2023-05-11 00:16:25,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +16: [2023-05-11 00:16:25,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +11: [2023-05-11 00:16:25,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +11: [2023-05-11 00:16:25,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +11: [2023-05-11 00:16:25,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +24: [2023-05-11 00:16:25,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +23: [2023-05-11 00:16:25,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +23: [2023-05-11 00:16:25,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +24: [2023-05-11 00:16:25,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 1: [2023-05-11 00:16:25,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 1: [2023-05-11 00:16:25,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 1: [2023-05-11 00:16:25,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +16: [2023-05-11 00:16:25,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +24: [2023-05-11 00:16:25,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 1: [2023-05-11 00:16:25,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +16: [2023-05-11 00:16:25,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +24: [2023-05-11 00:16:25,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +11: [2023-05-11 00:16:25,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +11: [2023-05-11 00:16:25,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +12: [2023-05-11 00:16:25,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +12: [2023-05-11 00:16:25,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +12: [2023-05-11 00:16:25,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +16: [2023-05-11 00:16:25,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +11: [2023-05-11 00:16:25,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +16: [2023-05-11 00:16:25,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +12: [2023-05-11 00:16:25,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +22: [2023-05-11 00:16:25,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +22: [2023-05-11 00:16:25,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +22: [2023-05-11 00:16:25,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +22: [2023-05-11 00:16:25,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +29: [2023-05-11 00:16:25,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +11: [2023-05-11 00:16:25,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +16: [2023-05-11 00:16:25,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +16: [2023-05-11 00:16:25,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +19: [2023-05-11 00:16:25,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +29: [2023-05-11 00:16:25,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +15: [2023-05-11 00:16:25,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +15: [2023-05-11 00:16:25,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 0: [2023-05-11 00:16:25,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +17: [2023-05-11 00:16:25,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 5: [2023-05-11 00:16:25,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 5: [2023-05-11 00:16:25,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 5: [2023-05-11 00:16:25,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +29: [2023-05-11 00:16:25,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +15: [2023-05-11 00:16:25,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +15: [2023-05-11 00:16:25,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +15: [2023-05-11 00:16:25,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +15: [2023-05-11 00:16:25,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +15: [2023-05-11 00:16:25,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +15: [2023-05-11 00:16:25,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +25: [2023-05-11 00:16:25,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +13: [2023-05-11 00:16:25,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +29: [2023-05-11 00:16:25,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +17: [2023-05-11 00:16:25,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +26: [2023-05-11 00:16:25,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +13: [2023-05-11 00:16:25,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +11: [2023-05-11 00:16:25,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 0: [2023-05-11 00:16:25,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +19: [2023-05-11 00:16:25,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +17: [2023-05-11 00:16:25,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +19: [2023-05-11 00:16:25,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +19: [2023-05-11 00:16:25,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +26: [2023-05-11 00:16:25,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +15: [2023-05-11 00:16:25,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +15: [2023-05-11 00:16:25,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +15: [2023-05-11 00:16:25,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +16: [2023-05-11 00:16:25,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +16: [2023-05-11 00:16:25,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +16: [2023-05-11 00:16:25,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +16: [2023-05-11 00:16:25,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 5: [2023-05-11 00:16:25,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +15: [2023-05-11 00:16:25,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +15: [2023-05-11 00:16:25,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +15: [2023-05-11 00:16:25,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +20: [2023-05-11 00:16:25,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +15: [2023-05-11 00:16:25,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 6: [2023-05-11 00:16:25,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 3: [2023-05-11 00:16:25,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 3: [2023-05-11 00:16:25,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 3: [2023-05-11 00:16:25,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 3: [2023-05-11 00:16:25,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +28: [2023-05-11 00:16:25,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +28: [2023-05-11 00:16:25,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +13: [2023-05-11 00:16:25,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +13: [2023-05-11 00:16:25,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +21: [2023-05-11 00:16:25,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +15: [2023-05-11 00:16:25,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +21: [2023-05-11 00:16:25,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +21: [2023-05-11 00:16:25,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +25: [2023-05-11 00:16:25,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +20: [2023-05-11 00:16:25,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +21: [2023-05-11 00:16:25,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +27: [2023-05-11 00:16:25,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +27: [2023-05-11 00:16:25,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +26: [2023-05-11 00:16:25,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +28: [2023-05-11 00:16:25,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +26: [2023-05-11 00:16:25,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +31: [2023-05-11 00:16:25,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +28: [2023-05-11 00:16:25,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +31: [2023-05-11 00:16:25,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +17: [2023-05-11 00:16:25,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +20: [2023-05-11 00:16:25,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +27: [2023-05-11 00:16:25,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +31: [2023-05-11 00:16:25,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +11: [2023-05-11 00:16:25,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +10: [2023-05-11 00:16:25,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +11: [2023-05-11 00:16:25,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +27: [2023-05-11 00:16:25,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +20: [2023-05-11 00:16:25,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 8: [2023-05-11 00:16:25,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 8: [2023-05-11 00:16:25,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +24: [2023-05-11 00:16:25,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +12: [2023-05-11 00:16:25,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +11: [2023-05-11 00:16:25,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +10: [2023-05-11 00:16:25,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +10: [2023-05-11 00:16:25,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +12: [2023-05-11 00:16:25,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +11: [2023-05-11 00:16:25,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 4: [2023-05-11 00:16:25,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 4: [2023-05-11 00:16:25,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +12: [2023-05-11 00:16:25,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 1: [2023-05-11 00:16:25,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 8: [2023-05-11 00:16:25,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +31: [2023-05-11 00:16:25,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +25: [2023-05-11 00:16:25,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +12: [2023-05-11 00:16:25,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +25: [2023-05-11 00:16:25,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 8: [2023-05-11 00:16:25,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +22: [2023-05-11 00:16:25,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +22: [2023-05-11 00:16:25,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +24: [2023-05-11 00:16:25,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 1: [2023-05-11 00:16:25,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +23: [2023-05-11 00:16:25,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +23: [2023-05-11 00:16:25,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +10: [2023-05-11 00:16:25,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +10: [2023-05-11 00:16:25,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +24: [2023-05-11 00:16:25,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +10: [2023-05-11 00:16:25,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +24: [2023-05-11 00:16:25,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +16: [2023-05-11 00:16:25,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +16: [2023-05-11 00:16:25,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +23: [2023-05-11 00:16:25,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +23: [2023-05-11 00:16:25,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +22: [2023-05-11 00:16:25,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +22: [2023-05-11 00:16:25,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +23: [2023-05-11 00:16:25,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +11: [2023-05-11 00:16:25,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +10: [2023-05-11 00:16:25,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +11: [2023-05-11 00:16:25,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +11: [2023-05-11 00:16:25,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +16: [2023-05-11 00:16:25,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +16: [2023-05-11 00:16:25,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +10: [2023-05-11 00:16:25,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +15: [2023-05-11 00:16:25,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +11: [2023-05-11 00:16:25,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +23: [2023-05-11 00:16:25,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +24: [2023-05-11 00:16:25,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +12: [2023-05-11 00:16:25,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +12: [2023-05-11 00:16:25,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +12: [2023-05-11 00:16:25,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +12: [2023-05-11 00:16:25,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +15: [2023-05-11 00:16:25,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 4: [2023-05-11 00:16:25,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +12: [2023-05-11 00:16:25,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +12: [2023-05-11 00:16:25,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +10: [2023-05-11 00:16:25,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 9: [2023-05-11 00:16:25,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 9: [2023-05-11 00:16:25,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +12: [2023-05-11 00:16:25,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +15: [2023-05-11 00:16:25,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +15: [2023-05-11 00:16:25,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +12: [2023-05-11 00:16:25,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +23: [2023-05-11 00:16:25,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +23: [2023-05-11 00:16:25,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +23: [2023-05-11 00:16:25,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +28: [2023-05-11 00:16:25,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +16: [2023-05-11 00:16:25,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +28: [2023-05-11 00:16:25,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +16: [2023-05-11 00:16:25,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +16: [2023-05-11 00:16:25,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +28: [2023-05-11 00:16:25,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +16: [2023-05-11 00:16:25,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +28: [2023-05-11 00:16:25,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 4: [2023-05-11 00:16:25,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +28: [2023-05-11 00:16:25,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +10: [2023-05-11 00:16:25,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +28: [2023-05-11 00:16:25,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +28: [2023-05-11 00:16:25,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +28: [2023-05-11 00:16:25,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +24: [2023-05-11 00:16:25,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +24: [2023-05-11 00:16:25,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +24: [2023-05-11 00:16:25,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +15: [2023-05-11 00:16:25,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +24: [2023-05-11 00:16:25,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +23: [2023-05-11 00:16:25,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +15: [2023-05-11 00:16:25,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +12: [2023-05-11 00:16:25,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +10: [2023-05-11 00:16:25,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +12: [2023-05-11 00:16:25,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +12: [2023-05-11 00:16:25,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +12: [2023-05-11 00:16:25,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 4: [2023-05-11 00:16:25,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 4: [2023-05-11 00:16:25,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 4: [2023-05-11 00:16:25,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +18: [2023-05-11 00:16:25,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +18: [2023-05-11 00:16:25,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +18: [2023-05-11 00:16:25,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +18: [2023-05-11 00:16:25,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. + 4: [2023-05-11 00:16:25,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt. +24: [2023-05-11 00:16:25,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +24: [2023-05-11 00:16:25,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +12: [2023-05-11 00:16:25,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +12: [2023-05-11 00:16:25,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +12: [2023-05-11 00:16:25,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +12: [2023-05-11 00:16:25,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +15: [2023-05-11 00:16:25,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +15: [2023-05-11 00:16:25,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +16: [2023-05-11 00:16:25,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +10: [2023-05-11 00:16:25,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +10: [2023-05-11 00:16:25,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +29: [2023-05-11 00:16:25,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +29: [2023-05-11 00:16:25,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +29: [2023-05-11 00:16:25,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +29: [2023-05-11 00:16:25,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +29: [2023-05-11 00:16:25,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +29: [2023-05-11 00:16:25,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +29: [2023-05-11 00:16:25,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +29: [2023-05-11 00:16:25,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +16: [2023-05-11 00:16:25,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +16: [2023-05-11 00:16:25,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +24: [2023-05-11 00:16:25,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +24: [2023-05-11 00:16:25,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +16: [2023-05-11 00:16:25,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +16: [2023-05-11 00:16:25,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +16: [2023-05-11 00:16:25,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +29: [2023-05-11 00:16:25,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +29: [2023-05-11 00:16:25,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +29: [2023-05-11 00:16:25,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +29: [2023-05-11 00:16:25,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +29: [2023-05-11 00:16:25,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +29: [2023-05-11 00:16:25,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +29: [2023-05-11 00:16:25,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +10: [2023-05-11 00:16:25,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +29: [2023-05-11 00:16:25,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +10: [2023-05-11 00:16:25,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +10: [2023-05-11 00:16:25,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +10: [2023-05-11 00:16:25,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +16: [2023-05-11 00:16:25,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +16: [2023-05-11 00:16:25,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +24: [2023-05-11 00:16:25,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +24: [2023-05-11 00:16:25,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +24: [2023-05-11 00:16:25,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +24: [2023-05-11 00:16:25,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +18: [2023-05-11 00:16:25,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +10: [2023-05-11 00:16:25,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +10: [2023-05-11 00:16:25,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +18: [2023-05-11 00:16:25,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +24: [2023-05-11 00:16:25,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +10: [2023-05-11 00:16:25,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +24: [2023-05-11 00:16:25,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +18: [2023-05-11 00:16:25,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +18: [2023-05-11 00:16:25,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +16: [2023-05-11 00:16:25,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +16: [2023-05-11 00:16:25,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 4: [2023-05-11 00:16:25,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +12: [2023-05-11 00:16:25,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 4: [2023-05-11 00:16:25,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +24: [2023-05-11 00:16:25,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 4: [2023-05-11 00:16:25,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +16: [2023-05-11 00:16:25,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +16: [2023-05-11 00:16:25,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +12: [2023-05-11 00:16:25,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +12: [2023-05-11 00:16:25,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +12: [2023-05-11 00:16:25,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 4: [2023-05-11 00:16:25,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +18: [2023-05-11 00:16:25,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +18: [2023-05-11 00:16:25,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +18: [2023-05-11 00:16:25,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +18: [2023-05-11 00:16:25,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +18: [2023-05-11 00:16:25,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +18: [2023-05-11 00:16:25,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +18: [2023-05-11 00:16:25,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +18: [2023-05-11 00:16:25,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +27: [2023-05-11 00:16:25,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +27: [2023-05-11 00:16:25,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +27: [2023-05-11 00:16:25,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +27: [2023-05-11 00:16:25,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +27: [2023-05-11 00:16:25,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +27: [2023-05-11 00:16:25,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +27: [2023-05-11 00:16:25,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +27: [2023-05-11 00:16:25,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +27: [2023-05-11 00:16:25,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +27: [2023-05-11 00:16:25,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +27: [2023-05-11 00:16:25,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +12: [2023-05-11 00:16:25,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +27: [2023-05-11 00:16:25,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +27: [2023-05-11 00:16:25,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +27: [2023-05-11 00:16:25,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +27: [2023-05-11 00:16:25,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +27: [2023-05-11 00:16:25,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +12: [2023-05-11 00:16:25,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +12: [2023-05-11 00:16:25,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 8: [2023-05-11 00:16:25,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +12: [2023-05-11 00:16:25,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +29: [2023-05-11 00:16:25,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +29: [2023-05-11 00:16:25,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +29: [2023-05-11 00:16:25,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +29: [2023-05-11 00:16:25,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 8: [2023-05-11 00:16:25,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 8: [2023-05-11 00:16:25,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 8: [2023-05-11 00:16:25,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 8: [2023-05-11 00:16:25,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 8: [2023-05-11 00:16:25,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 8: [2023-05-11 00:16:25,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 8: [2023-05-11 00:16:25,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +18: [2023-05-11 00:16:25,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 4: [2023-05-11 00:16:25,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 4: [2023-05-11 00:16:25,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 4: [2023-05-11 00:16:25,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +18: [2023-05-11 00:16:25,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +18: [2023-05-11 00:16:25,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +18: [2023-05-11 00:16:25,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +18: [2023-05-11 00:16:25,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +29: [2023-05-11 00:16:25,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +18: [2023-05-11 00:16:25,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +18: [2023-05-11 00:16:25,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +29: [2023-05-11 00:16:25,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +29: [2023-05-11 00:16:25,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +29: [2023-05-11 00:16:25,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 4: [2023-05-11 00:16:25,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 4: [2023-05-11 00:16:25,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 4: [2023-05-11 00:16:25,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +18: [2023-05-11 00:16:25,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +18: [2023-05-11 00:16:25,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +18: [2023-05-11 00:16:25,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +18: [2023-05-11 00:16:25,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +18: [2023-05-11 00:16:25,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 4: [2023-05-11 00:16:25,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +30: [2023-05-11 00:16:25,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +30: [2023-05-11 00:16:25,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +30: [2023-05-11 00:16:25,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +30: [2023-05-11 00:16:25,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +30: [2023-05-11 00:16:25,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +31: [2023-05-11 00:16:25,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +31: [2023-05-11 00:16:25,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +30: [2023-05-11 00:16:25,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +30: [2023-05-11 00:16:25,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +30: [2023-05-11 00:16:25,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +31: [2023-05-11 00:16:25,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +31: [2023-05-11 00:16:25,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +31: [2023-05-11 00:16:25,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +31: [2023-05-11 00:16:25,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +31: [2023-05-11 00:16:25,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +31: [2023-05-11 00:16:25,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +27: [2023-05-11 00:16:25,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +30: [2023-05-11 00:16:25,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +30: [2023-05-11 00:16:25,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +30: [2023-05-11 00:16:25,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +30: [2023-05-11 00:16:25,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +30: [2023-05-11 00:16:25,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +27: [2023-05-11 00:16:25,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +30: [2023-05-11 00:16:25,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +18: [2023-05-11 00:16:25,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +18: [2023-05-11 00:16:25,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +30: [2023-05-11 00:16:25,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +31: [2023-05-11 00:16:25,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +31: [2023-05-11 00:16:25,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +31: [2023-05-11 00:16:25,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +30: [2023-05-11 00:16:25,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +31: [2023-05-11 00:16:25,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +31: [2023-05-11 00:16:25,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +31: [2023-05-11 00:16:25,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +31: [2023-05-11 00:16:25,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +31: [2023-05-11 00:16:25,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +27: [2023-05-11 00:16:25,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +27: [2023-05-11 00:16:25,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +18: [2023-05-11 00:16:25,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +18: [2023-05-11 00:16:25,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 8: [2023-05-11 00:16:25,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +13: [2023-05-11 00:16:25,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +13: [2023-05-11 00:16:25,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +13: [2023-05-11 00:16:25,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +13: [2023-05-11 00:16:25,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +13: [2023-05-11 00:16:25,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +13: [2023-05-11 00:16:25,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +13: [2023-05-11 00:16:25,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +13: [2023-05-11 00:16:25,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +17: [2023-05-11 00:16:25,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +17: [2023-05-11 00:16:25,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +17: [2023-05-11 00:16:25,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +17: [2023-05-11 00:16:25,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +17: [2023-05-11 00:16:25,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +13: [2023-05-11 00:16:25,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +17: [2023-05-11 00:16:25,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +17: [2023-05-11 00:16:25,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +25: [2023-05-11 00:16:25,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +25: [2023-05-11 00:16:25,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +17: [2023-05-11 00:16:25,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +13: [2023-05-11 00:16:25,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +13: [2023-05-11 00:16:25,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +13: [2023-05-11 00:16:25,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +13: [2023-05-11 00:16:25,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +27: [2023-05-11 00:16:25,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +13: [2023-05-11 00:16:25,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +14: [2023-05-11 00:16:25,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +14: [2023-05-11 00:16:25,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +14: [2023-05-11 00:16:25,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +14: [2023-05-11 00:16:25,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +14: [2023-05-11 00:16:25,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +14: [2023-05-11 00:16:25,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +14: [2023-05-11 00:16:25,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +13: [2023-05-11 00:16:25,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +13: [2023-05-11 00:16:25,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +25: [2023-05-11 00:16:25,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +25: [2023-05-11 00:16:25,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +25: [2023-05-11 00:16:25,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +25: [2023-05-11 00:16:25,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +25: [2023-05-11 00:16:25,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +25: [2023-05-11 00:16:25,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +17: [2023-05-11 00:16:25,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +14: [2023-05-11 00:16:25,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +17: [2023-05-11 00:16:25,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +17: [2023-05-11 00:16:25,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +17: [2023-05-11 00:16:25,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +17: [2023-05-11 00:16:25,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +17: [2023-05-11 00:16:25,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +25: [2023-05-11 00:16:25,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +25: [2023-05-11 00:16:25,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +25: [2023-05-11 00:16:25,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +17: [2023-05-11 00:16:25,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +17: [2023-05-11 00:16:25,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +25: [2023-05-11 00:16:25,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +25: [2023-05-11 00:16:25,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +25: [2023-05-11 00:16:25,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +25: [2023-05-11 00:16:25,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +14: [2023-05-11 00:16:25,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +14: [2023-05-11 00:16:25,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +14: [2023-05-11 00:16:25,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 5: [2023-05-11 00:16:25,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +25: [2023-05-11 00:16:25,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +27: [2023-05-11 00:16:25,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 5: [2023-05-11 00:16:25,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 5: [2023-05-11 00:16:25,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 5: [2023-05-11 00:16:25,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 5: [2023-05-11 00:16:25,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 6: [2023-05-11 00:16:25,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 5: [2023-05-11 00:16:25,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +14: [2023-05-11 00:16:25,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +14: [2023-05-11 00:16:25,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 5: [2023-05-11 00:16:25,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 5: [2023-05-11 00:16:25,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +14: [2023-05-11 00:16:25,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +14: [2023-05-11 00:16:25,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +14: [2023-05-11 00:16:25,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 6: [2023-05-11 00:16:25,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 8: [2023-05-11 00:16:25,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +26: [2023-05-11 00:16:25,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +26: [2023-05-11 00:16:25,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +26: [2023-05-11 00:16:25,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +26: [2023-05-11 00:16:25,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +26: [2023-05-11 00:16:25,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +26: [2023-05-11 00:16:25,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +26: [2023-05-11 00:16:25,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +26: [2023-05-11 00:16:25,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 6: [2023-05-11 00:16:25,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +30: [2023-05-11 00:16:25,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +26: [2023-05-11 00:16:25,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +30: [2023-05-11 00:16:25,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +30: [2023-05-11 00:16:25,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +26: [2023-05-11 00:16:25,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +27: [2023-05-11 00:16:25,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +27: [2023-05-11 00:16:25,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +26: [2023-05-11 00:16:25,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +26: [2023-05-11 00:16:25,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +26: [2023-05-11 00:16:25,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +26: [2023-05-11 00:16:25,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +26: [2023-05-11 00:16:25,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +26: [2023-05-11 00:16:25,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +30: [2023-05-11 00:16:25,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +31: [2023-05-11 00:16:25,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +31: [2023-05-11 00:16:25,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +31: [2023-05-11 00:16:25,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +31: [2023-05-11 00:16:25,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +19: [2023-05-11 00:16:25,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +19: [2023-05-11 00:16:25,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +19: [2023-05-11 00:16:25,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +19: [2023-05-11 00:16:25,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +19: [2023-05-11 00:16:25,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +19: [2023-05-11 00:16:25,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +19: [2023-05-11 00:16:25,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +19: [2023-05-11 00:16:25,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +22: [2023-05-11 00:16:25,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +22: [2023-05-11 00:16:25,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +22: [2023-05-11 00:16:25,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +22: [2023-05-11 00:16:25,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +22: [2023-05-11 00:16:25,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +22: [2023-05-11 00:16:25,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +22: [2023-05-11 00:16:25,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +19: [2023-05-11 00:16:25,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +22: [2023-05-11 00:16:25,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +19: [2023-05-11 00:16:25,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +19: [2023-05-11 00:16:25,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +19: [2023-05-11 00:16:25,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +19: [2023-05-11 00:16:25,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +19: [2023-05-11 00:16:25,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 3: [2023-05-11 00:16:25,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 1: [2023-05-11 00:16:25,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 7: [2023-05-11 00:16:25,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 7: [2023-05-11 00:16:25,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 3: [2023-05-11 00:16:25,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +19: [2023-05-11 00:16:25,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 1: [2023-05-11 00:16:25,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 1: [2023-05-11 00:16:25,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 3: [2023-05-11 00:16:25,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +22: [2023-05-11 00:16:25,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 7: [2023-05-11 00:16:25,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +19: [2023-05-11 00:16:25,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +22: [2023-05-11 00:16:25,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 1: [2023-05-11 00:16:25,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +22: [2023-05-11 00:16:25,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +30: [2023-05-11 00:16:25,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +22: [2023-05-11 00:16:25,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +30: [2023-05-11 00:16:25,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 1: [2023-05-11 00:16:25,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 0: [2023-05-11 00:16:25,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 0: [2023-05-11 00:16:25,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 0: [2023-05-11 00:16:25,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 0: [2023-05-11 00:16:25,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 0: [2023-05-11 00:16:25,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 0: [2023-05-11 00:16:25,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 0: [2023-05-11 00:16:25,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +22: [2023-05-11 00:16:25,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +22: [2023-05-11 00:16:25,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +30: [2023-05-11 00:16:25,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 7: [2023-05-11 00:16:25,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 7: [2023-05-11 00:16:25,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 1: [2023-05-11 00:16:25,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +22: [2023-05-11 00:16:25,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +30: [2023-05-11 00:16:25,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +22: [2023-05-11 00:16:25,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 3: [2023-05-11 00:16:25,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 7: [2023-05-11 00:16:25,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 7: [2023-05-11 00:16:25,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 3: [2023-05-11 00:16:25,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 3: [2023-05-11 00:16:25,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 3: [2023-05-11 00:16:25,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 3: [2023-05-11 00:16:25,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 3: [2023-05-11 00:16:25,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 3: [2023-05-11 00:16:25,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 3: [2023-05-11 00:16:25,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 0: [2023-05-11 00:16:25,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 0: [2023-05-11 00:16:25,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 0: [2023-05-11 00:16:25,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +13: [2023-05-11 00:16:25,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +13: [2023-05-11 00:16:25,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +13: [2023-05-11 00:16:25,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 0: [2023-05-11 00:16:25,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 0: [2023-05-11 00:16:25,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +17: [2023-05-11 00:16:25,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +17: [2023-05-11 00:16:25,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +13: [2023-05-11 00:16:25,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +25: [2023-05-11 00:16:25,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +25: [2023-05-11 00:16:25,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +31: [2023-05-11 00:16:25,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +31: [2023-05-11 00:16:25,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +31: [2023-05-11 00:16:25,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +25: [2023-05-11 00:16:25,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +17: [2023-05-11 00:16:25,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +17: [2023-05-11 00:16:25,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +14: [2023-05-11 00:16:25,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +21: [2023-05-11 00:16:25,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +21: [2023-05-11 00:16:25,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +21: [2023-05-11 00:16:25,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +31: [2023-05-11 00:16:25,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +21: [2023-05-11 00:16:25,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +21: [2023-05-11 00:16:25,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +21: [2023-05-11 00:16:25,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +21: [2023-05-11 00:16:25,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +21: [2023-05-11 00:16:25,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +25: [2023-05-11 00:16:25,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +21: [2023-05-11 00:16:25,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +21: [2023-05-11 00:16:25,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +21: [2023-05-11 00:16:25,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +21: [2023-05-11 00:16:25,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +17: [2023-05-11 00:16:25,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +17: [2023-05-11 00:16:25,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +14: [2023-05-11 00:16:25,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +14: [2023-05-11 00:16:25,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +14: [2023-05-11 00:16:25,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +13: [2023-05-11 00:16:25,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +13: [2023-05-11 00:16:25,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +21: [2023-05-11 00:16:25,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +21: [2023-05-11 00:16:25,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +13: [2023-05-11 00:16:25,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +21: [2023-05-11 00:16:25,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +21: [2023-05-11 00:16:25,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +25: [2023-05-11 00:16:25,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +25: [2023-05-11 00:16:25,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +13: [2023-05-11 00:16:25,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 5: [2023-05-11 00:16:25,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 5: [2023-05-11 00:16:25,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +19: [2023-05-11 00:16:25,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +26: [2023-05-11 00:16:25,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +26: [2023-05-11 00:16:25,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +26: [2023-05-11 00:16:25,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +26: [2023-05-11 00:16:25,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 5: [2023-05-11 00:16:25,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +19: [2023-05-11 00:16:25,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +25: [2023-05-11 00:16:25,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +17: [2023-05-11 00:16:25,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 1: [2023-05-11 00:16:25,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +17: [2023-05-11 00:16:25,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +22: [2023-05-11 00:16:25,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +22: [2023-05-11 00:16:25,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +19: [2023-05-11 00:16:25,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +19: [2023-05-11 00:16:25,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +14: [2023-05-11 00:16:25,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +22: [2023-05-11 00:16:25,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +22: [2023-05-11 00:16:25,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 3: [2023-05-11 00:16:25,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +14: [2023-05-11 00:16:25,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +14: [2023-05-11 00:16:25,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +14: [2023-05-11 00:16:25,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 0: [2023-05-11 00:16:25,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 0: [2023-05-11 00:16:25,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +25: [2023-05-11 00:16:25,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +26: [2023-05-11 00:16:25,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +26: [2023-05-11 00:16:25,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +26: [2023-05-11 00:16:25,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +26: [2023-05-11 00:16:25,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +20: [2023-05-11 00:16:25,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +20: [2023-05-11 00:16:25,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +20: [2023-05-11 00:16:25,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +20: [2023-05-11 00:16:25,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +20: [2023-05-11 00:16:25,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +22: [2023-05-11 00:16:25,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +20: [2023-05-11 00:16:25,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +20: [2023-05-11 00:16:25,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +20: [2023-05-11 00:16:25,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +22: [2023-05-11 00:16:25,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +22: [2023-05-11 00:16:25,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +20: [2023-05-11 00:16:25,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +20: [2023-05-11 00:16:25,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +20: [2023-05-11 00:16:25,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +20: [2023-05-11 00:16:25,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +20: [2023-05-11 00:16:25,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... +21: [2023-05-11 00:16:25,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +20: [2023-05-11 00:16:25,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +20: [2023-05-11 00:16:25,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... +20: [2023-05-11 00:16:25,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 7: [2023-05-11 00:16:25,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +22: [2023-05-11 00:16:25,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +19: [2023-05-11 00:16:25,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 2: [2023-05-11 00:16:25,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 2: [2023-05-11 00:16:25,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 2: [2023-05-11 00:16:25,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 2: [2023-05-11 00:16:25,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 2: [2023-05-11 00:16:25,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +19: [2023-05-11 00:16:25,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 3: [2023-05-11 00:16:25,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +21: [2023-05-11 00:16:25,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +21: [2023-05-11 00:16:25,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +21: [2023-05-11 00:16:25,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +19: [2023-05-11 00:16:25,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +19: [2023-05-11 00:16:25,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 1: [2023-05-11 00:16:25,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 1: [2023-05-11 00:16:25,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 1: [2023-05-11 00:16:25,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 2: [2023-05-11 00:16:25,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 2: [2023-05-11 00:16:25,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 2: [2023-05-11 00:16:25,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt... + 2: [2023-05-11 00:16:25,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt... + 7: [2023-05-11 00:16:25,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 7: [2023-05-11 00:16:25,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +21: [2023-05-11 00:16:25,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 7: [2023-05-11 00:16:25,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +21: [2023-05-11 00:16:25,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +21: [2023-05-11 00:16:25,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +21: [2023-05-11 00:16:25,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +20: [2023-05-11 00:16:25,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +20: [2023-05-11 00:16:25,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +20: [2023-05-11 00:16:25,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +20: [2023-05-11 00:16:25,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 2: [2023-05-11 00:16:25,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 2: [2023-05-11 00:16:25,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 2: [2023-05-11 00:16:25,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. + 2: [2023-05-11 00:16:25,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt. +20: [2023-05-11 00:16:25,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +20: [2023-05-11 00:16:25,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +20: [2023-05-11 00:16:25,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +20: [2023-05-11 00:16:25,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +28: [2023-05-11 00:16:25,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +28: [2023-05-11 00:16:25,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +28: [2023-05-11 00:16:25,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +11: [2023-05-11 00:16:25,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +11: [2023-05-11 00:16:25,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +11: [2023-05-11 00:16:25,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +11: [2023-05-11 00:16:25,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +28: [2023-05-11 00:16:25,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 2: [2023-05-11 00:16:25,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +28: [2023-05-11 00:16:25,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +28: [2023-05-11 00:16:25,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +28: [2023-05-11 00:16:25,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +11: [2023-05-11 00:16:25,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +28: [2023-05-11 00:16:25,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +11: [2023-05-11 00:16:25,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +11: [2023-05-11 00:16:25,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +11: [2023-05-11 00:16:25,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +29: [2023-05-11 00:16:25,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +29: [2023-05-11 00:16:25,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +29: [2023-05-11 00:16:25,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +29: [2023-05-11 00:16:25,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +10: [2023-05-11 00:16:25,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +10: [2023-05-11 00:16:25,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +10: [2023-05-11 00:16:25,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +10: [2023-05-11 00:16:25,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +23: [2023-05-11 00:16:25,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +23: [2023-05-11 00:16:25,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +23: [2023-05-11 00:16:25,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +23: [2023-05-11 00:16:25,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +29: [2023-05-11 00:16:25,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +15: [2023-05-11 00:16:25,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +15: [2023-05-11 00:16:25,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +29: [2023-05-11 00:16:25,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +29: [2023-05-11 00:16:25,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +15: [2023-05-11 00:16:25,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +15: [2023-05-11 00:16:25,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +29: [2023-05-11 00:16:25,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +24: [2023-05-11 00:16:25,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +24: [2023-05-11 00:16:25,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +24: [2023-05-11 00:16:25,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +24: [2023-05-11 00:16:25,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +10: [2023-05-11 00:16:25,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +10: [2023-05-11 00:16:25,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +10: [2023-05-11 00:16:25,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +15: [2023-05-11 00:16:25,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +10: [2023-05-11 00:16:25,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +15: [2023-05-11 00:16:25,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +15: [2023-05-11 00:16:25,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +15: [2023-05-11 00:16:25,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +24: [2023-05-11 00:16:25,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +24: [2023-05-11 00:16:25,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +24: [2023-05-11 00:16:25,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 9: [2023-05-11 00:16:25,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 9: [2023-05-11 00:16:25,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +24: [2023-05-11 00:16:25,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +12: [2023-05-11 00:16:25,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +12: [2023-05-11 00:16:25,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +18: [2023-05-11 00:16:25,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +18: [2023-05-11 00:16:25,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +18: [2023-05-11 00:16:25,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +27: [2023-05-11 00:16:25,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +27: [2023-05-11 00:16:25,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +27: [2023-05-11 00:16:25,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +12: [2023-05-11 00:16:25,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +18: [2023-05-11 00:16:25,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +27: [2023-05-11 00:16:25,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +12: [2023-05-11 00:16:25,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 8: [2023-05-11 00:16:25,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 8: [2023-05-11 00:16:25,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 8: [2023-05-11 00:16:25,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 8: [2023-05-11 00:16:25,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 9: [2023-05-11 00:16:25,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +29: [2023-05-11 00:16:25,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +29: [2023-05-11 00:16:25,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +29: [2023-05-11 00:16:25,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +29: [2023-05-11 00:16:25,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +29: [2023-05-11 00:16:25,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +29: [2023-05-11 00:16:25,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +29: [2023-05-11 00:16:25,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +29: [2023-05-11 00:16:25,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +18: [2023-05-11 00:16:25,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +29: [2023-05-11 00:16:25,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +29: [2023-05-11 00:16:25,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +29: [2023-05-11 00:16:25,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +29: [2023-05-11 00:16:25,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +29: [2023-05-11 00:16:25,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +29: [2023-05-11 00:16:25,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +29: [2023-05-11 00:16:25,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +29: [2023-05-11 00:16:25,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +18: [2023-05-11 00:16:25,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +12: [2023-05-11 00:16:25,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +18: [2023-05-11 00:16:25,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +12: [2023-05-11 00:16:25,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +18: [2023-05-11 00:16:25,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 8: [2023-05-11 00:16:25,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 8: [2023-05-11 00:16:25,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +12: [2023-05-11 00:16:25,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +12: [2023-05-11 00:16:25,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +27: [2023-05-11 00:16:25,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 8: [2023-05-11 00:16:25,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +27: [2023-05-11 00:16:25,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 8: [2023-05-11 00:16:25,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +27: [2023-05-11 00:16:25,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +28: [2023-05-11 00:16:25,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +28: [2023-05-11 00:16:25,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +28: [2023-05-11 00:16:25,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +28: [2023-05-11 00:16:25,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +28: [2023-05-11 00:16:25,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +28: [2023-05-11 00:16:25,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +28: [2023-05-11 00:16:25,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +28: [2023-05-11 00:16:25,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +27: [2023-05-11 00:16:25,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +16: [2023-05-11 00:16:25,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +16: [2023-05-11 00:16:25,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +16: [2023-05-11 00:16:25,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +12: [2023-05-11 00:16:25,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +12: [2023-05-11 00:16:25,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +16: [2023-05-11 00:16:25,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +12: [2023-05-11 00:16:25,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +28: [2023-05-11 00:16:25,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +12: [2023-05-11 00:16:25,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +28: [2023-05-11 00:16:25,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +28: [2023-05-11 00:16:25,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +28: [2023-05-11 00:16:25,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +28: [2023-05-11 00:16:25,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +28: [2023-05-11 00:16:25,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +28: [2023-05-11 00:16:25,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +28: [2023-05-11 00:16:25,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +16: [2023-05-11 00:16:25,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +16: [2023-05-11 00:16:25,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +16: [2023-05-11 00:16:25,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +16: [2023-05-11 00:16:25,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +12: [2023-05-11 00:16:25,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +12: [2023-05-11 00:16:25,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +12: [2023-05-11 00:16:25,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +12: [2023-05-11 00:16:25,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +10: [2023-05-11 00:16:25,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +10: [2023-05-11 00:16:25,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +10: [2023-05-11 00:16:25,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +10: [2023-05-11 00:16:25,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +10: [2023-05-11 00:16:25,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +10: [2023-05-11 00:16:25,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +10: [2023-05-11 00:16:25,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +10: [2023-05-11 00:16:25,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +10: [2023-05-11 00:16:25,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +10: [2023-05-11 00:16:25,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +10: [2023-05-11 00:16:25,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +10: [2023-05-11 00:16:25,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +10: [2023-05-11 00:16:25,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +10: [2023-05-11 00:16:25,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +10: [2023-05-11 00:16:25,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +10: [2023-05-11 00:16:25,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +29: [2023-05-11 00:16:25,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +29: [2023-05-11 00:16:25,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +29: [2023-05-11 00:16:25,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +29: [2023-05-11 00:16:25,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +15: [2023-05-11 00:16:25,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +15: [2023-05-11 00:16:25,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +15: [2023-05-11 00:16:25,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +15: [2023-05-11 00:16:25,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +15: [2023-05-11 00:16:25,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +15: [2023-05-11 00:16:25,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +15: [2023-05-11 00:16:25,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +15: [2023-05-11 00:16:25,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +12: [2023-05-11 00:16:25,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +12: [2023-05-11 00:16:25,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +12: [2023-05-11 00:16:25,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 9: [2023-05-11 00:16:25,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +28: [2023-05-11 00:16:25,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +15: [2023-05-11 00:16:25,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +15: [2023-05-11 00:16:25,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +15: [2023-05-11 00:16:25,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +15: [2023-05-11 00:16:25,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +15: [2023-05-11 00:16:25,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +15: [2023-05-11 00:16:25,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +12: [2023-05-11 00:16:25,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +16: [2023-05-11 00:16:25,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +15: [2023-05-11 00:16:25,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +12: [2023-05-11 00:16:25,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +12: [2023-05-11 00:16:25,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +15: [2023-05-11 00:16:25,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +13: [2023-05-11 00:16:25,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +13: [2023-05-11 00:16:25,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +13: [2023-05-11 00:16:25,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +13: [2023-05-11 00:16:25,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 9: [2023-05-11 00:16:25,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 9: [2023-05-11 00:16:25,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 9: [2023-05-11 00:16:25,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 9: [2023-05-11 00:16:25,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +12: [2023-05-11 00:16:25,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +16: [2023-05-11 00:16:25,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 9: [2023-05-11 00:16:25,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +28: [2023-05-11 00:16:25,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +29: [2023-05-11 00:16:25,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +29: [2023-05-11 00:16:25,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +12: [2023-05-11 00:16:25,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +12: [2023-05-11 00:16:25,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +12: [2023-05-11 00:16:25,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +28: [2023-05-11 00:16:25,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +28: [2023-05-11 00:16:25,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +28: [2023-05-11 00:16:25,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +16: [2023-05-11 00:16:25,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +29: [2023-05-11 00:16:25,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +16: [2023-05-11 00:16:25,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +16: [2023-05-11 00:16:25,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +29: [2023-05-11 00:16:25,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +13: [2023-05-11 00:16:25,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +13: [2023-05-11 00:16:25,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +12: [2023-05-11 00:16:25,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +12: [2023-05-11 00:16:25,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +10: [2023-05-11 00:16:25,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +16: [2023-05-11 00:16:25,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 9: [2023-05-11 00:16:25,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +28: [2023-05-11 00:16:25,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +10: [2023-05-11 00:16:25,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +10: [2023-05-11 00:16:25,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +10: [2023-05-11 00:16:25,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +12: [2023-05-11 00:16:25,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +13: [2023-05-11 00:16:25,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +13: [2023-05-11 00:16:25,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +16: [2023-05-11 00:16:25,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +16: [2023-05-11 00:16:25,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +12: [2023-05-11 00:16:25,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +31: [2023-05-11 00:16:25,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +31: [2023-05-11 00:16:25,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +31: [2023-05-11 00:16:25,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +31: [2023-05-11 00:16:25,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +22: [2023-05-11 00:16:25,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +22: [2023-05-11 00:16:25,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +22: [2023-05-11 00:16:25,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +22: [2023-05-11 00:16:25,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +28: [2023-05-11 00:16:25,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +28: [2023-05-11 00:16:25,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +21: [2023-05-11 00:16:25,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +21: [2023-05-11 00:16:25,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +21: [2023-05-11 00:16:25,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +21: [2023-05-11 00:16:25,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 4: [2023-05-11 00:16:25,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 4: [2023-05-11 00:16:25,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 4: [2023-05-11 00:16:25,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 4: [2023-05-11 00:16:25,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 7: [2023-05-11 00:16:25,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 7: [2023-05-11 00:16:25,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 7: [2023-05-11 00:16:25,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 7: [2023-05-11 00:16:25,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +12: [2023-05-11 00:16:25,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +12: [2023-05-11 00:16:25,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +30: [2023-05-11 00:16:25,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +30: [2023-05-11 00:16:25,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +30: [2023-05-11 00:16:25,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 0: [2023-05-11 00:16:25,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 0: [2023-05-11 00:16:25,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +30: [2023-05-11 00:16:25,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +24: [2023-05-11 00:16:25,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +24: [2023-05-11 00:16:25,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 0: [2023-05-11 00:16:25,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +24: [2023-05-11 00:16:25,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +24: [2023-05-11 00:16:25,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +15: [2023-05-11 00:16:25,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 0: [2023-05-11 00:16:25,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 5: [2023-05-11 00:16:25,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 5: [2023-05-11 00:16:25,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 5: [2023-05-11 00:16:25,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +24: [2023-05-11 00:16:25,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +24: [2023-05-11 00:16:25,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +24: [2023-05-11 00:16:25,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +24: [2023-05-11 00:16:25,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +19: [2023-05-11 00:16:25,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +19: [2023-05-11 00:16:25,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +19: [2023-05-11 00:16:25,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +19: [2023-05-11 00:16:25,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 6: [2023-05-11 00:16:25,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 6: [2023-05-11 00:16:25,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 6: [2023-05-11 00:16:25,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +14: [2023-05-11 00:16:25,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +14: [2023-05-11 00:16:25,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +10: [2023-05-11 00:16:25,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +24: [2023-05-11 00:16:25,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +10: [2023-05-11 00:16:25,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +24: [2023-05-11 00:16:25,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +24: [2023-05-11 00:16:25,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +10: [2023-05-11 00:16:25,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +14: [2023-05-11 00:16:25,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +15: [2023-05-11 00:16:25,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +14: [2023-05-11 00:16:25,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +15: [2023-05-11 00:16:25,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +15: [2023-05-11 00:16:25,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +31: [2023-05-11 00:16:25,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +10: [2023-05-11 00:16:25,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +24: [2023-05-11 00:16:25,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +24: [2023-05-11 00:16:25,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +25: [2023-05-11 00:16:25,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +25: [2023-05-11 00:16:25,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +27: [2023-05-11 00:16:25,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +27: [2023-05-11 00:16:25,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +27: [2023-05-11 00:16:25,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +27: [2023-05-11 00:16:25,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +24: [2023-05-11 00:16:25,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +24: [2023-05-11 00:16:25,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +27: [2023-05-11 00:16:25,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +27: [2023-05-11 00:16:25,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +27: [2023-05-11 00:16:25,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +27: [2023-05-11 00:16:25,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +24: [2023-05-11 00:16:25,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 8: [2023-05-11 00:16:25,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +22: [2023-05-11 00:16:25,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 8: [2023-05-11 00:16:25,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +22: [2023-05-11 00:16:25,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 8: [2023-05-11 00:16:25,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +16: [2023-05-11 00:16:25,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +16: [2023-05-11 00:16:25,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +16: [2023-05-11 00:16:25,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +31: [2023-05-11 00:16:25,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +16: [2023-05-11 00:16:25,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +25: [2023-05-11 00:16:25,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +18: [2023-05-11 00:16:25,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +18: [2023-05-11 00:16:25,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +25: [2023-05-11 00:16:25,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +18: [2023-05-11 00:16:25,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +18: [2023-05-11 00:16:25,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +18: [2023-05-11 00:16:25,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +31: [2023-05-11 00:16:25,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +31: [2023-05-11 00:16:25,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +18: [2023-05-11 00:16:25,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +18: [2023-05-11 00:16:25,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +18: [2023-05-11 00:16:25,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 8: [2023-05-11 00:16:25,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +23: [2023-05-11 00:16:25,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +23: [2023-05-11 00:16:25,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +22: [2023-05-11 00:16:25,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +23: [2023-05-11 00:16:25,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +21: [2023-05-11 00:16:25,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 4: [2023-05-11 00:16:25,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +23: [2023-05-11 00:16:25,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +23: [2023-05-11 00:16:25,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +21: [2023-05-11 00:16:25,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +27: [2023-05-11 00:16:25,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +27: [2023-05-11 00:16:25,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +27: [2023-05-11 00:16:25,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +27: [2023-05-11 00:16:25,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +15: [2023-05-11 00:16:25,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +21: [2023-05-11 00:16:25,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +18: [2023-05-11 00:16:25,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +18: [2023-05-11 00:16:25,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +27: [2023-05-11 00:16:25,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +21: [2023-05-11 00:16:25,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 8: [2023-05-11 00:16:25,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 8: [2023-05-11 00:16:25,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 4: [2023-05-11 00:16:25,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +18: [2023-05-11 00:16:25,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 8: [2023-05-11 00:16:25,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +27: [2023-05-11 00:16:25,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +18: [2023-05-11 00:16:25,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +18: [2023-05-11 00:16:25,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 5: [2023-05-11 00:16:25,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +18: [2023-05-11 00:16:25,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +27: [2023-05-11 00:16:25,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +18: [2023-05-11 00:16:25,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 7: [2023-05-11 00:16:25,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +22: [2023-05-11 00:16:25,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +18: [2023-05-11 00:16:25,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 7: [2023-05-11 00:16:25,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 5: [2023-05-11 00:16:25,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 5: [2023-05-11 00:16:25,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +27: [2023-05-11 00:16:25,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 7: [2023-05-11 00:16:25,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 0: [2023-05-11 00:16:25,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +30: [2023-05-11 00:16:25,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +15: [2023-05-11 00:16:25,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +23: [2023-05-11 00:16:25,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +30: [2023-05-11 00:16:25,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +30: [2023-05-11 00:16:25,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +15: [2023-05-11 00:16:25,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +15: [2023-05-11 00:16:25,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +30: [2023-05-11 00:16:25,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 5: [2023-05-11 00:16:25,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 6: [2023-05-11 00:16:25,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +14: [2023-05-11 00:16:25,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +14: [2023-05-11 00:16:25,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 7: [2023-05-11 00:16:25,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 4: [2023-05-11 00:16:25,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +25: [2023-05-11 00:16:25,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +16: [2023-05-11 00:16:25,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +16: [2023-05-11 00:16:25,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +25: [2023-05-11 00:16:25,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +16: [2023-05-11 00:16:25,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +16: [2023-05-11 00:16:25,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +25: [2023-05-11 00:16:25,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +25: [2023-05-11 00:16:25,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +19: [2023-05-11 00:16:25,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +19: [2023-05-11 00:16:25,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +19: [2023-05-11 00:16:25,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +19: [2023-05-11 00:16:25,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +14: [2023-05-11 00:16:25,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +14: [2023-05-11 00:16:25,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +24: [2023-05-11 00:16:25,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +20: [2023-05-11 00:16:25,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +20: [2023-05-11 00:16:25,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +20: [2023-05-11 00:16:25,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +20: [2023-05-11 00:16:25,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +24: [2023-05-11 00:16:25,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +24: [2023-05-11 00:16:25,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +24: [2023-05-11 00:16:25,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +17: [2023-05-11 00:16:25,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +17: [2023-05-11 00:16:25,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +17: [2023-05-11 00:16:25,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 8: [2023-05-11 00:16:25,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +17: [2023-05-11 00:16:25,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +25: [2023-05-11 00:16:25,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +25: [2023-05-11 00:16:25,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +25: [2023-05-11 00:16:25,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +25: [2023-05-11 00:16:25,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +18: [2023-05-11 00:16:25,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +18: [2023-05-11 00:16:25,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +27: [2023-05-11 00:16:25,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 3: [2023-05-11 00:16:25,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 3: [2023-05-11 00:16:25,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 3: [2023-05-11 00:16:25,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +25: [2023-05-11 00:16:25,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +25: [2023-05-11 00:16:25,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +24: [2023-05-11 00:16:25,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +18: [2023-05-11 00:16:25,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +18: [2023-05-11 00:16:25,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 8: [2023-05-11 00:16:25,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +25: [2023-05-11 00:16:25,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +25: [2023-05-11 00:16:25,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +23: [2023-05-11 00:16:25,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 1: [2023-05-11 00:16:25,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 1: [2023-05-11 00:16:25,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 1: [2023-05-11 00:16:25,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +27: [2023-05-11 00:16:25,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +26: [2023-05-11 00:16:25,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +26: [2023-05-11 00:16:25,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +26: [2023-05-11 00:16:25,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 2: [2023-05-11 00:16:25,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 2: [2023-05-11 00:16:25,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 2: [2023-05-11 00:16:25,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +26: [2023-05-11 00:16:25,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. + 2: [2023-05-11 00:16:25,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt. +20: [2023-05-11 00:16:25,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +27: [2023-05-11 00:16:25,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +16: [2023-05-11 00:16:25,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +16: [2023-05-11 00:16:25,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +20: [2023-05-11 00:16:25,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +18: [2023-05-11 00:16:25,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +27: [2023-05-11 00:16:25,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +24: [2023-05-11 00:16:25,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +18: [2023-05-11 00:16:25,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 4: [2023-05-11 00:16:25,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 4: [2023-05-11 00:16:25,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +20: [2023-05-11 00:16:25,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +20: [2023-05-11 00:16:25,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +24: [2023-05-11 00:16:25,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +24: [2023-05-11 00:16:25,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 8: [2023-05-11 00:16:25,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 8: [2023-05-11 00:16:25,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +17: [2023-05-11 00:16:25,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +16: [2023-05-11 00:16:25,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +16: [2023-05-11 00:16:25,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +16: [2023-05-11 00:16:25,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +16: [2023-05-11 00:16:25,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +25: [2023-05-11 00:16:25,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +25: [2023-05-11 00:16:25,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +27: [2023-05-11 00:16:25,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +18: [2023-05-11 00:16:25,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +18: [2023-05-11 00:16:25,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +16: [2023-05-11 00:16:25,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +16: [2023-05-11 00:16:25,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +25: [2023-05-11 00:16:25,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 4: [2023-05-11 00:16:25,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +17: [2023-05-11 00:16:25,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +25: [2023-05-11 00:16:25,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +17: [2023-05-11 00:16:25,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 8: [2023-05-11 00:16:25,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 3: [2023-05-11 00:16:25,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 3: [2023-05-11 00:16:25,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +17: [2023-05-11 00:16:25,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 4: [2023-05-11 00:16:25,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 4: [2023-05-11 00:16:25,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +11: [2023-05-11 00:16:25,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +11: [2023-05-11 00:16:25,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +23: [2023-05-11 00:16:25,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 3: [2023-05-11 00:16:25,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +11: [2023-05-11 00:16:25,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +11: [2023-05-11 00:16:25,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +11: [2023-05-11 00:16:25,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +11: [2023-05-11 00:16:25,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +11: [2023-05-11 00:16:25,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +11: [2023-05-11 00:16:25,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +23: [2023-05-11 00:16:25,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 8: [2023-05-11 00:16:25,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +25: [2023-05-11 00:16:25,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +25: [2023-05-11 00:16:25,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +11: [2023-05-11 00:16:25,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +11: [2023-05-11 00:16:25,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +11: [2023-05-11 00:16:25,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +11: [2023-05-11 00:16:25,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +26: [2023-05-11 00:16:25,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +26: [2023-05-11 00:16:25,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +11: [2023-05-11 00:16:25,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 4: [2023-05-11 00:16:25,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 1: [2023-05-11 00:16:25,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 1: [2023-05-11 00:16:25,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +11: [2023-05-11 00:16:25,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 4: [2023-05-11 00:16:25,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +11: [2023-05-11 00:16:25,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +11: [2023-05-11 00:16:25,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +25: [2023-05-11 00:16:25,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +25: [2023-05-11 00:16:25,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +26: [2023-05-11 00:16:25,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +27: [2023-05-11 00:16:25,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 1: [2023-05-11 00:16:25,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +27: [2023-05-11 00:16:25,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +26: [2023-05-11 00:16:25,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +23: [2023-05-11 00:16:25,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 2: [2023-05-11 00:16:25,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +27: [2023-05-11 00:16:25,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +30: [2023-05-11 00:16:25,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +30: [2023-05-11 00:16:25,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +30: [2023-05-11 00:16:25,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +30: [2023-05-11 00:16:25,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +30: [2023-05-11 00:16:25,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +30: [2023-05-11 00:16:25,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +30: [2023-05-11 00:16:25,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +30: [2023-05-11 00:16:25,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +14: [2023-05-11 00:16:25,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +14: [2023-05-11 00:16:25,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +14: [2023-05-11 00:16:25,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +14: [2023-05-11 00:16:25,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +14: [2023-05-11 00:16:25,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +14: [2023-05-11 00:16:25,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +14: [2023-05-11 00:16:25,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +14: [2023-05-11 00:16:25,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +25: [2023-05-11 00:16:25,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +25: [2023-05-11 00:16:25,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +30: [2023-05-11 00:16:25,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +30: [2023-05-11 00:16:25,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +30: [2023-05-11 00:16:25,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +30: [2023-05-11 00:16:25,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +30: [2023-05-11 00:16:25,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +30: [2023-05-11 00:16:25,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +25: [2023-05-11 00:16:25,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +30: [2023-05-11 00:16:25,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +14: [2023-05-11 00:16:25,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +30: [2023-05-11 00:16:25,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +14: [2023-05-11 00:16:25,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +14: [2023-05-11 00:16:25,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +14: [2023-05-11 00:16:25,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +14: [2023-05-11 00:16:25,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +14: [2023-05-11 00:16:25,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +14: [2023-05-11 00:16:25,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +25: [2023-05-11 00:16:25,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +14: [2023-05-11 00:16:25,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 4: [2023-05-11 00:16:25,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +25: [2023-05-11 00:16:25,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +25: [2023-05-11 00:16:25,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +11: [2023-05-11 00:16:25,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +25: [2023-05-11 00:16:25,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +11: [2023-05-11 00:16:25,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +25: [2023-05-11 00:16:25,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +11: [2023-05-11 00:16:25,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +11: [2023-05-11 00:16:25,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +21: [2023-05-11 00:16:25,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +21: [2023-05-11 00:16:25,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +21: [2023-05-11 00:16:25,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +21: [2023-05-11 00:16:25,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +21: [2023-05-11 00:16:25,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +21: [2023-05-11 00:16:25,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +21: [2023-05-11 00:16:25,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +21: [2023-05-11 00:16:25,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 6: [2023-05-11 00:16:25,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +21: [2023-05-11 00:16:25,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +21: [2023-05-11 00:16:25,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +21: [2023-05-11 00:16:25,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +21: [2023-05-11 00:16:25,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +21: [2023-05-11 00:16:25,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +21: [2023-05-11 00:16:25,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 6: [2023-05-11 00:16:25,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +21: [2023-05-11 00:16:25,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +21: [2023-05-11 00:16:25,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +11: [2023-05-11 00:16:25,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +11: [2023-05-11 00:16:25,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +30: [2023-05-11 00:16:25,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +30: [2023-05-11 00:16:25,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +30: [2023-05-11 00:16:25,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +30: [2023-05-11 00:16:25,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +11: [2023-05-11 00:16:25,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +11: [2023-05-11 00:16:25,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +14: [2023-05-11 00:16:25,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +14: [2023-05-11 00:16:25,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +14: [2023-05-11 00:16:25,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +14: [2023-05-11 00:16:25,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +30: [2023-05-11 00:16:25,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +30: [2023-05-11 00:16:25,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +30: [2023-05-11 00:16:25,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +30: [2023-05-11 00:16:25,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +14: [2023-05-11 00:16:25,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +14: [2023-05-11 00:16:25,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +14: [2023-05-11 00:16:25,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +14: [2023-05-11 00:16:25,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +21: [2023-05-11 00:16:25,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +20: [2023-05-11 00:16:25,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +20: [2023-05-11 00:16:25,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +20: [2023-05-11 00:16:25,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +21: [2023-05-11 00:16:25,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +21: [2023-05-11 00:16:25,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +21: [2023-05-11 00:16:25,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +20: [2023-05-11 00:16:25,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +20: [2023-05-11 00:16:25,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +20: [2023-05-11 00:16:25,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +20: [2023-05-11 00:16:25,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 6: [2023-05-11 00:16:25,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +20: [2023-05-11 00:16:25,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +20: [2023-05-11 00:16:25,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +20: [2023-05-11 00:16:25,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 3: [2023-05-11 00:16:25,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +31: [2023-05-11 00:16:25,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +31: [2023-05-11 00:16:25,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +20: [2023-05-11 00:16:25,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +13: [2023-05-11 00:16:25,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +13: [2023-05-11 00:16:25,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +13: [2023-05-11 00:16:25,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +13: [2023-05-11 00:16:25,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +13: [2023-05-11 00:16:25,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +13: [2023-05-11 00:16:25,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +13: [2023-05-11 00:16:25,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +13: [2023-05-11 00:16:25,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +31: [2023-05-11 00:16:25,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +31: [2023-05-11 00:16:25,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +31: [2023-05-11 00:16:25,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +31: [2023-05-11 00:16:25,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 7: [2023-05-11 00:16:25,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +31: [2023-05-11 00:16:25,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +31: [2023-05-11 00:16:25,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +20: [2023-05-11 00:16:25,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 3: [2023-05-11 00:16:25,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 7: [2023-05-11 00:16:25,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 3: [2023-05-11 00:16:25,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +17: [2023-05-11 00:16:25,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +17: [2023-05-11 00:16:25,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +17: [2023-05-11 00:16:25,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 3: [2023-05-11 00:16:25,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +20: [2023-05-11 00:16:25,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +17: [2023-05-11 00:16:25,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +17: [2023-05-11 00:16:25,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +17: [2023-05-11 00:16:25,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +17: [2023-05-11 00:16:25,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +20: [2023-05-11 00:16:25,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 7: [2023-05-11 00:16:25,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +20: [2023-05-11 00:16:25,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +17: [2023-05-11 00:16:25,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +13: [2023-05-11 00:16:25,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +20: [2023-05-11 00:16:25,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 3: [2023-05-11 00:16:25,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 3: [2023-05-11 00:16:25,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 3: [2023-05-11 00:16:25,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 7: [2023-05-11 00:16:25,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 3: [2023-05-11 00:16:25,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +31: [2023-05-11 00:16:25,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +13: [2023-05-11 00:16:25,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 7: [2023-05-11 00:16:25,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +13: [2023-05-11 00:16:25,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +31: [2023-05-11 00:16:25,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 0: [2023-05-11 00:16:25,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 0: [2023-05-11 00:16:25,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 0: [2023-05-11 00:16:25,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 0: [2023-05-11 00:16:25,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 0: [2023-05-11 00:16:25,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 0: [2023-05-11 00:16:25,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 0: [2023-05-11 00:16:25,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +13: [2023-05-11 00:16:25,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +17: [2023-05-11 00:16:25,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +13: [2023-05-11 00:16:25,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +13: [2023-05-11 00:16:25,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +31: [2023-05-11 00:16:25,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +13: [2023-05-11 00:16:25,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +31: [2023-05-11 00:16:25,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +13: [2023-05-11 00:16:25,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 0: [2023-05-11 00:16:25,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +31: [2023-05-11 00:16:25,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +17: [2023-05-11 00:16:25,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +17: [2023-05-11 00:16:25,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +17: [2023-05-11 00:16:25,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +17: [2023-05-11 00:16:25,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +17: [2023-05-11 00:16:25,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +17: [2023-05-11 00:16:25,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +17: [2023-05-11 00:16:25,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +21: [2023-05-11 00:16:25,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +31: [2023-05-11 00:16:25,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +31: [2023-05-11 00:16:25,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 0: [2023-05-11 00:16:25,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 0: [2023-05-11 00:16:25,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +31: [2023-05-11 00:16:25,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 0: [2023-05-11 00:16:25,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +21: [2023-05-11 00:16:25,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 6: [2023-05-11 00:16:25,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +21: [2023-05-11 00:16:25,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +21: [2023-05-11 00:16:25,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 7: [2023-05-11 00:16:25,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +20: [2023-05-11 00:16:25,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +20: [2023-05-11 00:16:25,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 2: [2023-05-11 00:16:25,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 2: [2023-05-11 00:16:25,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 2: [2023-05-11 00:16:25,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 2: [2023-05-11 00:16:25,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 2: [2023-05-11 00:16:25,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 2: [2023-05-11 00:16:25,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 2: [2023-05-11 00:16:25,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 2: [2023-05-11 00:16:25,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +26: [2023-05-11 00:16:25,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +26: [2023-05-11 00:16:25,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +26: [2023-05-11 00:16:25,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +26: [2023-05-11 00:16:25,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +26: [2023-05-11 00:16:25,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +26: [2023-05-11 00:16:25,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +26: [2023-05-11 00:16:25,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 5: [2023-05-11 00:16:25,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +26: [2023-05-11 00:16:25,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 5: [2023-05-11 00:16:25,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 5: [2023-05-11 00:16:25,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 5: [2023-05-11 00:16:25,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +20: [2023-05-11 00:16:25,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +20: [2023-05-11 00:16:25,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +19: [2023-05-11 00:16:25,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +19: [2023-05-11 00:16:25,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +19: [2023-05-11 00:16:25,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +19: [2023-05-11 00:16:25,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +19: [2023-05-11 00:16:25,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +26: [2023-05-11 00:16:25,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +13: [2023-05-11 00:16:25,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +19: [2023-05-11 00:16:25,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +19: [2023-05-11 00:16:25,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +19: [2023-05-11 00:16:25,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +31: [2023-05-11 00:16:25,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +26: [2023-05-11 00:16:25,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +26: [2023-05-11 00:16:25,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +17: [2023-05-11 00:16:25,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +17: [2023-05-11 00:16:25,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +26: [2023-05-11 00:16:25,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +26: [2023-05-11 00:16:25,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 2: [2023-05-11 00:16:25,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 2: [2023-05-11 00:16:25,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +26: [2023-05-11 00:16:25,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +13: [2023-05-11 00:16:25,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +13: [2023-05-11 00:16:25,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +13: [2023-05-11 00:16:25,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +26: [2023-05-11 00:16:25,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 7: [2023-05-11 00:16:25,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +26: [2023-05-11 00:16:25,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +31: [2023-05-11 00:16:25,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +31: [2023-05-11 00:16:25,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +31: [2023-05-11 00:16:25,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +20: [2023-05-11 00:16:25,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +19: [2023-05-11 00:16:25,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +20: [2023-05-11 00:16:25,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +17: [2023-05-11 00:16:25,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +17: [2023-05-11 00:16:25,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +19: [2023-05-11 00:16:25,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 7: [2023-05-11 00:16:25,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +19: [2023-05-11 00:16:25,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +19: [2023-05-11 00:16:25,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +19: [2023-05-11 00:16:25,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +19: [2023-05-11 00:16:25,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 7: [2023-05-11 00:16:25,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +19: [2023-05-11 00:16:25,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 0: [2023-05-11 00:16:25,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 0: [2023-05-11 00:16:25,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 0: [2023-05-11 00:16:25,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +19: [2023-05-11 00:16:25,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 3: [2023-05-11 00:16:25,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 7: [2023-05-11 00:16:25,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +20: [2023-05-11 00:16:25,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 3: [2023-05-11 00:16:25,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +20: [2023-05-11 00:16:25,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +17: [2023-05-11 00:16:25,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +17: [2023-05-11 00:16:25,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 3: [2023-05-11 00:16:25,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 3: [2023-05-11 00:16:25,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +13: [2023-05-11 00:16:25,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +31: [2023-05-11 00:16:25,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +13: [2023-05-11 00:16:25,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +13: [2023-05-11 00:16:25,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +13: [2023-05-11 00:16:25,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +17: [2023-05-11 00:16:25,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +17: [2023-05-11 00:16:25,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +31: [2023-05-11 00:16:25,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +31: [2023-05-11 00:16:25,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 7: [2023-05-11 00:16:25,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +31: [2023-05-11 00:16:25,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 0: [2023-05-11 00:16:25,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 1: [2023-05-11 00:16:25,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +22: [2023-05-11 00:16:25,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +22: [2023-05-11 00:16:25,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +22: [2023-05-11 00:16:25,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +22: [2023-05-11 00:16:25,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +22: [2023-05-11 00:16:25,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +22: [2023-05-11 00:16:25,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +22: [2023-05-11 00:16:25,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +22: [2023-05-11 00:16:25,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 7: [2023-05-11 00:16:25,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 1: [2023-05-11 00:16:25,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 2: [2023-05-11 00:16:25,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... +22: [2023-05-11 00:16:25,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +22: [2023-05-11 00:16:25,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +22: [2023-05-11 00:16:25,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 1: [2023-05-11 00:16:25,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +22: [2023-05-11 00:16:25,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 1: [2023-05-11 00:16:25,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +22: [2023-05-11 00:16:25,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 2: [2023-05-11 00:16:25,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 2: [2023-05-11 00:16:25,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +22: [2023-05-11 00:16:25,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 1: [2023-05-11 00:16:25,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 1: [2023-05-11 00:16:25,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +22: [2023-05-11 00:16:25,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... +22: [2023-05-11 00:16:25,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 1: [2023-05-11 00:16:25,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt... + 1: [2023-05-11 00:16:25,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 1: [2023-05-11 00:16:25,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt... + 2: [2023-05-11 00:16:25,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +19: [2023-05-11 00:16:25,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +26: [2023-05-11 00:16:25,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +26: [2023-05-11 00:16:25,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +26: [2023-05-11 00:16:25,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +26: [2023-05-11 00:16:25,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 5: [2023-05-11 00:16:25,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 5: [2023-05-11 00:16:25,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 5: [2023-05-11 00:16:25,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 5: [2023-05-11 00:16:25,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +19: [2023-05-11 00:16:25,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +19: [2023-05-11 00:16:25,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +19: [2023-05-11 00:16:25,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 2: [2023-05-11 00:16:25,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 2: [2023-05-11 00:16:25,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +26: [2023-05-11 00:16:25,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +26: [2023-05-11 00:16:25,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +26: [2023-05-11 00:16:25,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +26: [2023-05-11 00:16:25,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +19: [2023-05-11 00:16:25,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +19: [2023-05-11 00:16:25,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +19: [2023-05-11 00:16:25,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +22: [2023-05-11 00:16:25,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +19: [2023-05-11 00:16:25,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +22: [2023-05-11 00:16:25,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +22: [2023-05-11 00:16:25,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +22: [2023-05-11 00:16:25,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. + 1: [2023-05-11 00:16:25,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt. +22: [2023-05-11 00:16:25,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +28: [2023-05-11 00:16:25,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +28: [2023-05-11 00:16:25,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +28: [2023-05-11 00:16:25,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +28: [2023-05-11 00:16:25,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 1: [2023-05-11 00:16:25,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +22: [2023-05-11 00:16:25,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +22: [2023-05-11 00:16:25,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 1: [2023-05-11 00:16:25,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 1: [2023-05-11 00:16:25,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 1: [2023-05-11 00:16:25,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +22: [2023-05-11 00:16:25,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +28: [2023-05-11 00:16:25,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +10: [2023-05-11 00:16:25,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +10: [2023-05-11 00:16:25,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +28: [2023-05-11 00:16:25,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +28: [2023-05-11 00:16:25,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +28: [2023-05-11 00:16:25,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +10: [2023-05-11 00:16:25,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +10: [2023-05-11 00:16:25,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +10: [2023-05-11 00:16:25,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +10: [2023-05-11 00:16:25,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +10: [2023-05-11 00:16:25,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +10: [2023-05-11 00:16:25,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +29: [2023-05-11 00:16:25,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +29: [2023-05-11 00:16:25,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +29: [2023-05-11 00:16:25,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +29: [2023-05-11 00:16:25,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +29: [2023-05-11 00:16:25,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +29: [2023-05-11 00:16:25,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +29: [2023-05-11 00:16:25,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +29: [2023-05-11 00:16:25,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +28: [2023-05-11 00:16:26,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +28: [2023-05-11 00:16:26,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +28: [2023-05-11 00:16:26,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +28: [2023-05-11 00:16:26,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +28: [2023-05-11 00:16:26,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +28: [2023-05-11 00:16:26,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +10: [2023-05-11 00:16:26,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +28: [2023-05-11 00:16:26,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +28: [2023-05-11 00:16:26,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +28: [2023-05-11 00:16:26,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +28: [2023-05-11 00:16:26,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +28: [2023-05-11 00:16:26,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +10: [2023-05-11 00:16:26,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +28: [2023-05-11 00:16:26,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +16: [2023-05-11 00:16:26,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +16: [2023-05-11 00:16:26,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +16: [2023-05-11 00:16:26,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +16: [2023-05-11 00:16:26,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +16: [2023-05-11 00:16:26,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +16: [2023-05-11 00:16:26,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +16: [2023-05-11 00:16:26,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +15: [2023-05-11 00:16:26,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +15: [2023-05-11 00:16:26,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +16: [2023-05-11 00:16:26,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +15: [2023-05-11 00:16:26,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +15: [2023-05-11 00:16:26,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +10: [2023-05-11 00:16:26,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +28: [2023-05-11 00:16:26,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +28: [2023-05-11 00:16:26,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +15: [2023-05-11 00:16:26,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +15: [2023-05-11 00:16:26,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +15: [2023-05-11 00:16:26,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +15: [2023-05-11 00:16:26,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +29: [2023-05-11 00:16:26,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +10: [2023-05-11 00:16:26,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +10: [2023-05-11 00:16:26,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +28: [2023-05-11 00:16:26,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +28: [2023-05-11 00:16:26,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +29: [2023-05-11 00:16:26,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +29: [2023-05-11 00:16:26,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +28: [2023-05-11 00:16:26,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +28: [2023-05-11 00:16:26,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +16: [2023-05-11 00:16:26,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +15: [2023-05-11 00:16:26,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +15: [2023-05-11 00:16:26,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +15: [2023-05-11 00:16:26,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +16: [2023-05-11 00:16:26,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +16: [2023-05-11 00:16:26,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +10: [2023-05-11 00:16:26,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +10: [2023-05-11 00:16:26,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +10: [2023-05-11 00:16:26,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +28: [2023-05-11 00:16:26,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +15: [2023-05-11 00:16:26,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +12: [2023-05-11 00:16:26,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +12: [2023-05-11 00:16:26,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +12: [2023-05-11 00:16:26,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +12: [2023-05-11 00:16:26,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +28: [2023-05-11 00:16:26,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +28: [2023-05-11 00:16:26,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +29: [2023-05-11 00:16:26,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +12: [2023-05-11 00:16:26,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +12: [2023-05-11 00:16:26,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +12: [2023-05-11 00:16:26,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +12: [2023-05-11 00:16:26,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +28: [2023-05-11 00:16:26,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +29: [2023-05-11 00:16:26,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +29: [2023-05-11 00:16:26,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +15: [2023-05-11 00:16:26,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +29: [2023-05-11 00:16:26,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +15: [2023-05-11 00:16:26,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +28: [2023-05-11 00:16:26,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +15: [2023-05-11 00:16:26,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +16: [2023-05-11 00:16:26,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +28: [2023-05-11 00:16:26,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +15: [2023-05-11 00:16:26,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +16: [2023-05-11 00:16:26,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +16: [2023-05-11 00:16:26,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +16: [2023-05-11 00:16:26,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +29: [2023-05-11 00:16:26,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +12: [2023-05-11 00:16:26,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +24: [2023-05-11 00:16:26,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +24: [2023-05-11 00:16:26,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +24: [2023-05-11 00:16:26,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +24: [2023-05-11 00:16:26,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +24: [2023-05-11 00:16:26,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +24: [2023-05-11 00:16:26,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +24: [2023-05-11 00:16:26,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +24: [2023-05-11 00:16:26,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +18: [2023-05-11 00:16:26,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +18: [2023-05-11 00:16:26,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +18: [2023-05-11 00:16:26,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +18: [2023-05-11 00:16:26,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +18: [2023-05-11 00:16:26,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +18: [2023-05-11 00:16:26,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +18: [2023-05-11 00:16:26,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +18: [2023-05-11 00:16:26,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +12: [2023-05-11 00:16:26,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +12: [2023-05-11 00:16:26,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +16: [2023-05-11 00:16:26,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +12: [2023-05-11 00:16:26,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +15: [2023-05-11 00:16:26,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +15: [2023-05-11 00:16:26,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +15: [2023-05-11 00:16:26,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +15: [2023-05-11 00:16:26,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +24: [2023-05-11 00:16:26,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +12: [2023-05-11 00:16:26,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +12: [2023-05-11 00:16:26,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +12: [2023-05-11 00:16:26,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +15: [2023-05-11 00:16:26,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +12: [2023-05-11 00:16:26,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +15: [2023-05-11 00:16:26,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +15: [2023-05-11 00:16:26,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +18: [2023-05-11 00:16:26,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +18: [2023-05-11 00:16:26,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +18: [2023-05-11 00:16:26,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +24: [2023-05-11 00:16:26,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +24: [2023-05-11 00:16:26,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +15: [2023-05-11 00:16:26,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +18: [2023-05-11 00:16:26,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +24: [2023-05-11 00:16:26,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +18: [2023-05-11 00:16:26,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +15: [2023-05-11 00:16:26,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +15: [2023-05-11 00:16:26,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +24: [2023-05-11 00:16:26,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +18: [2023-05-11 00:16:26,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +18: [2023-05-11 00:16:26,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +18: [2023-05-11 00:16:26,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +15: [2023-05-11 00:16:26,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +15: [2023-05-11 00:16:26,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +15: [2023-05-11 00:16:26,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +15: [2023-05-11 00:16:26,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +12: [2023-05-11 00:16:26,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +12: [2023-05-11 00:16:26,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +12: [2023-05-11 00:16:26,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +12: [2023-05-11 00:16:26,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +15: [2023-05-11 00:16:26,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +15: [2023-05-11 00:16:26,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +24: [2023-05-11 00:16:26,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +24: [2023-05-11 00:16:26,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +24: [2023-05-11 00:16:26,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +23: [2023-05-11 00:16:26,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +23: [2023-05-11 00:16:26,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +23: [2023-05-11 00:16:26,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +23: [2023-05-11 00:16:26,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +23: [2023-05-11 00:16:26,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +23: [2023-05-11 00:16:26,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +23: [2023-05-11 00:16:26,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +23: [2023-05-11 00:16:26,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +12: [2023-05-11 00:16:26,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +12: [2023-05-11 00:16:26,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +12: [2023-05-11 00:16:26,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +12: [2023-05-11 00:16:26,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +27: [2023-05-11 00:16:26,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +27: [2023-05-11 00:16:26,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +27: [2023-05-11 00:16:26,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +27: [2023-05-11 00:16:26,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +10: [2023-05-11 00:16:26,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +10: [2023-05-11 00:16:26,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +10: [2023-05-11 00:16:26,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 9: [2023-05-11 00:16:26,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +10: [2023-05-11 00:16:26,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +25: [2023-05-11 00:16:26,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +25: [2023-05-11 00:16:26,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +25: [2023-05-11 00:16:26,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +25: [2023-05-11 00:16:26,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +25: [2023-05-11 00:16:26,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +25: [2023-05-11 00:16:26,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +23: [2023-05-11 00:16:26,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +23: [2023-05-11 00:16:26,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +31: [2023-05-11 00:16:26,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +31: [2023-05-11 00:16:26,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +31: [2023-05-11 00:16:26,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +31: [2023-05-11 00:16:26,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +31: [2023-05-11 00:16:26,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +31: [2023-05-11 00:16:26,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +25: [2023-05-11 00:16:26,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +31: [2023-05-11 00:16:26,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +25: [2023-05-11 00:16:26,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +31: [2023-05-11 00:16:26,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +23: [2023-05-11 00:16:26,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +23: [2023-05-11 00:16:26,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +23: [2023-05-11 00:16:26,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +23: [2023-05-11 00:16:26,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +23: [2023-05-11 00:16:26,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +23: [2023-05-11 00:16:26,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +27: [2023-05-11 00:16:26,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +27: [2023-05-11 00:16:26,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +12: [2023-05-11 00:16:26,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +12: [2023-05-11 00:16:26,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +12: [2023-05-11 00:16:26,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +12: [2023-05-11 00:16:26,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +12: [2023-05-11 00:16:26,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +12: [2023-05-11 00:16:26,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 4: [2023-05-11 00:16:26,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +27: [2023-05-11 00:16:26,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +27: [2023-05-11 00:16:26,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +10: [2023-05-11 00:16:26,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +10: [2023-05-11 00:16:26,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +10: [2023-05-11 00:16:26,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +18: [2023-05-11 00:16:26,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +18: [2023-05-11 00:16:26,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +18: [2023-05-11 00:16:26,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +25: [2023-05-11 00:16:26,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +18: [2023-05-11 00:16:26,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +30: [2023-05-11 00:16:26,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +30: [2023-05-11 00:16:26,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +30: [2023-05-11 00:16:26,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +25: [2023-05-11 00:16:26,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +30: [2023-05-11 00:16:26,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +31: [2023-05-11 00:16:26,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +25: [2023-05-11 00:16:26,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +25: [2023-05-11 00:16:26,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +10: [2023-05-11 00:16:26,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +23: [2023-05-11 00:16:26,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +23: [2023-05-11 00:16:26,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +31: [2023-05-11 00:16:26,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +23: [2023-05-11 00:16:26,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +23: [2023-05-11 00:16:26,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +31: [2023-05-11 00:16:26,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +31: [2023-05-11 00:16:26,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +23: [2023-05-11 00:16:26,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +23: [2023-05-11 00:16:26,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +23: [2023-05-11 00:16:26,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +12: [2023-05-11 00:16:26,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +12: [2023-05-11 00:16:26,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +25: [2023-05-11 00:16:26,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +29: [2023-05-11 00:16:26,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +29: [2023-05-11 00:16:26,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +25: [2023-05-11 00:16:26,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +29: [2023-05-11 00:16:26,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 9: [2023-05-11 00:16:26,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +30: [2023-05-11 00:16:26,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +30: [2023-05-11 00:16:26,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +30: [2023-05-11 00:16:26,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +30: [2023-05-11 00:16:26,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +18: [2023-05-11 00:16:26,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +18: [2023-05-11 00:16:26,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +18: [2023-05-11 00:16:26,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +18: [2023-05-11 00:16:26,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +25: [2023-05-11 00:16:26,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +23: [2023-05-11 00:16:26,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +29: [2023-05-11 00:16:26,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +31: [2023-05-11 00:16:26,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +31: [2023-05-11 00:16:26,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +24: [2023-05-11 00:16:26,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +24: [2023-05-11 00:16:26,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +24: [2023-05-11 00:16:26,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +24: [2023-05-11 00:16:26,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +10: [2023-05-11 00:16:26,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +10: [2023-05-11 00:16:26,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +31: [2023-05-11 00:16:26,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +23: [2023-05-11 00:16:26,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +25: [2023-05-11 00:16:26,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +23: [2023-05-11 00:16:26,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +23: [2023-05-11 00:16:26,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +23: [2023-05-11 00:16:26,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 5: [2023-05-11 00:16:26,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +31: [2023-05-11 00:16:26,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +23: [2023-05-11 00:16:26,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +23: [2023-05-11 00:16:26,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 6: [2023-05-11 00:16:26,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +10: [2023-05-11 00:16:26,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +10: [2023-05-11 00:16:26,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +21: [2023-05-11 00:16:26,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +21: [2023-05-11 00:16:26,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +10: [2023-05-11 00:16:26,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +10: [2023-05-11 00:16:26,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 4: [2023-05-11 00:16:26,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +21: [2023-05-11 00:16:26,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +21: [2023-05-11 00:16:26,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +21: [2023-05-11 00:16:26,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +21: [2023-05-11 00:16:26,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +29: [2023-05-11 00:16:26,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +21: [2023-05-11 00:16:26,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +23: [2023-05-11 00:16:26,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +23: [2023-05-11 00:16:26,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +21: [2023-05-11 00:16:26,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +29: [2023-05-11 00:16:26,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +18: [2023-05-11 00:16:26,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +18: [2023-05-11 00:16:26,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +29: [2023-05-11 00:16:26,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +18: [2023-05-11 00:16:26,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +18: [2023-05-11 00:16:26,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 1: [2023-05-11 00:16:26,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +18: [2023-05-11 00:16:26,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +18: [2023-05-11 00:16:26,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +24: [2023-05-11 00:16:26,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +10: [2023-05-11 00:16:26,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +10: [2023-05-11 00:16:26,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 1: [2023-05-11 00:16:26,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 9: [2023-05-11 00:16:26,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 9: [2023-05-11 00:16:26,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 9: [2023-05-11 00:16:26,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 5: [2023-05-11 00:16:26,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +29: [2023-05-11 00:16:26,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +24: [2023-05-11 00:16:26,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +24: [2023-05-11 00:16:26,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +24: [2023-05-11 00:16:26,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +21: [2023-05-11 00:16:26,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +18: [2023-05-11 00:16:26,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 8: [2023-05-11 00:16:26,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 8: [2023-05-11 00:16:26,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 8: [2023-05-11 00:16:26,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +18: [2023-05-11 00:16:26,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 6: [2023-05-11 00:16:26,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +21: [2023-05-11 00:16:26,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +29: [2023-05-11 00:16:26,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +29: [2023-05-11 00:16:26,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +22: [2023-05-11 00:16:26,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +22: [2023-05-11 00:16:26,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +22: [2023-05-11 00:16:26,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +22: [2023-05-11 00:16:26,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +29: [2023-05-11 00:16:26,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +22: [2023-05-11 00:16:26,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +29: [2023-05-11 00:16:26,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +22: [2023-05-11 00:16:26,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +22: [2023-05-11 00:16:26,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +22: [2023-05-11 00:16:26,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +21: [2023-05-11 00:16:26,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +29: [2023-05-11 00:16:26,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +29: [2023-05-11 00:16:26,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +24: [2023-05-11 00:16:26,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +24: [2023-05-11 00:16:26,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +24: [2023-05-11 00:16:26,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +24: [2023-05-11 00:16:26,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +22: [2023-05-11 00:16:26,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +21: [2023-05-11 00:16:26,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +22: [2023-05-11 00:16:26,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +24: [2023-05-11 00:16:26,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +24: [2023-05-11 00:16:26,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +22: [2023-05-11 00:16:26,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +22: [2023-05-11 00:16:26,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +24: [2023-05-11 00:16:26,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +21: [2023-05-11 00:16:26,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +21: [2023-05-11 00:16:26,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +29: [2023-05-11 00:16:26,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +24: [2023-05-11 00:16:26,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 9: [2023-05-11 00:16:26,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 9: [2023-05-11 00:16:26,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +22: [2023-05-11 00:16:26,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +29: [2023-05-11 00:16:26,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +22: [2023-05-11 00:16:26,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +22: [2023-05-11 00:16:26,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 8: [2023-05-11 00:16:26,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 8: [2023-05-11 00:16:26,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +22: [2023-05-11 00:16:26,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +21: [2023-05-11 00:16:26,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +21: [2023-05-11 00:16:26,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 9: [2023-05-11 00:16:26,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +19: [2023-05-11 00:16:26,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +19: [2023-05-11 00:16:26,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +19: [2023-05-11 00:16:26,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +19: [2023-05-11 00:16:26,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +20: [2023-05-11 00:16:26,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +19: [2023-05-11 00:16:26,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +20: [2023-05-11 00:16:26,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +20: [2023-05-11 00:16:26,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +19: [2023-05-11 00:16:26,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +19: [2023-05-11 00:16:26,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +19: [2023-05-11 00:16:26,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +20: [2023-05-11 00:16:26,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +11: [2023-05-11 00:16:26,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +11: [2023-05-11 00:16:26,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +11: [2023-05-11 00:16:26,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +11: [2023-05-11 00:16:26,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +27: [2023-05-11 00:16:26,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +27: [2023-05-11 00:16:26,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +27: [2023-05-11 00:16:26,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +27: [2023-05-11 00:16:26,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +11: [2023-05-11 00:16:26,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +11: [2023-05-11 00:16:26,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +14: [2023-05-11 00:16:26,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +14: [2023-05-11 00:16:26,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +11: [2023-05-11 00:16:26,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +14: [2023-05-11 00:16:26,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +14: [2023-05-11 00:16:26,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +27: [2023-05-11 00:16:26,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +27: [2023-05-11 00:16:26,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +27: [2023-05-11 00:16:26,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +27: [2023-05-11 00:16:26,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +27: [2023-05-11 00:16:26,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +27: [2023-05-11 00:16:26,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +27: [2023-05-11 00:16:26,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +27: [2023-05-11 00:16:26,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 7: [2023-05-11 00:16:26,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +20: [2023-05-11 00:16:26,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +20: [2023-05-11 00:16:26,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +20: [2023-05-11 00:16:26,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +20: [2023-05-11 00:16:26,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +27: [2023-05-11 00:16:26,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +27: [2023-05-11 00:16:26,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +27: [2023-05-11 00:16:26,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +27: [2023-05-11 00:16:26,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 7: [2023-05-11 00:16:26,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +11: [2023-05-11 00:16:26,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +14: [2023-05-11 00:16:26,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +14: [2023-05-11 00:16:26,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +14: [2023-05-11 00:16:26,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +14: [2023-05-11 00:16:26,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +21: [2023-05-11 00:16:26,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +21: [2023-05-11 00:16:26,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +21: [2023-05-11 00:16:26,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +21: [2023-05-11 00:16:26,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +30: [2023-05-11 00:16:26,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +30: [2023-05-11 00:16:26,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +30: [2023-05-11 00:16:26,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +30: [2023-05-11 00:16:26,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +30: [2023-05-11 00:16:26,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +30: [2023-05-11 00:16:26,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +30: [2023-05-11 00:16:26,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +30: [2023-05-11 00:16:26,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +30: [2023-05-11 00:16:26,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +30: [2023-05-11 00:16:26,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +30: [2023-05-11 00:16:26,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +30: [2023-05-11 00:16:26,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +30: [2023-05-11 00:16:26,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +30: [2023-05-11 00:16:26,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +30: [2023-05-11 00:16:26,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +30: [2023-05-11 00:16:26,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +19: [2023-05-11 00:16:26,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +11: [2023-05-11 00:16:26,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +19: [2023-05-11 00:16:26,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +19: [2023-05-11 00:16:26,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +19: [2023-05-11 00:16:26,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +14: [2023-05-11 00:16:26,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +27: [2023-05-11 00:16:26,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +27: [2023-05-11 00:16:26,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +21: [2023-05-11 00:16:26,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +21: [2023-05-11 00:16:26,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +11: [2023-05-11 00:16:26,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +11: [2023-05-11 00:16:26,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +27: [2023-05-11 00:16:26,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +21: [2023-05-11 00:16:26,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +20: [2023-05-11 00:16:26,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +20: [2023-05-11 00:16:26,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +21: [2023-05-11 00:16:26,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +20: [2023-05-11 00:16:26,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +20: [2023-05-11 00:16:26,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +30: [2023-05-11 00:16:26,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +14: [2023-05-11 00:16:26,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +14: [2023-05-11 00:16:26,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +30: [2023-05-11 00:16:26,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +11: [2023-05-11 00:16:26,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +14: [2023-05-11 00:16:26,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +14: [2023-05-11 00:16:26,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +11: [2023-05-11 00:16:26,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +30: [2023-05-11 00:16:26,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +30: [2023-05-11 00:16:26,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +20: [2023-05-11 00:16:26,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +19: [2023-05-11 00:16:26,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +11: [2023-05-11 00:16:26,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +27: [2023-05-11 00:16:26,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +11: [2023-05-11 00:16:26,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +25: [2023-05-11 00:16:26,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +25: [2023-05-11 00:16:26,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 3: [2023-05-11 00:16:26,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +20: [2023-05-11 00:16:26,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +25: [2023-05-11 00:16:26,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +25: [2023-05-11 00:16:26,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +20: [2023-05-11 00:16:26,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +19: [2023-05-11 00:16:26,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +19: [2023-05-11 00:16:26,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +27: [2023-05-11 00:16:26,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +21: [2023-05-11 00:16:26,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +20: [2023-05-11 00:16:26,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +21: [2023-05-11 00:16:26,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +21: [2023-05-11 00:16:26,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +21: [2023-05-11 00:16:26,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +30: [2023-05-11 00:16:26,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +21: [2023-05-11 00:16:26,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +21: [2023-05-11 00:16:26,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +30: [2023-05-11 00:16:26,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +30: [2023-05-11 00:16:26,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +14: [2023-05-11 00:16:26,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +14: [2023-05-11 00:16:26,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +19: [2023-05-11 00:16:26,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +14: [2023-05-11 00:16:26,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +30: [2023-05-11 00:16:26,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +27: [2023-05-11 00:16:26,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +27: [2023-05-11 00:16:26,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +21: [2023-05-11 00:16:26,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +21: [2023-05-11 00:16:26,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +11: [2023-05-11 00:16:26,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +25: [2023-05-11 00:16:26,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +25: [2023-05-11 00:16:26,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +27: [2023-05-11 00:16:26,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +25: [2023-05-11 00:16:26,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +25: [2023-05-11 00:16:26,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +31: [2023-05-11 00:16:26,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +31: [2023-05-11 00:16:26,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +31: [2023-05-11 00:16:26,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +31: [2023-05-11 00:16:26,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 2: [2023-05-11 00:16:26,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +25: [2023-05-11 00:16:26,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +25: [2023-05-11 00:16:26,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 2: [2023-05-11 00:16:26,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +25: [2023-05-11 00:16:26,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +25: [2023-05-11 00:16:26,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 0: [2023-05-11 00:16:26,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +25: [2023-05-11 00:16:26,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +25: [2023-05-11 00:16:26,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +31: [2023-05-11 00:16:26,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +17: [2023-05-11 00:16:26,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +17: [2023-05-11 00:16:26,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +17: [2023-05-11 00:16:26,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +17: [2023-05-11 00:16:26,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +17: [2023-05-11 00:16:26,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +17: [2023-05-11 00:16:26,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +13: [2023-05-11 00:16:26,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +13: [2023-05-11 00:16:26,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +13: [2023-05-11 00:16:26,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +13: [2023-05-11 00:16:26,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +13: [2023-05-11 00:16:26,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +13: [2023-05-11 00:16:26,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +13: [2023-05-11 00:16:26,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +17: [2023-05-11 00:16:26,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +17: [2023-05-11 00:16:26,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +13: [2023-05-11 00:16:26,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +25: [2023-05-11 00:16:26,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +25: [2023-05-11 00:16:26,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +31: [2023-05-11 00:16:26,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +31: [2023-05-11 00:16:26,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +31: [2023-05-11 00:16:26,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +26: [2023-05-11 00:16:26,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +26: [2023-05-11 00:16:26,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +26: [2023-05-11 00:16:26,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +26: [2023-05-11 00:16:26,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +26: [2023-05-11 00:16:26,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +26: [2023-05-11 00:16:26,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +26: [2023-05-11 00:16:26,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +26: [2023-05-11 00:16:26,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +28: [2023-05-11 00:16:26,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +28: [2023-05-11 00:16:26,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +28: [2023-05-11 00:16:26,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +28: [2023-05-11 00:16:26,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +28: [2023-05-11 00:16:26,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +28: [2023-05-11 00:16:26,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +28: [2023-05-11 00:16:26,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +11: [2023-05-11 00:16:26,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +11: [2023-05-11 00:16:26,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +28: [2023-05-11 00:16:26,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +11: [2023-05-11 00:16:26,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +31: [2023-05-11 00:16:26,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +11: [2023-05-11 00:16:26,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +31: [2023-05-11 00:16:26,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 4: [2023-05-11 00:16:26,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 4: [2023-05-11 00:16:26,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 4: [2023-05-11 00:16:26,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 4: [2023-05-11 00:16:26,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +13: [2023-05-11 00:16:26,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +20: [2023-05-11 00:16:26,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +20: [2023-05-11 00:16:26,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +20: [2023-05-11 00:16:26,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +20: [2023-05-11 00:16:26,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +16: [2023-05-11 00:16:26,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +16: [2023-05-11 00:16:26,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +16: [2023-05-11 00:16:26,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +16: [2023-05-11 00:16:26,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +17: [2023-05-11 00:16:26,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +17: [2023-05-11 00:16:26,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +13: [2023-05-11 00:16:26,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +13: [2023-05-11 00:16:26,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +13: [2023-05-11 00:16:26,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +12: [2023-05-11 00:16:26,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +12: [2023-05-11 00:16:26,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +12: [2023-05-11 00:16:26,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +12: [2023-05-11 00:16:26,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +17: [2023-05-11 00:16:26,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +17: [2023-05-11 00:16:26,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 3: [2023-05-11 00:16:26,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 3: [2023-05-11 00:16:26,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 3: [2023-05-11 00:16:26,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +11: [2023-05-11 00:16:26,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +11: [2023-05-11 00:16:26,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 2: [2023-05-11 00:16:26,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 2: [2023-05-11 00:16:26,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 2: [2023-05-11 00:16:26,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +13: [2023-05-11 00:16:26,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +28: [2023-05-11 00:16:26,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 6: [2023-05-11 00:16:26,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 6: [2023-05-11 00:16:26,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +13: [2023-05-11 00:16:26,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +13: [2023-05-11 00:16:26,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 7: [2023-05-11 00:16:26,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 6: [2023-05-11 00:16:26,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +13: [2023-05-11 00:16:26,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +11: [2023-05-11 00:16:26,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +31: [2023-05-11 00:16:26,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +31: [2023-05-11 00:16:26,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +31: [2023-05-11 00:16:26,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +31: [2023-05-11 00:16:26,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +31: [2023-05-11 00:16:26,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +31: [2023-05-11 00:16:26,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 7: [2023-05-11 00:16:26,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 7: [2023-05-11 00:16:26,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +20: [2023-05-11 00:16:26,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +28: [2023-05-11 00:16:26,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +28: [2023-05-11 00:16:26,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +20: [2023-05-11 00:16:26,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +26: [2023-05-11 00:16:26,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +26: [2023-05-11 00:16:26,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +26: [2023-05-11 00:16:26,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +20: [2023-05-11 00:16:26,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +20: [2023-05-11 00:16:26,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +16: [2023-05-11 00:16:26,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +16: [2023-05-11 00:16:26,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +15: [2023-05-11 00:16:26,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +15: [2023-05-11 00:16:26,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +15: [2023-05-11 00:16:26,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +14: [2023-05-11 00:16:26,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +14: [2023-05-11 00:16:26,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +15: [2023-05-11 00:16:26,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +28: [2023-05-11 00:16:26,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +28: [2023-05-11 00:16:26,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +11: [2023-05-11 00:16:26,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +13: [2023-05-11 00:16:26,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +28: [2023-05-11 00:16:26,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +14: [2023-05-11 00:16:26,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +28: [2023-05-11 00:16:26,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +14: [2023-05-11 00:16:26,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +16: [2023-05-11 00:16:26,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +16: [2023-05-11 00:16:26,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +12: [2023-05-11 00:16:26,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +12: [2023-05-11 00:16:26,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +18: [2023-05-11 00:16:26,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 3: [2023-05-11 00:16:26,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +18: [2023-05-11 00:16:26,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +18: [2023-05-11 00:16:26,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +12: [2023-05-11 00:16:26,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +13: [2023-05-11 00:16:26,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +12: [2023-05-11 00:16:26,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +18: [2023-05-11 00:16:26,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 3: [2023-05-11 00:16:26,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +13: [2023-05-11 00:16:26,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +17: [2023-05-11 00:16:26,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +17: [2023-05-11 00:16:26,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +17: [2023-05-11 00:16:26,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +26: [2023-05-11 00:16:26,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +13: [2023-05-11 00:16:26,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +10: [2023-05-11 00:16:26,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +10: [2023-05-11 00:16:26,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +10: [2023-05-11 00:16:26,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +10: [2023-05-11 00:16:26,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 7: [2023-05-11 00:16:26,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +26: [2023-05-11 00:16:26,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +26: [2023-05-11 00:16:26,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +26: [2023-05-11 00:16:26,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +26: [2023-05-11 00:16:26,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +13: [2023-05-11 00:16:26,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +17: [2023-05-11 00:16:26,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +13: [2023-05-11 00:16:26,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +15: [2023-05-11 00:16:26,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +14: [2023-05-11 00:16:26,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +14: [2023-05-11 00:16:26,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +26: [2023-05-11 00:16:26,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +15: [2023-05-11 00:16:26,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +13: [2023-05-11 00:16:26,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +13: [2023-05-11 00:16:26,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +20: [2023-05-11 00:16:26,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +28: [2023-05-11 00:16:26,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +28: [2023-05-11 00:16:26,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +28: [2023-05-11 00:16:26,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +28: [2023-05-11 00:16:26,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +28: [2023-05-11 00:16:26,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +28: [2023-05-11 00:16:26,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +20: [2023-05-11 00:16:26,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +15: [2023-05-11 00:16:26,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +15: [2023-05-11 00:16:26,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +18: [2023-05-11 00:16:26,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +14: [2023-05-11 00:16:26,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +28: [2023-05-11 00:16:26,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +14: [2023-05-11 00:16:26,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +23: [2023-05-11 00:16:26,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +23: [2023-05-11 00:16:26,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +17: [2023-05-11 00:16:26,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +17: [2023-05-11 00:16:26,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +17: [2023-05-11 00:16:26,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +17: [2023-05-11 00:16:26,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +23: [2023-05-11 00:16:26,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +23: [2023-05-11 00:16:26,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +11: [2023-05-11 00:16:26,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +26: [2023-05-11 00:16:26,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +26: [2023-05-11 00:16:26,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +11: [2023-05-11 00:16:26,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +11: [2023-05-11 00:16:26,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 5: [2023-05-11 00:16:26,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +11: [2023-05-11 00:16:26,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +11: [2023-05-11 00:16:26,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +11: [2023-05-11 00:16:26,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +16: [2023-05-11 00:16:26,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +16: [2023-05-11 00:16:26,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +26: [2023-05-11 00:16:26,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +18: [2023-05-11 00:16:26,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +16: [2023-05-11 00:16:26,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +16: [2023-05-11 00:16:26,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +20: [2023-05-11 00:16:26,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 9: [2023-05-11 00:16:26,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 9: [2023-05-11 00:16:26,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +20: [2023-05-11 00:16:26,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 5: [2023-05-11 00:16:26,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 9: [2023-05-11 00:16:26,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +11: [2023-05-11 00:16:26,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +11: [2023-05-11 00:16:26,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +10: [2023-05-11 00:16:26,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +18: [2023-05-11 00:16:26,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +18: [2023-05-11 00:16:26,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +13: [2023-05-11 00:16:26,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +26: [2023-05-11 00:16:26,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +13: [2023-05-11 00:16:26,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 5: [2023-05-11 00:16:26,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +26: [2023-05-11 00:16:26,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +13: [2023-05-11 00:16:26,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +26: [2023-05-11 00:16:26,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +13: [2023-05-11 00:16:26,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +16: [2023-05-11 00:16:26,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +16: [2023-05-11 00:16:26,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +16: [2023-05-11 00:16:26,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +16: [2023-05-11 00:16:26,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +28: [2023-05-11 00:16:26,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 3: [2023-05-11 00:16:26,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 4: [2023-05-11 00:16:26,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +13: [2023-05-11 00:16:26,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +13: [2023-05-11 00:16:26,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 7: [2023-05-11 00:16:26,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +29: [2023-05-11 00:16:26,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +29: [2023-05-11 00:16:26,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +29: [2023-05-11 00:16:26,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +29: [2023-05-11 00:16:26,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +29: [2023-05-11 00:16:26,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +29: [2023-05-11 00:16:26,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +26: [2023-05-11 00:16:26,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +20: [2023-05-11 00:16:26,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +20: [2023-05-11 00:16:26,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 4: [2023-05-11 00:16:26,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +10: [2023-05-11 00:16:26,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 3: [2023-05-11 00:16:26,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +10: [2023-05-11 00:16:26,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 7: [2023-05-11 00:16:26,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +28: [2023-05-11 00:16:26,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +17: [2023-05-11 00:16:26,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +28: [2023-05-11 00:16:26,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 6: [2023-05-11 00:16:26,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 6: [2023-05-11 00:16:26,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +29: [2023-05-11 00:16:26,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +17: [2023-05-11 00:16:26,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +17: [2023-05-11 00:16:26,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +29: [2023-05-11 00:16:26,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +29: [2023-05-11 00:16:26,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +14: [2023-05-11 00:16:26,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +14: [2023-05-11 00:16:26,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +14: [2023-05-11 00:16:26,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +13: [2023-05-11 00:16:26,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +13: [2023-05-11 00:16:26,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +17: [2023-05-11 00:16:26,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +14: [2023-05-11 00:16:26,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +14: [2023-05-11 00:16:26,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +14: [2023-05-11 00:16:26,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +10: [2023-05-11 00:16:26,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +20: [2023-05-11 00:16:26,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +23: [2023-05-11 00:16:26,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +20: [2023-05-11 00:16:26,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +29: [2023-05-11 00:16:26,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +29: [2023-05-11 00:16:26,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 1: [2023-05-11 00:16:26,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 7: [2023-05-11 00:16:26,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +28: [2023-05-11 00:16:26,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +29: [2023-05-11 00:16:26,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +23: [2023-05-11 00:16:26,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 4: [2023-05-11 00:16:26,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 1: [2023-05-11 00:16:26,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 9: [2023-05-11 00:16:26,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +26: [2023-05-11 00:16:26,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +26: [2023-05-11 00:16:26,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +26: [2023-05-11 00:16:26,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +26: [2023-05-11 00:16:26,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +23: [2023-05-11 00:16:26,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +23: [2023-05-11 00:16:26,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +26: [2023-05-11 00:16:26,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +26: [2023-05-11 00:16:26,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +14: [2023-05-11 00:16:26,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +14: [2023-05-11 00:16:26,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 2: [2023-05-11 00:16:26,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 2: [2023-05-11 00:16:26,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 6: [2023-05-11 00:16:26,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +26: [2023-05-11 00:16:26,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +26: [2023-05-11 00:16:26,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 2: [2023-05-11 00:16:26,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 2: [2023-05-11 00:16:26,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +17: [2023-05-11 00:16:26,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +28: [2023-05-11 00:16:26,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +17: [2023-05-11 00:16:26,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +17: [2023-05-11 00:16:26,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +17: [2023-05-11 00:16:26,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 1: [2023-05-11 00:16:26,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +29: [2023-05-11 00:16:26,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +24: [2023-05-11 00:16:26,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +24: [2023-05-11 00:16:26,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +24: [2023-05-11 00:16:26,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 0: [2023-05-11 00:16:26,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 0: [2023-05-11 00:16:26,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +19: [2023-05-11 00:16:26,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +19: [2023-05-11 00:16:26,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +19: [2023-05-11 00:16:26,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +19: [2023-05-11 00:16:26,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +24: [2023-05-11 00:16:26,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 0: [2023-05-11 00:16:26,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. + 0: [2023-05-11 00:16:26,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +28: [2023-05-11 00:16:26,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +28: [2023-05-11 00:16:26,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +17: [2023-05-11 00:16:26,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +17: [2023-05-11 00:16:26,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 5: [2023-05-11 00:16:26,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +29: [2023-05-11 00:16:26,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +28: [2023-05-11 00:16:26,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +28: [2023-05-11 00:16:26,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +29: [2023-05-11 00:16:26,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +29: [2023-05-11 00:16:26,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +28: [2023-05-11 00:16:26,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +29: [2023-05-11 00:16:26,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 5: [2023-05-11 00:16:26,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +17: [2023-05-11 00:16:26,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +29: [2023-05-11 00:16:26,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +17: [2023-05-11 00:16:26,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 0: [2023-05-11 00:16:26,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +24: [2023-05-11 00:16:26,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +24: [2023-05-11 00:16:26,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +24: [2023-05-11 00:16:26,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +19: [2023-05-11 00:16:26,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +19: [2023-05-11 00:16:26,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +19: [2023-05-11 00:16:26,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +24: [2023-05-11 00:16:26,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +15: [2023-05-11 00:16:26,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +15: [2023-05-11 00:16:26,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +15: [2023-05-11 00:16:26,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +15: [2023-05-11 00:16:26,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +15: [2023-05-11 00:16:26,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +29: [2023-05-11 00:16:26,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +15: [2023-05-11 00:16:26,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +29: [2023-05-11 00:16:26,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +15: [2023-05-11 00:16:26,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +15: [2023-05-11 00:16:26,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +15: [2023-05-11 00:16:26,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +15: [2023-05-11 00:16:26,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +15: [2023-05-11 00:16:26,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +15: [2023-05-11 00:16:26,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 1: [2023-05-11 00:16:26,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 1: [2023-05-11 00:16:26,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +19: [2023-05-11 00:16:26,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +29: [2023-05-11 00:16:26,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +15: [2023-05-11 00:16:26,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +15: [2023-05-11 00:16:26,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +15: [2023-05-11 00:16:26,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +29: [2023-05-11 00:16:26,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +15: [2023-05-11 00:16:26,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +29: [2023-05-11 00:16:26,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +24: [2023-05-11 00:16:26,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +24: [2023-05-11 00:16:26,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +24: [2023-05-11 00:16:26,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +29: [2023-05-11 00:16:26,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +29: [2023-05-11 00:16:26,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +24: [2023-05-11 00:16:26,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +24: [2023-05-11 00:16:26,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 0: [2023-05-11 00:16:26,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +24: [2023-05-11 00:16:26,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +24: [2023-05-11 00:16:26,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +24: [2023-05-11 00:16:26,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +16: [2023-05-11 00:16:26,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +16: [2023-05-11 00:16:26,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +15: [2023-05-11 00:16:26,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +24: [2023-05-11 00:16:26,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +15: [2023-05-11 00:16:26,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +24: [2023-05-11 00:16:26,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +29: [2023-05-11 00:16:26,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +24: [2023-05-11 00:16:26,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +24: [2023-05-11 00:16:26,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 8: [2023-05-11 00:16:26,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 8: [2023-05-11 00:16:26,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 8: [2023-05-11 00:16:26,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +16: [2023-05-11 00:16:26,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +15: [2023-05-11 00:16:26,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +24: [2023-05-11 00:16:26,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +24: [2023-05-11 00:16:26,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +16: [2023-05-11 00:16:26,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +16: [2023-05-11 00:16:26,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +19: [2023-05-11 00:16:26,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +29: [2023-05-11 00:16:26,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +16: [2023-05-11 00:16:26,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +19: [2023-05-11 00:16:26,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 0: [2023-05-11 00:16:26,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 0: [2023-05-11 00:16:26,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 0: [2023-05-11 00:16:26,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +16: [2023-05-11 00:16:26,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +16: [2023-05-11 00:16:26,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +19: [2023-05-11 00:16:26,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +19: [2023-05-11 00:16:26,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +19: [2023-05-11 00:16:26,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +19: [2023-05-11 00:16:26,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +15: [2023-05-11 00:16:26,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +15: [2023-05-11 00:16:26,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +19: [2023-05-11 00:16:26,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +19: [2023-05-11 00:16:26,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... + 8: [2023-05-11 00:16:26,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +16: [2023-05-11 00:16:26,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +15: [2023-05-11 00:16:26,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +29: [2023-05-11 00:16:26,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +29: [2023-05-11 00:16:26,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +24: [2023-05-11 00:16:26,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +24: [2023-05-11 00:16:26,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +29: [2023-05-11 00:16:26,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +24: [2023-05-11 00:16:26,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +29: [2023-05-11 00:16:26,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +10: [2023-05-11 00:16:26,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +10: [2023-05-11 00:16:26,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +10: [2023-05-11 00:16:26,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +16: [2023-05-11 00:16:26,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +10: [2023-05-11 00:16:26,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +10: [2023-05-11 00:16:26,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +10: [2023-05-11 00:16:26,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +29: [2023-05-11 00:16:26,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +10: [2023-05-11 00:16:26,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +10: [2023-05-11 00:16:26,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +10: [2023-05-11 00:16:26,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +10: [2023-05-11 00:16:26,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +10: [2023-05-11 00:16:26,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +15: [2023-05-11 00:16:26,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +10: [2023-05-11 00:16:26,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +24: [2023-05-11 00:16:26,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +10: [2023-05-11 00:16:26,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +10: [2023-05-11 00:16:26,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +10: [2023-05-11 00:16:26,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +10: [2023-05-11 00:16:26,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +24: [2023-05-11 00:16:26,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +27: [2023-05-11 00:16:26,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +22: [2023-05-11 00:16:26,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +22: [2023-05-11 00:16:26,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +22: [2023-05-11 00:16:26,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +27: [2023-05-11 00:16:26,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +27: [2023-05-11 00:16:26,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +15: [2023-05-11 00:16:26,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +27: [2023-05-11 00:16:26,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +22: [2023-05-11 00:16:26,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt. +16: [2023-05-11 00:16:26,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +30: [2023-05-11 00:16:26,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +30: [2023-05-11 00:16:26,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +30: [2023-05-11 00:16:26,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +30: [2023-05-11 00:16:26,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +10: [2023-05-11 00:16:26,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +21: [2023-05-11 00:16:26,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +21: [2023-05-11 00:16:26,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +21: [2023-05-11 00:16:26,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +10: [2023-05-11 00:16:26,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +21: [2023-05-11 00:16:26,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +24: [2023-05-11 00:16:26,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +24: [2023-05-11 00:16:26,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +16: [2023-05-11 00:16:26,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +16: [2023-05-11 00:16:26,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +24: [2023-05-11 00:16:26,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +22: [2023-05-11 00:16:26,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +27: [2023-05-11 00:16:26,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +22: [2023-05-11 00:16:26,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +21: [2023-05-11 00:16:26,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +16: [2023-05-11 00:16:26,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +30: [2023-05-11 00:16:26,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +30: [2023-05-11 00:16:26,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +16: [2023-05-11 00:16:26,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +16: [2023-05-11 00:16:26,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +21: [2023-05-11 00:16:26,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +21: [2023-05-11 00:16:26,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +22: [2023-05-11 00:16:26,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +22: [2023-05-11 00:16:26,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt... +24: [2023-05-11 00:16:26,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +24: [2023-05-11 00:16:26,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +21: [2023-05-11 00:16:26,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +30: [2023-05-11 00:16:26,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +30: [2023-05-11 00:16:26,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +27: [2023-05-11 00:16:26,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +27: [2023-05-11 00:16:26,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +10: [2023-05-11 00:16:26,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +10: [2023-05-11 00:16:26,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +27: [2023-05-11 00:16:26,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +22: [2023-05-11 00:16:26,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +22: [2023-05-11 00:16:26,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +10: [2023-05-11 00:16:26,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +22: [2023-05-11 00:16:26,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +22: [2023-05-11 00:16:26,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +10: [2023-05-11 00:16:26,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +31: [2023-05-11 00:16:26,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +31: [2023-05-11 00:16:26,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +31: [2023-05-11 00:16:26,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +31: [2023-05-11 00:16:26,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +20: [2023-05-11 00:16:26,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +20: [2023-05-11 00:16:26,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +20: [2023-05-11 00:16:26,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +20: [2023-05-11 00:16:26,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +22: [2023-05-11 00:16:26,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +22: [2023-05-11 00:16:26,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +22: [2023-05-11 00:16:26,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt. +22: [2023-05-11 00:16:26,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt... +10: [2023-05-11 00:16:26,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +21: [2023-05-11 00:16:26,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +21: [2023-05-11 00:16:26,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +21: [2023-05-11 00:16:26,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +21: [2023-05-11 00:16:26,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +25: [2023-05-11 00:16:26,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +25: [2023-05-11 00:16:26,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +21: [2023-05-11 00:16:26,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +21: [2023-05-11 00:16:26,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +21: [2023-05-11 00:16:26,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +25: [2023-05-11 00:16:26,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +21: [2023-05-11 00:16:26,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +21: [2023-05-11 00:16:26,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +25: [2023-05-11 00:16:26,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +31: [2023-05-11 00:16:26,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +31: [2023-05-11 00:16:26,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +21: [2023-05-11 00:16:26,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +21: [2023-05-11 00:16:26,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +20: [2023-05-11 00:16:26,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +21: [2023-05-11 00:16:26,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +21: [2023-05-11 00:16:26,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +20: [2023-05-11 00:16:26,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +31: [2023-05-11 00:16:26,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +31: [2023-05-11 00:16:26,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +21: [2023-05-11 00:16:26,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +10: [2023-05-11 00:16:26,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +20: [2023-05-11 00:16:26,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +21: [2023-05-11 00:16:26,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +21: [2023-05-11 00:16:26,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +12: [2023-05-11 00:16:26,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +12: [2023-05-11 00:16:26,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +12: [2023-05-11 00:16:26,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +20: [2023-05-11 00:16:26,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +12: [2023-05-11 00:16:26,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +16: [2023-05-11 00:16:26,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +16: [2023-05-11 00:16:26,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +16: [2023-05-11 00:16:26,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +16: [2023-05-11 00:16:26,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +12: [2023-05-11 00:16:26,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +12: [2023-05-11 00:16:26,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +12: [2023-05-11 00:16:26,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +12: [2023-05-11 00:16:26,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +12: [2023-05-11 00:16:26,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +12: [2023-05-11 00:16:26,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +12: [2023-05-11 00:16:26,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +18: [2023-05-11 00:16:26,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +18: [2023-05-11 00:16:26,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +12: [2023-05-11 00:16:26,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +12: [2023-05-11 00:16:26,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +25: [2023-05-11 00:16:26,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +18: [2023-05-11 00:16:26,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +12: [2023-05-11 00:16:26,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +25: [2023-05-11 00:16:26,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +12: [2023-05-11 00:16:26,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +12: [2023-05-11 00:16:26,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +21: [2023-05-11 00:16:26,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +18: [2023-05-11 00:16:26,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +18: [2023-05-11 00:16:26,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +18: [2023-05-11 00:16:26,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +18: [2023-05-11 00:16:26,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +18: [2023-05-11 00:16:26,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +18: [2023-05-11 00:16:26,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +18: [2023-05-11 00:16:26,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +25: [2023-05-11 00:16:26,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +25: [2023-05-11 00:16:26,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +18: [2023-05-11 00:16:26,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +16: [2023-05-11 00:16:26,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +16: [2023-05-11 00:16:26,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +18: [2023-05-11 00:16:26,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +18: [2023-05-11 00:16:26,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +18: [2023-05-11 00:16:26,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +18: [2023-05-11 00:16:26,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +18: [2023-05-11 00:16:26,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +21: [2023-05-11 00:16:26,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +21: [2023-05-11 00:16:26,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +16: [2023-05-11 00:16:26,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +16: [2023-05-11 00:16:26,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +21: [2023-05-11 00:16:26,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +18: [2023-05-11 00:16:26,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +12: [2023-05-11 00:16:26,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +21: [2023-05-11 00:16:26,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +21: [2023-05-11 00:16:26,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +18: [2023-05-11 00:16:26,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +16: [2023-05-11 00:16:26,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +16: [2023-05-11 00:16:26,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 6: [2023-05-11 00:16:26,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +12: [2023-05-11 00:16:26,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +21: [2023-05-11 00:16:26,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 9: [2023-05-11 00:16:26,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 9: [2023-05-11 00:16:26,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 9: [2023-05-11 00:16:26,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +16: [2023-05-11 00:16:26,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +14: [2023-05-11 00:16:26,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 9: [2023-05-11 00:16:26,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +14: [2023-05-11 00:16:26,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +14: [2023-05-11 00:16:26,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +11: [2023-05-11 00:16:26,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +11: [2023-05-11 00:16:26,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +14: [2023-05-11 00:16:26,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +16: [2023-05-11 00:16:26,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +23: [2023-05-11 00:16:26,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +23: [2023-05-11 00:16:26,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +23: [2023-05-11 00:16:26,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +23: [2023-05-11 00:16:26,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +23: [2023-05-11 00:16:26,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +23: [2023-05-11 00:16:26,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +11: [2023-05-11 00:16:26,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +11: [2023-05-11 00:16:26,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +23: [2023-05-11 00:16:26,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +23: [2023-05-11 00:16:26,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +23: [2023-05-11 00:16:26,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +23: [2023-05-11 00:16:26,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +23: [2023-05-11 00:16:26,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +23: [2023-05-11 00:16:26,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 9: [2023-05-11 00:16:26,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +12: [2023-05-11 00:16:26,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +12: [2023-05-11 00:16:26,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +23: [2023-05-11 00:16:26,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +23: [2023-05-11 00:16:26,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +18: [2023-05-11 00:16:26,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +18: [2023-05-11 00:16:26,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +12: [2023-05-11 00:16:26,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +23: [2023-05-11 00:16:26,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 5: [2023-05-11 00:16:26,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +23: [2023-05-11 00:16:26,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +21: [2023-05-11 00:16:26,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +16: [2023-05-11 00:16:26,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +16: [2023-05-11 00:16:26,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +11: [2023-05-11 00:16:26,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +31: [2023-05-11 00:16:26,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 1: [2023-05-11 00:16:26,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +16: [2023-05-11 00:16:26,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +16: [2023-05-11 00:16:26,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +31: [2023-05-11 00:16:26,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +11: [2023-05-11 00:16:26,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +31: [2023-05-11 00:16:26,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +18: [2023-05-11 00:16:26,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +11: [2023-05-11 00:16:26,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +11: [2023-05-11 00:16:26,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +14: [2023-05-11 00:16:26,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +14: [2023-05-11 00:16:26,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 2: [2023-05-11 00:16:26,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 2: [2023-05-11 00:16:26,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +14: [2023-05-11 00:16:26,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +14: [2023-05-11 00:16:26,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +14: [2023-05-11 00:16:26,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +14: [2023-05-11 00:16:26,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +18: [2023-05-11 00:16:26,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +31: [2023-05-11 00:16:26,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +31: [2023-05-11 00:16:26,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +31: [2023-05-11 00:16:26,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +31: [2023-05-11 00:16:26,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +31: [2023-05-11 00:16:26,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +31: [2023-05-11 00:16:26,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +31: [2023-05-11 00:16:26,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +23: [2023-05-11 00:16:26,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +12: [2023-05-11 00:16:26,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 3: [2023-05-11 00:16:26,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 6: [2023-05-11 00:16:26,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +18: [2023-05-11 00:16:26,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +12: [2023-05-11 00:16:26,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +31: [2023-05-11 00:16:26,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +31: [2023-05-11 00:16:26,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +23: [2023-05-11 00:16:26,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +23: [2023-05-11 00:16:26,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 5: [2023-05-11 00:16:26,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +17: [2023-05-11 00:16:26,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +17: [2023-05-11 00:16:26,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +17: [2023-05-11 00:16:26,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +12: [2023-05-11 00:16:26,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +14: [2023-05-11 00:16:26,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +14: [2023-05-11 00:16:26,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +14: [2023-05-11 00:16:26,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +14: [2023-05-11 00:16:26,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +31: [2023-05-11 00:16:26,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +31: [2023-05-11 00:16:26,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +31: [2023-05-11 00:16:26,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +31: [2023-05-11 00:16:26,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +18: [2023-05-11 00:16:26,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +30: [2023-05-11 00:16:26,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +30: [2023-05-11 00:16:26,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +30: [2023-05-11 00:16:26,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +30: [2023-05-11 00:16:26,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +17: [2023-05-11 00:16:26,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 4: [2023-05-11 00:16:26,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +14: [2023-05-11 00:16:26,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 4: [2023-05-11 00:16:26,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +14: [2023-05-11 00:16:26,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +30: [2023-05-11 00:16:26,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +30: [2023-05-11 00:16:26,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +30: [2023-05-11 00:16:26,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +30: [2023-05-11 00:16:26,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +30: [2023-05-11 00:16:26,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +30: [2023-05-11 00:16:26,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 6: [2023-05-11 00:16:26,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +30: [2023-05-11 00:16:26,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +30: [2023-05-11 00:16:26,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 2: [2023-05-11 00:16:26,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +23: [2023-05-11 00:16:26,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +30: [2023-05-11 00:16:26,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +30: [2023-05-11 00:16:26,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 1: [2023-05-11 00:16:26,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +30: [2023-05-11 00:16:26,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +30: [2023-05-11 00:16:26,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 4: [2023-05-11 00:16:26,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +23: [2023-05-11 00:16:26,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +17: [2023-05-11 00:16:26,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +23: [2023-05-11 00:16:26,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +17: [2023-05-11 00:16:26,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +17: [2023-05-11 00:16:26,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +17: [2023-05-11 00:16:26,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +14: [2023-05-11 00:16:26,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +30: [2023-05-11 00:16:26,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +30: [2023-05-11 00:16:26,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +23: [2023-05-11 00:16:26,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +25: [2023-05-11 00:16:26,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +25: [2023-05-11 00:16:26,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +25: [2023-05-11 00:16:26,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +25: [2023-05-11 00:16:26,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +25: [2023-05-11 00:16:26,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +25: [2023-05-11 00:16:26,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +25: [2023-05-11 00:16:26,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +25: [2023-05-11 00:16:26,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +25: [2023-05-11 00:16:26,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +25: [2023-05-11 00:16:26,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +25: [2023-05-11 00:16:26,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 5: [2023-05-11 00:16:26,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +25: [2023-05-11 00:16:26,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +25: [2023-05-11 00:16:26,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 1: [2023-05-11 00:16:26,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +25: [2023-05-11 00:16:26,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +25: [2023-05-11 00:16:26,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +25: [2023-05-11 00:16:26,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 6: [2023-05-11 00:16:26,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +31: [2023-05-11 00:16:26,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +14: [2023-05-11 00:16:26,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +14: [2023-05-11 00:16:26,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +22: [2023-05-11 00:16:26,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +22: [2023-05-11 00:16:26,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +22: [2023-05-11 00:16:26,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +22: [2023-05-11 00:16:26,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +22: [2023-05-11 00:16:26,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +22: [2023-05-11 00:16:26,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +30: [2023-05-11 00:16:26,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +14: [2023-05-11 00:16:26,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +30: [2023-05-11 00:16:26,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +23: [2023-05-11 00:16:26,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +14: [2023-05-11 00:16:26,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +22: [2023-05-11 00:16:26,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +22: [2023-05-11 00:16:26,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +14: [2023-05-11 00:16:26,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +14: [2023-05-11 00:16:26,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +11: [2023-05-11 00:16:26,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +11: [2023-05-11 00:16:26,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +31: [2023-05-11 00:16:26,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +11: [2023-05-11 00:16:26,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +11: [2023-05-11 00:16:26,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +25: [2023-05-11 00:16:26,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +11: [2023-05-11 00:16:26,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +11: [2023-05-11 00:16:26,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +11: [2023-05-11 00:16:26,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +11: [2023-05-11 00:16:26,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +11: [2023-05-11 00:16:26,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +11: [2023-05-11 00:16:26,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +11: [2023-05-11 00:16:26,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +11: [2023-05-11 00:16:26,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +11: [2023-05-11 00:16:26,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +11: [2023-05-11 00:16:26,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +31: [2023-05-11 00:16:26,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +31: [2023-05-11 00:16:26,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +31: [2023-05-11 00:16:26,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +11: [2023-05-11 00:16:26,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +11: [2023-05-11 00:16:26,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +14: [2023-05-11 00:16:26,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +22: [2023-05-11 00:16:26,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +14: [2023-05-11 00:16:26,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +25: [2023-05-11 00:16:26,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 1: [2023-05-11 00:16:26,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +27: [2023-05-11 00:16:26,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +27: [2023-05-11 00:16:26,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +22: [2023-05-11 00:16:26,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +27: [2023-05-11 00:16:26,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +27: [2023-05-11 00:16:26,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +27: [2023-05-11 00:16:26,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +27: [2023-05-11 00:16:26,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +27: [2023-05-11 00:16:26,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +27: [2023-05-11 00:16:26,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +27: [2023-05-11 00:16:26,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +27: [2023-05-11 00:16:26,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +30: [2023-05-11 00:16:26,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +27: [2023-05-11 00:16:26,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +14: [2023-05-11 00:16:26,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +14: [2023-05-11 00:16:26,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 1: [2023-05-11 00:16:26,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 4: [2023-05-11 00:16:26,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +31: [2023-05-11 00:16:26,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +30: [2023-05-11 00:16:26,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 0: [2023-05-11 00:16:26,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 0: [2023-05-11 00:16:26,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +31: [2023-05-11 00:16:26,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +31: [2023-05-11 00:16:26,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 1: [2023-05-11 00:16:26,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 4: [2023-05-11 00:16:26,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +14: [2023-05-11 00:16:26,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 3: [2023-05-11 00:16:26,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 7: [2023-05-11 00:16:26,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +11: [2023-05-11 00:16:26,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +27: [2023-05-11 00:16:26,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +27: [2023-05-11 00:16:26,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 7: [2023-05-11 00:16:26,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 4: [2023-05-11 00:16:26,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +27: [2023-05-11 00:16:26,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +27: [2023-05-11 00:16:26,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +27: [2023-05-11 00:16:26,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 4: [2023-05-11 00:16:26,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +11: [2023-05-11 00:16:26,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +22: [2023-05-11 00:16:26,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +14: [2023-05-11 00:16:26,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +14: [2023-05-11 00:16:26,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +25: [2023-05-11 00:16:26,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +30: [2023-05-11 00:16:26,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +16: [2023-05-11 00:16:26,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +16: [2023-05-11 00:16:26,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +16: [2023-05-11 00:16:26,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +16: [2023-05-11 00:16:26,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +30: [2023-05-11 00:16:26,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +16: [2023-05-11 00:16:26,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +16: [2023-05-11 00:16:26,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +22: [2023-05-11 00:16:26,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +29: [2023-05-11 00:16:26,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +29: [2023-05-11 00:16:26,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +29: [2023-05-11 00:16:26,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +29: [2023-05-11 00:16:26,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +29: [2023-05-11 00:16:26,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +16: [2023-05-11 00:16:26,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +16: [2023-05-11 00:16:26,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +29: [2023-05-11 00:16:26,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +29: [2023-05-11 00:16:26,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +25: [2023-05-11 00:16:26,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +24: [2023-05-11 00:16:26,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +24: [2023-05-11 00:16:26,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +24: [2023-05-11 00:16:26,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +24: [2023-05-11 00:16:26,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +24: [2023-05-11 00:16:26,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +24: [2023-05-11 00:16:26,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +24: [2023-05-11 00:16:26,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +24: [2023-05-11 00:16:26,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +29: [2023-05-11 00:16:26,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +26: [2023-05-11 00:16:26,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +26: [2023-05-11 00:16:26,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +26: [2023-05-11 00:16:26,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +27: [2023-05-11 00:16:26,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +26: [2023-05-11 00:16:26,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +26: [2023-05-11 00:16:26,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +26: [2023-05-11 00:16:26,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +26: [2023-05-11 00:16:26,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +26: [2023-05-11 00:16:26,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +11: [2023-05-11 00:16:26,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +26: [2023-05-11 00:16:26,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +15: [2023-05-11 00:16:26,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +15: [2023-05-11 00:16:26,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +15: [2023-05-11 00:16:26,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +15: [2023-05-11 00:16:26,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +15: [2023-05-11 00:16:26,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +15: [2023-05-11 00:16:26,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +15: [2023-05-11 00:16:26,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +15: [2023-05-11 00:16:26,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +26: [2023-05-11 00:16:26,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +26: [2023-05-11 00:16:26,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 3: [2023-05-11 00:16:26,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +11: [2023-05-11 00:16:26,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +22: [2023-05-11 00:16:26,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +22: [2023-05-11 00:16:26,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +22: [2023-05-11 00:16:26,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +25: [2023-05-11 00:16:26,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +14: [2023-05-11 00:16:26,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +14: [2023-05-11 00:16:26,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +22: [2023-05-11 00:16:26,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +25: [2023-05-11 00:16:26,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +25: [2023-05-11 00:16:26,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +20: [2023-05-11 00:16:26,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +20: [2023-05-11 00:16:26,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +20: [2023-05-11 00:16:26,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +20: [2023-05-11 00:16:26,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +20: [2023-05-11 00:16:26,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +20: [2023-05-11 00:16:26,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +20: [2023-05-11 00:16:26,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +20: [2023-05-11 00:16:26,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +20: [2023-05-11 00:16:26,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +20: [2023-05-11 00:16:26,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +26: [2023-05-11 00:16:26,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 7: [2023-05-11 00:16:26,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +20: [2023-05-11 00:16:26,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 7: [2023-05-11 00:16:26,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +20: [2023-05-11 00:16:26,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +27: [2023-05-11 00:16:26,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +20: [2023-05-11 00:16:26,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +20: [2023-05-11 00:16:26,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +20: [2023-05-11 00:16:26,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +22: [2023-05-11 00:16:26,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +20: [2023-05-11 00:16:26,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +24: [2023-05-11 00:16:26,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +16: [2023-05-11 00:16:26,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +22: [2023-05-11 00:16:26,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +11: [2023-05-11 00:16:26,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +27: [2023-05-11 00:16:26,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +29: [2023-05-11 00:16:26,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 3: [2023-05-11 00:16:26,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +11: [2023-05-11 00:16:26,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +26: [2023-05-11 00:16:26,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +15: [2023-05-11 00:16:26,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +15: [2023-05-11 00:16:26,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +16: [2023-05-11 00:16:26,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +22: [2023-05-11 00:16:26,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +16: [2023-05-11 00:16:26,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +16: [2023-05-11 00:16:26,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +26: [2023-05-11 00:16:26,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +25: [2023-05-11 00:16:26,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +15: [2023-05-11 00:16:26,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +22: [2023-05-11 00:16:26,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +22: [2023-05-11 00:16:26,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +29: [2023-05-11 00:16:26,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +29: [2023-05-11 00:16:26,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +24: [2023-05-11 00:16:26,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +24: [2023-05-11 00:16:26,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +24: [2023-05-11 00:16:26,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +15: [2023-05-11 00:16:26,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +19: [2023-05-11 00:16:26,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +19: [2023-05-11 00:16:26,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +19: [2023-05-11 00:16:26,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +19: [2023-05-11 00:16:26,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +20: [2023-05-11 00:16:26,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +19: [2023-05-11 00:16:26,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +19: [2023-05-11 00:16:26,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +26: [2023-05-11 00:16:26,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +26: [2023-05-11 00:16:26,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +17: [2023-05-11 00:16:26,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +17: [2023-05-11 00:16:26,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 8: [2023-05-11 00:16:26,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 5: [2023-05-11 00:16:26,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 5: [2023-05-11 00:16:26,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 5: [2023-05-11 00:16:26,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 0: [2023-05-11 00:16:26,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +22: [2023-05-11 00:16:26,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +22: [2023-05-11 00:16:26,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +11: [2023-05-11 00:16:26,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +20: [2023-05-11 00:16:26,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +22: [2023-05-11 00:16:26,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +19: [2023-05-11 00:16:26,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +19: [2023-05-11 00:16:26,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +27: [2023-05-11 00:16:26,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +17: [2023-05-11 00:16:26,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +17: [2023-05-11 00:16:26,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +16: [2023-05-11 00:16:26,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +15: [2023-05-11 00:16:26,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +29: [2023-05-11 00:16:26,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +15: [2023-05-11 00:16:26,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +24: [2023-05-11 00:16:26,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +11: [2023-05-11 00:16:26,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +16: [2023-05-11 00:16:26,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +16: [2023-05-11 00:16:26,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 2: [2023-05-11 00:16:26,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 2: [2023-05-11 00:16:26,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +19: [2023-05-11 00:16:26,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +27: [2023-05-11 00:16:26,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +17: [2023-05-11 00:16:26,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +17: [2023-05-11 00:16:26,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +17: [2023-05-11 00:16:26,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +17: [2023-05-11 00:16:26,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +17: [2023-05-11 00:16:26,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +17: [2023-05-11 00:16:26,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +26: [2023-05-11 00:16:26,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +15: [2023-05-11 00:16:26,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +24: [2023-05-11 00:16:26,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +26: [2023-05-11 00:16:26,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +20: [2023-05-11 00:16:26,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +19: [2023-05-11 00:16:26,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +15: [2023-05-11 00:16:26,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +16: [2023-05-11 00:16:26,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +13: [2023-05-11 00:16:26,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 2: [2023-05-11 00:16:26,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +13: [2023-05-11 00:16:26,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. + 5: [2023-05-11 00:16:26,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +17: [2023-05-11 00:16:26,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +26: [2023-05-11 00:16:26,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +26: [2023-05-11 00:16:26,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +26: [2023-05-11 00:16:26,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +17: [2023-05-11 00:16:26,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +17: [2023-05-11 00:16:26,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +27: [2023-05-11 00:16:26,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +24: [2023-05-11 00:16:26,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +13: [2023-05-11 00:16:26,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +19: [2023-05-11 00:16:26,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +17: [2023-05-11 00:16:26,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +17: [2023-05-11 00:16:26,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +24: [2023-05-11 00:16:26,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +17: [2023-05-11 00:16:26,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +19: [2023-05-11 00:16:26,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +13: [2023-05-11 00:16:26,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt. +22: [2023-05-11 00:16:26,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +29: [2023-05-11 00:16:26,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +29: [2023-05-11 00:16:26,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +22: [2023-05-11 00:16:26,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +20: [2023-05-11 00:16:26,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +26: [2023-05-11 00:16:26,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +26: [2023-05-11 00:16:26,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +19: [2023-05-11 00:16:26,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 1: [2023-05-11 00:16:26,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 8: [2023-05-11 00:16:26,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +19: [2023-05-11 00:16:26,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +29: [2023-05-11 00:16:26,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 0: [2023-05-11 00:16:26,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 0: [2023-05-11 00:16:26,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +19: [2023-05-11 00:16:26,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +13: [2023-05-11 00:16:26,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +13: [2023-05-11 00:16:26,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +20: [2023-05-11 00:16:26,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +13: [2023-05-11 00:16:26,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +19: [2023-05-11 00:16:26,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +19: [2023-05-11 00:16:26,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +26: [2023-05-11 00:16:26,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +26: [2023-05-11 00:16:26,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +27: [2023-05-11 00:16:26,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +26: [2023-05-11 00:16:26,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +29: [2023-05-11 00:16:26,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +20: [2023-05-11 00:16:26,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 5: [2023-05-11 00:16:26,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +19: [2023-05-11 00:16:26,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +13: [2023-05-11 00:16:26,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +27: [2023-05-11 00:16:26,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +22: [2023-05-11 00:16:26,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +20: [2023-05-11 00:16:26,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +26: [2023-05-11 00:16:26,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +22: [2023-05-11 00:16:26,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +22: [2023-05-11 00:16:26,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +19: [2023-05-11 00:16:26,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 5: [2023-05-11 00:16:26,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +22: [2023-05-11 00:16:26,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +26: [2023-05-11 00:16:26,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +22: [2023-05-11 00:16:26,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 5: [2023-05-11 00:16:26,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +26: [2023-05-11 00:16:26,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 8: [2023-05-11 00:16:26,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +22: [2023-05-11 00:16:26,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +20: [2023-05-11 00:16:26,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +26: [2023-05-11 00:16:26,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +26: [2023-05-11 00:16:26,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +13: [2023-05-11 00:16:26,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +13: [2023-05-11 00:16:26,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 5: [2023-05-11 00:16:26,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +13: [2023-05-11 00:16:26,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +13: [2023-05-11 00:16:26,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +13: [2023-05-11 00:16:26,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +13: [2023-05-11 00:16:26,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +13: [2023-05-11 00:16:26,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +13: [2023-05-11 00:16:26,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +13: [2023-05-11 00:16:26,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +17: [2023-05-11 00:16:26,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +17: [2023-05-11 00:16:26,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +19: [2023-05-11 00:16:26,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +13: [2023-05-11 00:16:26,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... +26: [2023-05-11 00:16:26,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +13: [2023-05-11 00:16:26,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +13: [2023-05-11 00:16:26,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +17: [2023-05-11 00:16:26,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +17: [2023-05-11 00:16:26,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +19: [2023-05-11 00:16:26,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +19: [2023-05-11 00:16:26,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 5: [2023-05-11 00:16:26,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +28: [2023-05-11 00:16:26,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +28: [2023-05-11 00:16:26,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +28: [2023-05-11 00:16:26,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +28: [2023-05-11 00:16:26,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +28: [2023-05-11 00:16:26,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +28: [2023-05-11 00:16:26,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +17: [2023-05-11 00:16:26,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +28: [2023-05-11 00:16:26,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +17: [2023-05-11 00:16:26,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +19: [2023-05-11 00:16:26,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... + 0: [2023-05-11 00:16:26,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +28: [2023-05-11 00:16:26,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +21: [2023-05-11 00:16:26,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +21: [2023-05-11 00:16:26,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +21: [2023-05-11 00:16:26,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +21: [2023-05-11 00:16:26,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +21: [2023-05-11 00:16:26,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +21: [2023-05-11 00:16:26,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +21: [2023-05-11 00:16:26,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +21: [2023-05-11 00:16:26,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +17: [2023-05-11 00:16:26,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +17: [2023-05-11 00:16:26,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +19: [2023-05-11 00:16:26,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +13: [2023-05-11 00:16:26,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +13: [2023-05-11 00:16:26,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +19: [2023-05-11 00:16:26,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +19: [2023-05-11 00:16:26,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +19: [2023-05-11 00:16:26,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +19: [2023-05-11 00:16:26,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +13: [2023-05-11 00:16:26,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +13: [2023-05-11 00:16:26,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +28: [2023-05-11 00:16:26,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +13: [2023-05-11 00:16:26,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +19: [2023-05-11 00:16:26,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +13: [2023-05-11 00:16:26,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +28: [2023-05-11 00:16:26,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +28: [2023-05-11 00:16:26,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +21: [2023-05-11 00:16:26,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +19: [2023-05-11 00:16:26,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt... +19: [2023-05-11 00:16:26,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +10: [2023-05-11 00:16:26,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +10: [2023-05-11 00:16:26,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +10: [2023-05-11 00:16:26,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +10: [2023-05-11 00:16:26,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +10: [2023-05-11 00:16:26,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +10: [2023-05-11 00:16:26,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +10: [2023-05-11 00:16:26,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +19: [2023-05-11 00:16:26,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +21: [2023-05-11 00:16:26,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +21: [2023-05-11 00:16:26,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +10: [2023-05-11 00:16:26,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +21: [2023-05-11 00:16:26,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +13: [2023-05-11 00:16:26,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +28: [2023-05-11 00:16:26,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +28: [2023-05-11 00:16:26,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +28: [2023-05-11 00:16:26,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +18: [2023-05-11 00:16:26,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +18: [2023-05-11 00:16:26,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +18: [2023-05-11 00:16:26,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +18: [2023-05-11 00:16:26,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +18: [2023-05-11 00:16:26,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +18: [2023-05-11 00:16:26,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +18: [2023-05-11 00:16:26,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +18: [2023-05-11 00:16:26,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +21: [2023-05-11 00:16:26,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +28: [2023-05-11 00:16:26,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +13: [2023-05-11 00:16:26,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +13: [2023-05-11 00:16:26,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt. +10: [2023-05-11 00:16:26,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +21: [2023-05-11 00:16:26,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +21: [2023-05-11 00:16:26,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +10: [2023-05-11 00:16:26,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +13: [2023-05-11 00:16:26,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +28: [2023-05-11 00:16:26,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +10: [2023-05-11 00:16:26,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +21: [2023-05-11 00:16:26,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +13: [2023-05-11 00:16:26,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +13: [2023-05-11 00:16:26,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 5: [2023-05-11 00:16:26,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +12: [2023-05-11 00:16:26,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +12: [2023-05-11 00:16:26,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +12: [2023-05-11 00:16:26,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +12: [2023-05-11 00:16:26,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +12: [2023-05-11 00:16:26,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +12: [2023-05-11 00:16:26,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +18: [2023-05-11 00:16:26,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +18: [2023-05-11 00:16:26,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +18: [2023-05-11 00:16:26,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +12: [2023-05-11 00:16:26,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +12: [2023-05-11 00:16:26,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +10: [2023-05-11 00:16:26,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +10: [2023-05-11 00:16:26,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +29: [2023-05-11 00:16:26,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +29: [2023-05-11 00:16:26,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +29: [2023-05-11 00:16:26,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 5: [2023-05-11 00:16:26,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +29: [2023-05-11 00:16:26,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +10: [2023-05-11 00:16:26,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 5: [2023-05-11 00:16:26,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +18: [2023-05-11 00:16:26,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +10: [2023-05-11 00:16:26,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +10: [2023-05-11 00:16:26,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +10: [2023-05-11 00:16:26,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +10: [2023-05-11 00:16:26,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +12: [2023-05-11 00:16:26,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +12: [2023-05-11 00:16:26,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +12: [2023-05-11 00:16:26,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +18: [2023-05-11 00:16:26,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +18: [2023-05-11 00:16:26,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +18: [2023-05-11 00:16:26,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +30: [2023-05-11 00:16:26,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +30: [2023-05-11 00:16:26,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +30: [2023-05-11 00:16:26,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +30: [2023-05-11 00:16:26,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +30: [2023-05-11 00:16:26,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +30: [2023-05-11 00:16:26,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +30: [2023-05-11 00:16:26,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +30: [2023-05-11 00:16:26,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +10: [2023-05-11 00:16:26,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +29: [2023-05-11 00:16:26,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +29: [2023-05-11 00:16:26,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +29: [2023-05-11 00:16:26,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +12: [2023-05-11 00:16:26,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +12: [2023-05-11 00:16:26,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +18: [2023-05-11 00:16:26,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +29: [2023-05-11 00:16:26,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +10: [2023-05-11 00:16:26,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +10: [2023-05-11 00:16:26,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +12: [2023-05-11 00:16:26,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +12: [2023-05-11 00:16:26,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +10: [2023-05-11 00:16:26,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +12: [2023-05-11 00:16:26,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +10: [2023-05-11 00:16:26,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +12: [2023-05-11 00:16:26,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +12: [2023-05-11 00:16:26,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +30: [2023-05-11 00:16:26,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +30: [2023-05-11 00:16:26,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +10: [2023-05-11 00:16:26,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +30: [2023-05-11 00:16:26,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +29: [2023-05-11 00:16:26,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +29: [2023-05-11 00:16:26,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +29: [2023-05-11 00:16:26,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +29: [2023-05-11 00:16:26,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 6: [2023-05-11 00:16:26,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +12: [2023-05-11 00:16:26,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +12: [2023-05-11 00:16:26,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +12: [2023-05-11 00:16:26,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +29: [2023-05-11 00:16:26,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +29: [2023-05-11 00:16:26,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 6: [2023-05-11 00:16:26,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +29: [2023-05-11 00:16:26,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +29: [2023-05-11 00:16:26,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +12: [2023-05-11 00:16:26,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +30: [2023-05-11 00:16:26,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +30: [2023-05-11 00:16:26,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +10: [2023-05-11 00:16:26,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +30: [2023-05-11 00:16:26,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +10: [2023-05-11 00:16:26,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 1: [2023-05-11 00:16:26,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +10: [2023-05-11 00:16:26,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +10: [2023-05-11 00:16:26,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +10: [2023-05-11 00:16:26,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +10: [2023-05-11 00:16:26,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +10: [2023-05-11 00:16:26,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +10: [2023-05-11 00:16:26,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +12: [2023-05-11 00:16:26,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +12: [2023-05-11 00:16:26,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +30: [2023-05-11 00:16:26,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +12: [2023-05-11 00:16:26,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +12: [2023-05-11 00:16:26,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 6: [2023-05-11 00:16:26,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +12: [2023-05-11 00:16:26,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +12: [2023-05-11 00:16:26,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 6: [2023-05-11 00:16:26,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +23: [2023-05-11 00:16:26,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +23: [2023-05-11 00:16:26,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +23: [2023-05-11 00:16:26,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +23: [2023-05-11 00:16:26,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +23: [2023-05-11 00:16:26,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +23: [2023-05-11 00:16:26,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +23: [2023-05-11 00:16:26,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +23: [2023-05-11 00:16:26,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +12: [2023-05-11 00:16:26,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +12: [2023-05-11 00:16:26,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +12: [2023-05-11 00:16:26,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +12: [2023-05-11 00:16:26,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +30: [2023-05-11 00:16:26,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +14: [2023-05-11 00:16:26,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +14: [2023-05-11 00:16:26,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +14: [2023-05-11 00:16:26,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +14: [2023-05-11 00:16:26,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +14: [2023-05-11 00:16:26,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +14: [2023-05-11 00:16:26,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +14: [2023-05-11 00:16:26,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +14: [2023-05-11 00:16:26,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 4: [2023-05-11 00:16:26,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +14: [2023-05-11 00:16:26,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 9: [2023-05-11 00:16:26,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +15: [2023-05-11 00:16:26,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 6: [2023-05-11 00:16:26,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +16: [2023-05-11 00:16:26,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +23: [2023-05-11 00:16:26,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +23: [2023-05-11 00:16:26,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +31: [2023-05-11 00:16:26,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 1: [2023-05-11 00:16:26,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +23: [2023-05-11 00:16:26,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +23: [2023-05-11 00:16:26,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +15: [2023-05-11 00:16:26,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +31: [2023-05-11 00:16:26,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +31: [2023-05-11 00:16:26,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +31: [2023-05-11 00:16:26,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +18: [2023-05-11 00:16:26,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +31: [2023-05-11 00:16:26,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +31: [2023-05-11 00:16:26,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +28: [2023-05-11 00:16:26,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 9: [2023-05-11 00:16:26,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +15: [2023-05-11 00:16:26,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +16: [2023-05-11 00:16:26,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +15: [2023-05-11 00:16:26,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +31: [2023-05-11 00:16:26,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +31: [2023-05-11 00:16:26,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +28: [2023-05-11 00:16:26,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 3: [2023-05-11 00:16:26,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +31: [2023-05-11 00:16:26,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +31: [2023-05-11 00:16:26,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 3: [2023-05-11 00:16:26,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +18: [2023-05-11 00:16:26,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 4: [2023-05-11 00:16:26,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +14: [2023-05-11 00:16:26,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +14: [2023-05-11 00:16:26,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +14: [2023-05-11 00:16:26,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +15: [2023-05-11 00:16:26,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +16: [2023-05-11 00:16:26,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +31: [2023-05-11 00:16:26,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +16: [2023-05-11 00:16:26,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +31: [2023-05-11 00:16:26,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 9: [2023-05-11 00:16:26,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +16: [2023-05-11 00:16:26,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +28: [2023-05-11 00:16:26,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 9: [2023-05-11 00:16:26,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +18: [2023-05-11 00:16:26,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +18: [2023-05-11 00:16:26,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +15: [2023-05-11 00:16:26,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +22: [2023-05-11 00:16:26,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +22: [2023-05-11 00:16:26,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +22: [2023-05-11 00:16:26,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +22: [2023-05-11 00:16:26,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +22: [2023-05-11 00:16:26,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +22: [2023-05-11 00:16:26,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +31: [2023-05-11 00:16:26,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +22: [2023-05-11 00:16:26,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +22: [2023-05-11 00:16:26,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +23: [2023-05-11 00:16:26,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +23: [2023-05-11 00:16:26,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +28: [2023-05-11 00:16:26,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +23: [2023-05-11 00:16:26,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +31: [2023-05-11 00:16:26,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +23: [2023-05-11 00:16:26,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +15: [2023-05-11 00:16:26,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +18: [2023-05-11 00:16:26,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +15: [2023-05-11 00:16:26,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +28: [2023-05-11 00:16:26,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +28: [2023-05-11 00:16:26,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +16: [2023-05-11 00:16:26,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +14: [2023-05-11 00:16:26,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 9: [2023-05-11 00:16:26,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +18: [2023-05-11 00:16:26,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +16: [2023-05-11 00:16:26,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +14: [2023-05-11 00:16:26,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +28: [2023-05-11 00:16:26,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +31: [2023-05-11 00:16:26,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +31: [2023-05-11 00:16:26,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +28: [2023-05-11 00:16:26,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +16: [2023-05-11 00:16:26,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +18: [2023-05-11 00:16:26,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +18: [2023-05-11 00:16:26,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +15: [2023-05-11 00:16:26,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +14: [2023-05-11 00:16:26,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +31: [2023-05-11 00:16:26,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +31: [2023-05-11 00:16:26,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +15: [2023-05-11 00:16:26,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +22: [2023-05-11 00:16:26,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +24: [2023-05-11 00:16:26,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +24: [2023-05-11 00:16:26,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +24: [2023-05-11 00:16:26,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +22: [2023-05-11 00:16:26,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +24: [2023-05-11 00:16:26,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +15: [2023-05-11 00:16:26,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +30: [2023-05-11 00:16:26,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +30: [2023-05-11 00:16:26,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +15: [2023-05-11 00:16:26,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 9: [2023-05-11 00:16:26,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +14: [2023-05-11 00:16:26,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +18: [2023-05-11 00:16:26,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +28: [2023-05-11 00:16:26,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +28: [2023-05-11 00:16:26,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +18: [2023-05-11 00:16:26,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +22: [2023-05-11 00:16:26,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +16: [2023-05-11 00:16:26,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +16: [2023-05-11 00:16:26,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +22: [2023-05-11 00:16:26,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +18: [2023-05-11 00:16:26,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +18: [2023-05-11 00:16:26,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +11: [2023-05-11 00:16:26,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +11: [2023-05-11 00:16:26,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +16: [2023-05-11 00:16:26,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +11: [2023-05-11 00:16:26,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +11: [2023-05-11 00:16:26,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +16: [2023-05-11 00:16:26,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +11: [2023-05-11 00:16:26,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +11: [2023-05-11 00:16:26,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +11: [2023-05-11 00:16:26,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +11: [2023-05-11 00:16:26,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +30: [2023-05-11 00:16:26,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +30: [2023-05-11 00:16:26,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +28: [2023-05-11 00:16:26,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +28: [2023-05-11 00:16:26,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +31: [2023-05-11 00:16:26,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +15: [2023-05-11 00:16:26,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +22: [2023-05-11 00:16:26,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +15: [2023-05-11 00:16:26,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 9: [2023-05-11 00:16:26,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +31: [2023-05-11 00:16:26,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +31: [2023-05-11 00:16:26,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 7: [2023-05-11 00:16:26,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +16: [2023-05-11 00:16:26,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +16: [2023-05-11 00:16:26,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 7: [2023-05-11 00:16:26,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +24: [2023-05-11 00:16:26,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +31: [2023-05-11 00:16:26,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +28: [2023-05-11 00:16:26,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +28: [2023-05-11 00:16:26,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +31: [2023-05-11 00:16:26,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +18: [2023-05-11 00:16:26,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +18: [2023-05-11 00:16:26,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +15: [2023-05-11 00:16:26,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +15: [2023-05-11 00:16:26,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +30: [2023-05-11 00:16:26,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +30: [2023-05-11 00:16:26,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +31: [2023-05-11 00:16:26,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +16: [2023-05-11 00:16:26,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +16: [2023-05-11 00:16:26,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +24: [2023-05-11 00:16:26,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +24: [2023-05-11 00:16:26,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +22: [2023-05-11 00:16:26,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +22: [2023-05-11 00:16:26,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +22: [2023-05-11 00:16:26,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +27: [2023-05-11 00:16:26,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +27: [2023-05-11 00:16:26,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +27: [2023-05-11 00:16:26,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +27: [2023-05-11 00:16:26,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +27: [2023-05-11 00:16:26,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +24: [2023-05-11 00:16:26,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +27: [2023-05-11 00:16:26,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +27: [2023-05-11 00:16:26,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +18: [2023-05-11 00:16:26,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +18: [2023-05-11 00:16:26,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +28: [2023-05-11 00:16:26,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +30: [2023-05-11 00:16:26,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +27: [2023-05-11 00:16:26,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +28: [2023-05-11 00:16:26,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +11: [2023-05-11 00:16:26,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +30: [2023-05-11 00:16:26,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +11: [2023-05-11 00:16:26,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 9: [2023-05-11 00:16:26,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 7: [2023-05-11 00:16:26,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +31: [2023-05-11 00:16:26,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +31: [2023-05-11 00:16:26,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +31: [2023-05-11 00:16:26,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +11: [2023-05-11 00:16:26,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +11: [2023-05-11 00:16:26,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +31: [2023-05-11 00:16:26,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 6: [2023-05-11 00:16:26,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +24: [2023-05-11 00:16:26,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +24: [2023-05-11 00:16:26,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +24: [2023-05-11 00:16:26,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +24: [2023-05-11 00:16:26,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 7: [2023-05-11 00:16:26,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +24: [2023-05-11 00:16:26,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +24: [2023-05-11 00:16:26,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +11: [2023-05-11 00:16:26,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +24: [2023-05-11 00:16:26,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +24: [2023-05-11 00:16:26,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +21: [2023-05-11 00:16:26,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +27: [2023-05-11 00:16:26,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +27: [2023-05-11 00:16:26,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +27: [2023-05-11 00:16:26,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +11: [2023-05-11 00:16:26,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +25: [2023-05-11 00:16:26,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 3: [2023-05-11 00:16:26,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 3: [2023-05-11 00:16:26,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +25: [2023-05-11 00:16:26,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +25: [2023-05-11 00:16:26,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +25: [2023-05-11 00:16:26,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +23: [2023-05-11 00:16:26,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +23: [2023-05-11 00:16:26,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 6: [2023-05-11 00:16:26,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 6: [2023-05-11 00:16:26,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +21: [2023-05-11 00:16:26,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +11: [2023-05-11 00:16:26,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +31: [2023-05-11 00:16:26,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +31: [2023-05-11 00:16:26,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +31: [2023-05-11 00:16:26,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +31: [2023-05-11 00:16:26,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +23: [2023-05-11 00:16:26,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 3: [2023-05-11 00:16:26,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +14: [2023-05-11 00:16:26,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +14: [2023-05-11 00:16:26,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 3: [2023-05-11 00:16:26,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 7: [2023-05-11 00:16:26,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +14: [2023-05-11 00:16:26,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +14: [2023-05-11 00:16:26,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 7: [2023-05-11 00:16:26,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +11: [2023-05-11 00:16:26,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +21: [2023-05-11 00:16:26,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +23: [2023-05-11 00:16:26,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 6: [2023-05-11 00:16:26,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +21: [2023-05-11 00:16:26,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +30: [2023-05-11 00:16:26,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +30: [2023-05-11 00:16:26,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 7: [2023-05-11 00:16:26,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +30: [2023-05-11 00:16:26,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +30: [2023-05-11 00:16:26,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +30: [2023-05-11 00:16:26,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +30: [2023-05-11 00:16:26,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +21: [2023-05-11 00:16:26,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +23: [2023-05-11 00:16:26,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 6: [2023-05-11 00:16:26,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +27: [2023-05-11 00:16:26,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +27: [2023-05-11 00:16:26,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +21: [2023-05-11 00:16:26,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +23: [2023-05-11 00:16:26,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +25: [2023-05-11 00:16:26,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +25: [2023-05-11 00:16:26,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +27: [2023-05-11 00:16:26,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 2: [2023-05-11 00:16:26,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +23: [2023-05-11 00:16:26,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +27: [2023-05-11 00:16:26,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +25: [2023-05-11 00:16:26,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +14: [2023-05-11 00:16:26,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +14: [2023-05-11 00:16:26,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +14: [2023-05-11 00:16:26,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +27: [2023-05-11 00:16:26,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +23: [2023-05-11 00:16:26,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +25: [2023-05-11 00:16:26,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +21: [2023-05-11 00:16:26,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +26: [2023-05-11 00:16:26,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +26: [2023-05-11 00:16:26,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +26: [2023-05-11 00:16:26,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +26: [2023-05-11 00:16:26,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +21: [2023-05-11 00:16:26,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +26: [2023-05-11 00:16:26,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +26: [2023-05-11 00:16:26,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +26: [2023-05-11 00:16:26,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +26: [2023-05-11 00:16:26,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +14: [2023-05-11 00:16:26,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +30: [2023-05-11 00:16:26,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +30: [2023-05-11 00:16:26,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +20: [2023-05-11 00:16:26,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +20: [2023-05-11 00:16:26,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +20: [2023-05-11 00:16:26,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +20: [2023-05-11 00:16:26,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +20: [2023-05-11 00:16:26,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +20: [2023-05-11 00:16:26,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +20: [2023-05-11 00:16:26,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +20: [2023-05-11 00:16:26,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +27: [2023-05-11 00:16:26,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +27: [2023-05-11 00:16:26,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 4: [2023-05-11 00:16:26,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 0: [2023-05-11 00:16:26,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +25: [2023-05-11 00:16:26,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +25: [2023-05-11 00:16:26,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 6: [2023-05-11 00:16:26,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +27: [2023-05-11 00:16:26,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +23: [2023-05-11 00:16:26,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +25: [2023-05-11 00:16:26,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +25: [2023-05-11 00:16:26,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +25: [2023-05-11 00:16:26,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +23: [2023-05-11 00:16:26,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +27: [2023-05-11 00:16:26,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +27: [2023-05-11 00:16:26,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 4: [2023-05-11 00:16:26,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +25: [2023-05-11 00:16:26,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +25: [2023-05-11 00:16:26,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +25: [2023-05-11 00:16:26,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 3: [2023-05-11 00:16:26,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 3: [2023-05-11 00:16:26,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 6: [2023-05-11 00:16:26,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +27: [2023-05-11 00:16:26,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +27: [2023-05-11 00:16:26,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +25: [2023-05-11 00:16:26,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +21: [2023-05-11 00:16:26,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +14: [2023-05-11 00:16:26,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +21: [2023-05-11 00:16:26,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +14: [2023-05-11 00:16:26,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +25: [2023-05-11 00:16:26,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +25: [2023-05-11 00:16:26,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +25: [2023-05-11 00:16:26,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +14: [2023-05-11 00:16:26,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +14: [2023-05-11 00:16:26,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 2: [2023-05-11 00:16:26,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +21: [2023-05-11 00:16:26,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +21: [2023-05-11 00:16:26,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +23: [2023-05-11 00:16:26,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +23: [2023-05-11 00:16:26,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +23: [2023-05-11 00:16:26,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +23: [2023-05-11 00:16:26,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +23: [2023-05-11 00:16:26,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +23: [2023-05-11 00:16:26,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +14: [2023-05-11 00:16:26,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +14: [2023-05-11 00:16:26,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 6: [2023-05-11 00:16:26,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 6: [2023-05-11 00:16:26,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 3: [2023-05-11 00:16:26,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 3: [2023-05-11 00:16:26,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 4: [2023-05-11 00:16:26,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +27: [2023-05-11 00:16:26,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +20: [2023-05-11 00:16:26,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +20: [2023-05-11 00:16:26,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +11: [2023-05-11 00:16:26,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 8: [2023-05-11 00:16:26,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +21: [2023-05-11 00:16:26,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +21: [2023-05-11 00:16:26,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +11: [2023-05-11 00:16:26,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +11: [2023-05-11 00:16:26,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +11: [2023-05-11 00:16:26,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +20: [2023-05-11 00:16:26,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +20: [2023-05-11 00:16:26,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +14: [2023-05-11 00:16:26,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +14: [2023-05-11 00:16:26,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +25: [2023-05-11 00:16:26,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +25: [2023-05-11 00:16:26,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +25: [2023-05-11 00:16:26,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +25: [2023-05-11 00:16:26,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +25: [2023-05-11 00:16:26,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 4: [2023-05-11 00:16:26,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +21: [2023-05-11 00:16:26,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +21: [2023-05-11 00:16:26,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 2: [2023-05-11 00:16:26,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 4: [2023-05-11 00:16:26,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 2: [2023-05-11 00:16:26,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +26: [2023-05-11 00:16:26,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +26: [2023-05-11 00:16:26,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +26: [2023-05-11 00:16:26,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +26: [2023-05-11 00:16:26,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +25: [2023-05-11 00:16:26,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 2: [2023-05-11 00:16:26,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +27: [2023-05-11 00:16:26,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +27: [2023-05-11 00:16:26,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 0: [2023-05-11 00:16:26,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 0: [2023-05-11 00:16:26,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +20: [2023-05-11 00:16:26,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +11: [2023-05-11 00:16:26,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +11: [2023-05-11 00:16:26,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +20: [2023-05-11 00:16:26,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +25: [2023-05-11 00:16:26,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +25: [2023-05-11 00:16:26,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +20: [2023-05-11 00:16:26,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +11: [2023-05-11 00:16:26,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +11: [2023-05-11 00:16:26,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +27: [2023-05-11 00:16:26,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +27: [2023-05-11 00:16:26,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +20: [2023-05-11 00:16:26,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +26: [2023-05-11 00:16:26,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +26: [2023-05-11 00:16:26,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +25: [2023-05-11 00:16:26,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +26: [2023-05-11 00:16:26,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +26: [2023-05-11 00:16:26,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +19: [2023-05-11 00:16:26,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 8: [2023-05-11 00:16:26,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +27: [2023-05-11 00:16:26,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +27: [2023-05-11 00:16:26,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +27: [2023-05-11 00:16:26,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +27: [2023-05-11 00:16:26,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 0: [2023-05-11 00:16:26,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +25: [2023-05-11 00:16:26,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +19: [2023-05-11 00:16:26,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +19: [2023-05-11 00:16:26,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +20: [2023-05-11 00:16:26,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 8: [2023-05-11 00:16:26,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +19: [2023-05-11 00:16:26,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +20: [2023-05-11 00:16:26,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +25: [2023-05-11 00:16:26,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +13: [2023-05-11 00:16:26,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +13: [2023-05-11 00:16:26,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +13: [2023-05-11 00:16:26,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +13: [2023-05-11 00:16:26,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +13: [2023-05-11 00:16:26,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +13: [2023-05-11 00:16:26,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +13: [2023-05-11 00:16:26,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +13: [2023-05-11 00:16:26,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +25: [2023-05-11 00:16:26,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 4: [2023-05-11 00:16:26,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +19: [2023-05-11 00:16:26,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +19: [2023-05-11 00:16:26,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +19: [2023-05-11 00:16:26,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +19: [2023-05-11 00:16:26,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +20: [2023-05-11 00:16:26,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 8: [2023-05-11 00:16:26,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 4: [2023-05-11 00:16:26,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +20: [2023-05-11 00:16:26,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +19: [2023-05-11 00:16:26,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +19: [2023-05-11 00:16:26,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +11: [2023-05-11 00:16:26,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +11: [2023-05-11 00:16:26,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +11: [2023-05-11 00:16:26,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 4: [2023-05-11 00:16:26,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +11: [2023-05-11 00:16:26,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +11: [2023-05-11 00:16:26,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +11: [2023-05-11 00:16:26,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +11: [2023-05-11 00:16:26,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 4: [2023-05-11 00:16:26,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +19: [2023-05-11 00:16:26,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +19: [2023-05-11 00:16:26,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +11: [2023-05-11 00:16:26,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 8: [2023-05-11 00:16:26,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +20: [2023-05-11 00:16:26,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 0: [2023-05-11 00:16:26,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +20: [2023-05-11 00:16:26,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +19: [2023-05-11 00:16:26,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +19: [2023-05-11 00:16:26,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +19: [2023-05-11 00:16:26,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +20: [2023-05-11 00:16:26,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +19: [2023-05-11 00:16:26,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +20: [2023-05-11 00:16:26,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +13: [2023-05-11 00:16:26,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 8: [2023-05-11 00:16:26,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +24: [2023-05-11 00:16:26,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +24: [2023-05-11 00:16:26,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +24: [2023-05-11 00:16:26,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +24: [2023-05-11 00:16:26,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +19: [2023-05-11 00:16:26,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +13: [2023-05-11 00:16:26,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +13: [2023-05-11 00:16:26,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +19: [2023-05-11 00:16:26,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +24: [2023-05-11 00:16:26,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +24: [2023-05-11 00:16:26,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +24: [2023-05-11 00:16:26,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +24: [2023-05-11 00:16:26,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +13: [2023-05-11 00:16:26,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +13: [2023-05-11 00:16:26,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +20: [2023-05-11 00:16:26,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +20: [2023-05-11 00:16:26,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 1: [2023-05-11 00:16:26,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 1: [2023-05-11 00:16:26,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +19: [2023-05-11 00:16:26,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +20: [2023-05-11 00:16:26,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +20: [2023-05-11 00:16:26,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +20: [2023-05-11 00:16:26,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +20: [2023-05-11 00:16:26,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +19: [2023-05-11 00:16:26,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +13: [2023-05-11 00:16:26,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +13: [2023-05-11 00:16:26,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 8: [2023-05-11 00:16:26,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 8: [2023-05-11 00:16:26,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 8: [2023-05-11 00:16:26,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 1: [2023-05-11 00:16:26,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +19: [2023-05-11 00:16:26,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 1: [2023-05-11 00:16:26,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +20: [2023-05-11 00:16:26,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +20: [2023-05-11 00:16:26,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +13: [2023-05-11 00:16:26,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +13: [2023-05-11 00:16:26,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +24: [2023-05-11 00:16:26,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +24: [2023-05-11 00:16:26,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +19: [2023-05-11 00:16:26,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +13: [2023-05-11 00:16:26,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 8: [2023-05-11 00:16:26,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 8: [2023-05-11 00:16:26,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +13: [2023-05-11 00:16:26,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +19: [2023-05-11 00:16:26,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +19: [2023-05-11 00:16:26,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +19: [2023-05-11 00:16:26,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +13: [2023-05-11 00:16:26,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +13: [2023-05-11 00:16:26,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +22: [2023-05-11 00:16:26,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +22: [2023-05-11 00:16:26,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +22: [2023-05-11 00:16:26,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +22: [2023-05-11 00:16:26,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +19: [2023-05-11 00:16:26,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 7: [2023-05-11 00:16:26,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 7: [2023-05-11 00:16:26,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 7: [2023-05-11 00:16:26,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +19: [2023-05-11 00:16:26,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +19: [2023-05-11 00:16:26,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 1: [2023-05-11 00:16:26,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +24: [2023-05-11 00:16:26,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +13: [2023-05-11 00:16:26,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 1: [2023-05-11 00:16:26,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +24: [2023-05-11 00:16:26,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +19: [2023-05-11 00:16:26,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +13: [2023-05-11 00:16:27,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +19: [2023-05-11 00:16:27,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +19: [2023-05-11 00:16:27,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +13: [2023-05-11 00:16:27,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +22: [2023-05-11 00:16:27,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +22: [2023-05-11 00:16:27,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +19: [2023-05-11 00:16:27,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +22: [2023-05-11 00:16:27,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +22: [2023-05-11 00:16:27,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 7: [2023-05-11 00:16:27,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +13: [2023-05-11 00:16:27,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +13: [2023-05-11 00:16:27,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 7: [2023-05-11 00:16:27,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +29: [2023-05-11 00:16:27,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +24: [2023-05-11 00:16:27,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +29: [2023-05-11 00:16:27,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +29: [2023-05-11 00:16:27,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +29: [2023-05-11 00:16:27,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +26: [2023-05-11 00:16:27,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +29: [2023-05-11 00:16:27,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +29: [2023-05-11 00:16:27,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +29: [2023-05-11 00:16:27,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +26: [2023-05-11 00:16:27,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +17: [2023-05-11 00:16:27,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +17: [2023-05-11 00:16:27,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +17: [2023-05-11 00:16:27,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +17: [2023-05-11 00:16:27,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +17: [2023-05-11 00:16:27,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +26: [2023-05-11 00:16:27,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +29: [2023-05-11 00:16:27,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +17: [2023-05-11 00:16:27,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +17: [2023-05-11 00:16:27,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +17: [2023-05-11 00:16:27,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +16: [2023-05-11 00:16:27,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +16: [2023-05-11 00:16:27,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +16: [2023-05-11 00:16:27,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +16: [2023-05-11 00:16:27,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +26: [2023-05-11 00:16:27,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +15: [2023-05-11 00:16:27,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +15: [2023-05-11 00:16:27,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +15: [2023-05-11 00:16:27,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +15: [2023-05-11 00:16:27,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +16: [2023-05-11 00:16:27,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +16: [2023-05-11 00:16:27,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +17: [2023-05-11 00:16:27,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +17: [2023-05-11 00:16:27,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 1: [2023-05-11 00:16:27,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +15: [2023-05-11 00:16:27,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +15: [2023-05-11 00:16:27,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +15: [2023-05-11 00:16:27,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +15: [2023-05-11 00:16:27,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +16: [2023-05-11 00:16:27,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +16: [2023-05-11 00:16:27,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +17: [2023-05-11 00:16:27,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +17: [2023-05-11 00:16:27,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 1: [2023-05-11 00:16:27,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 1: [2023-05-11 00:16:27,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +13: [2023-05-11 00:16:27,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +13: [2023-05-11 00:16:27,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +13: [2023-05-11 00:16:27,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +13: [2023-05-11 00:16:27,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 7: [2023-05-11 00:16:27,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 1: [2023-05-11 00:16:27,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 1: [2023-05-11 00:16:27,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +24: [2023-05-11 00:16:27,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +24: [2023-05-11 00:16:27,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +22: [2023-05-11 00:16:27,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +22: [2023-05-11 00:16:27,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +26: [2023-05-11 00:16:27,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +26: [2023-05-11 00:16:27,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +22: [2023-05-11 00:16:27,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +22: [2023-05-11 00:16:27,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +22: [2023-05-11 00:16:27,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +22: [2023-05-11 00:16:27,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +29: [2023-05-11 00:16:27,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +22: [2023-05-11 00:16:27,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +22: [2023-05-11 00:16:27,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +17: [2023-05-11 00:16:27,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +26: [2023-05-11 00:16:27,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +26: [2023-05-11 00:16:27,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +15: [2023-05-11 00:16:27,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +13: [2023-05-11 00:16:27,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +15: [2023-05-11 00:16:27,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 7: [2023-05-11 00:16:27,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 2: [2023-05-11 00:16:27,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 2: [2023-05-11 00:16:27,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 2: [2023-05-11 00:16:27,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +15: [2023-05-11 00:16:27,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. +13: [2023-05-11 00:16:27,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +16: [2023-05-11 00:16:27,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +16: [2023-05-11 00:16:27,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +17: [2023-05-11 00:16:27,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +17: [2023-05-11 00:16:27,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +17: [2023-05-11 00:16:27,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +24: [2023-05-11 00:16:27,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +15: [2023-05-11 00:16:27,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +29: [2023-05-11 00:16:27,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +29: [2023-05-11 00:16:27,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +29: [2023-05-11 00:16:27,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +26: [2023-05-11 00:16:27,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +15: [2023-05-11 00:16:27,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +17: [2023-05-11 00:16:27,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +26: [2023-05-11 00:16:27,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +26: [2023-05-11 00:16:27,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +26: [2023-05-11 00:16:27,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +17: [2023-05-11 00:16:27,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +17: [2023-05-11 00:16:27,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 0: [2023-05-11 00:16:27,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 0: [2023-05-11 00:16:27,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 0: [2023-05-11 00:16:27,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 5: [2023-05-11 00:16:27,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +29: [2023-05-11 00:16:27,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +16: [2023-05-11 00:16:27,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +15: [2023-05-11 00:16:27,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 5: [2023-05-11 00:16:27,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 5: [2023-05-11 00:16:27,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +26: [2023-05-11 00:16:27,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +26: [2023-05-11 00:16:27,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +26: [2023-05-11 00:16:27,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +26: [2023-05-11 00:16:27,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +15: [2023-05-11 00:16:27,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +16: [2023-05-11 00:16:27,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +15: [2023-05-11 00:16:27,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 5: [2023-05-11 00:16:27,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt. + 5: [2023-05-11 00:16:27,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +29: [2023-05-11 00:16:27,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +17: [2023-05-11 00:16:27,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +29: [2023-05-11 00:16:27,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +17: [2023-05-11 00:16:27,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +17: [2023-05-11 00:16:27,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +16: [2023-05-11 00:16:27,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +17: [2023-05-11 00:16:27,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 0: [2023-05-11 00:16:27,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +17: [2023-05-11 00:16:27,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +29: [2023-05-11 00:16:27,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +17: [2023-05-11 00:16:27,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +17: [2023-05-11 00:16:27,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +17: [2023-05-11 00:16:27,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +28: [2023-05-11 00:16:27,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +28: [2023-05-11 00:16:27,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +28: [2023-05-11 00:16:27,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +28: [2023-05-11 00:16:27,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt... +18: [2023-05-11 00:16:27,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +18: [2023-05-11 00:16:27,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +16: [2023-05-11 00:16:27,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +18: [2023-05-11 00:16:27,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +10: [2023-05-11 00:16:27,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +10: [2023-05-11 00:16:27,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +10: [2023-05-11 00:16:27,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +10: [2023-05-11 00:16:27,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +10: [2023-05-11 00:16:27,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +10: [2023-05-11 00:16:27,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +10: [2023-05-11 00:16:27,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +28: [2023-05-11 00:16:27,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +28: [2023-05-11 00:16:27,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +28: [2023-05-11 00:16:27,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +18: [2023-05-11 00:16:27,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +18: [2023-05-11 00:16:27,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +18: [2023-05-11 00:16:27,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +16: [2023-05-11 00:16:27,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +17: [2023-05-11 00:16:27,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +17: [2023-05-11 00:16:27,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +12: [2023-05-11 00:16:27,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +12: [2023-05-11 00:16:27,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +12: [2023-05-11 00:16:27,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +17: [2023-05-11 00:16:27,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +17: [2023-05-11 00:16:27,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +28: [2023-05-11 00:16:27,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +17: [2023-05-11 00:16:27,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 5: [2023-05-11 00:16:27,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +12: [2023-05-11 00:16:27,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +18: [2023-05-11 00:16:27,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +18: [2023-05-11 00:16:27,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +12: [2023-05-11 00:16:27,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +10: [2023-05-11 00:16:27,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +12: [2023-05-11 00:16:27,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +16: [2023-05-11 00:16:27,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +12: [2023-05-11 00:16:27,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +12: [2023-05-11 00:16:27,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +18: [2023-05-11 00:16:27,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 2: [2023-05-11 00:16:27,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 2: [2023-05-11 00:16:27,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 2: [2023-05-11 00:16:27,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 2: [2023-05-11 00:16:27,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 0: [2023-05-11 00:16:27,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +10: [2023-05-11 00:16:27,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... + 0: [2023-05-11 00:16:27,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +28: [2023-05-11 00:16:27,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +28: [2023-05-11 00:16:27,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. +28: [2023-05-11 00:16:27,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +10: [2023-05-11 00:16:27,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +10: [2023-05-11 00:16:27,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +12: [2023-05-11 00:16:27,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +18: [2023-05-11 00:16:27,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +28: [2023-05-11 00:16:27,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +28: [2023-05-11 00:16:27,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +28: [2023-05-11 00:16:27,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt... +10: [2023-05-11 00:16:27,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +10: [2023-05-11 00:16:27,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +10: [2023-05-11 00:16:27,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 9: [2023-05-11 00:16:27,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 9: [2023-05-11 00:16:27,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 9: [2023-05-11 00:16:27,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 5: [2023-05-11 00:16:27,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +18: [2023-05-11 00:16:27,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +10: [2023-05-11 00:16:27,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +10: [2023-05-11 00:16:27,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +10: [2023-05-11 00:16:27,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 5: [2023-05-11 00:16:27,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +18: [2023-05-11 00:16:27,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +30: [2023-05-11 00:16:27,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +30: [2023-05-11 00:16:27,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +15: [2023-05-11 00:16:27,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +15: [2023-05-11 00:16:27,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +15: [2023-05-11 00:16:27,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +15: [2023-05-11 00:16:27,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +28: [2023-05-11 00:16:27,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +10: [2023-05-11 00:16:27,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +12: [2023-05-11 00:16:27,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +21: [2023-05-11 00:16:27,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +21: [2023-05-11 00:16:27,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +21: [2023-05-11 00:16:27,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +30: [2023-05-11 00:16:27,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +21: [2023-05-11 00:16:27,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +21: [2023-05-11 00:16:27,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +21: [2023-05-11 00:16:27,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +21: [2023-05-11 00:16:27,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +21: [2023-05-11 00:16:27,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +18: [2023-05-11 00:16:27,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +30: [2023-05-11 00:16:27,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +30: [2023-05-11 00:16:27,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +30: [2023-05-11 00:16:27,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +10: [2023-05-11 00:16:27,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +12: [2023-05-11 00:16:27,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +30: [2023-05-11 00:16:27,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +30: [2023-05-11 00:16:27,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +12: [2023-05-11 00:16:27,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +18: [2023-05-11 00:16:27,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +18: [2023-05-11 00:16:27,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +15: [2023-05-11 00:16:27,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +28: [2023-05-11 00:16:27,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +15: [2023-05-11 00:16:27,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +15: [2023-05-11 00:16:27,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +10: [2023-05-11 00:16:27,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +15: [2023-05-11 00:16:27,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +30: [2023-05-11 00:16:27,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +18: [2023-05-11 00:16:27,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +10: [2023-05-11 00:16:27,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +21: [2023-05-11 00:16:27,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +10: [2023-05-11 00:16:27,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +10: [2023-05-11 00:16:27,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +12: [2023-05-11 00:16:27,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +12: [2023-05-11 00:16:27,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +16: [2023-05-11 00:16:27,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +16: [2023-05-11 00:16:27,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +16: [2023-05-11 00:16:27,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +16: [2023-05-11 00:16:27,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +30: [2023-05-11 00:16:27,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +10: [2023-05-11 00:16:27,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +21: [2023-05-11 00:16:27,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +21: [2023-05-11 00:16:27,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +21: [2023-05-11 00:16:27,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +30: [2023-05-11 00:16:27,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +21: [2023-05-11 00:16:27,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +15: [2023-05-11 00:16:27,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +12: [2023-05-11 00:16:27,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +15: [2023-05-11 00:16:27,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +16: [2023-05-11 00:16:27,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +16: [2023-05-11 00:16:27,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +12: [2023-05-11 00:16:27,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +30: [2023-05-11 00:16:27,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +15: [2023-05-11 00:16:27,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +15: [2023-05-11 00:16:27,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +15: [2023-05-11 00:16:27,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +15: [2023-05-11 00:16:27,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +16: [2023-05-11 00:16:27,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +16: [2023-05-11 00:16:27,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +10: [2023-05-11 00:16:27,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +10: [2023-05-11 00:16:27,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +10: [2023-05-11 00:16:27,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +10: [2023-05-11 00:16:27,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +14: [2023-05-11 00:16:27,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +14: [2023-05-11 00:16:27,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +14: [2023-05-11 00:16:27,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +14: [2023-05-11 00:16:27,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +14: [2023-05-11 00:16:27,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +14: [2023-05-11 00:16:27,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +14: [2023-05-11 00:16:27,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +21: [2023-05-11 00:16:27,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +30: [2023-05-11 00:16:27,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +14: [2023-05-11 00:16:27,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +21: [2023-05-11 00:16:27,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +21: [2023-05-11 00:16:27,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +30: [2023-05-11 00:16:27,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +15: [2023-05-11 00:16:27,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +15: [2023-05-11 00:16:27,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +16: [2023-05-11 00:16:27,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +16: [2023-05-11 00:16:27,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +16: [2023-05-11 00:16:27,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +16: [2023-05-11 00:16:27,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +30: [2023-05-11 00:16:27,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +14: [2023-05-11 00:16:27,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +30: [2023-05-11 00:16:27,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +31: [2023-05-11 00:16:27,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +31: [2023-05-11 00:16:27,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +31: [2023-05-11 00:16:27,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +31: [2023-05-11 00:16:27,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +31: [2023-05-11 00:16:27,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +14: [2023-05-11 00:16:27,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +14: [2023-05-11 00:16:27,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +10: [2023-05-11 00:16:27,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +10: [2023-05-11 00:16:27,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +31: [2023-05-11 00:16:27,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +31: [2023-05-11 00:16:27,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +31: [2023-05-11 00:16:27,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +14: [2023-05-11 00:16:27,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +10: [2023-05-11 00:16:27,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +10: [2023-05-11 00:16:27,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +16: [2023-05-11 00:16:27,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +16: [2023-05-11 00:16:27,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +31: [2023-05-11 00:16:27,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +14: [2023-05-11 00:16:27,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +14: [2023-05-11 00:16:27,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +16: [2023-05-11 00:16:27,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +16: [2023-05-11 00:16:27,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 3: [2023-05-11 00:16:27,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +31: [2023-05-11 00:16:27,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 1: [2023-05-11 00:16:27,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 1: [2023-05-11 00:16:27,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 1: [2023-05-11 00:16:27,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +14: [2023-05-11 00:16:27,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 1: [2023-05-11 00:16:27,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +31: [2023-05-11 00:16:27,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +31: [2023-05-11 00:16:27,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +14: [2023-05-11 00:16:27,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 1: [2023-05-11 00:16:27,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 1: [2023-05-11 00:16:27,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +31: [2023-05-11 00:16:27,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +31: [2023-05-11 00:16:27,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 7: [2023-05-11 00:16:27,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +27: [2023-05-11 00:16:27,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +27: [2023-05-11 00:16:27,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +27: [2023-05-11 00:16:27,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +27: [2023-05-11 00:16:27,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +27: [2023-05-11 00:16:27,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +27: [2023-05-11 00:16:27,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +31: [2023-05-11 00:16:27,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +31: [2023-05-11 00:16:27,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +31: [2023-05-11 00:16:27,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 3: [2023-05-11 00:16:27,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +29: [2023-05-11 00:16:27,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +29: [2023-05-11 00:16:27,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +29: [2023-05-11 00:16:27,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +29: [2023-05-11 00:16:27,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +27: [2023-05-11 00:16:27,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +27: [2023-05-11 00:16:27,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +31: [2023-05-11 00:16:27,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +31: [2023-05-11 00:16:27,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 9: [2023-05-11 00:16:27,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 9: [2023-05-11 00:16:27,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 9: [2023-05-11 00:16:27,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 1: [2023-05-11 00:16:27,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +31: [2023-05-11 00:16:27,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +31: [2023-05-11 00:16:27,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 6: [2023-05-11 00:16:27,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 9: [2023-05-11 00:16:27,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 9: [2023-05-11 00:16:27,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +31: [2023-05-11 00:16:27,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +31: [2023-05-11 00:16:27,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +31: [2023-05-11 00:16:27,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +12: [2023-05-11 00:16:27,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +12: [2023-05-11 00:16:27,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +12: [2023-05-11 00:16:27,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +12: [2023-05-11 00:16:27,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +29: [2023-05-11 00:16:27,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +27: [2023-05-11 00:16:27,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +27: [2023-05-11 00:16:27,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +29: [2023-05-11 00:16:27,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +29: [2023-05-11 00:16:27,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +22: [2023-05-11 00:16:27,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +22: [2023-05-11 00:16:27,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +29: [2023-05-11 00:16:27,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +22: [2023-05-11 00:16:27,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +22: [2023-05-11 00:16:27,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +22: [2023-05-11 00:16:27,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +22: [2023-05-11 00:16:27,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +22: [2023-05-11 00:16:27,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +31: [2023-05-11 00:16:27,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +31: [2023-05-11 00:16:27,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 9: [2023-05-11 00:16:27,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 6: [2023-05-11 00:16:27,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 6: [2023-05-11 00:16:27,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +22: [2023-05-11 00:16:27,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 6: [2023-05-11 00:16:27,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +12: [2023-05-11 00:16:27,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +27: [2023-05-11 00:16:27,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +27: [2023-05-11 00:16:27,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +29: [2023-05-11 00:16:27,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +12: [2023-05-11 00:16:27,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +29: [2023-05-11 00:16:27,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 1: [2023-05-11 00:16:27,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +29: [2023-05-11 00:16:27,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +29: [2023-05-11 00:16:27,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +27: [2023-05-11 00:16:27,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +12: [2023-05-11 00:16:27,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +12: [2023-05-11 00:16:27,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +22: [2023-05-11 00:16:27,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +31: [2023-05-11 00:16:27,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +31: [2023-05-11 00:16:27,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +23: [2023-05-11 00:16:27,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +31: [2023-05-11 00:16:27,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +31: [2023-05-11 00:16:27,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +31: [2023-05-11 00:16:27,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +31: [2023-05-11 00:16:27,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +22: [2023-05-11 00:16:27,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +23: [2023-05-11 00:16:27,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +23: [2023-05-11 00:16:27,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +23: [2023-05-11 00:16:27,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +23: [2023-05-11 00:16:27,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +23: [2023-05-11 00:16:27,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 4: [2023-05-11 00:16:27,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 4: [2023-05-11 00:16:27,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +23: [2023-05-11 00:16:27,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +23: [2023-05-11 00:16:27,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +27: [2023-05-11 00:16:27,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +27: [2023-05-11 00:16:27,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +22: [2023-05-11 00:16:27,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 6: [2023-05-11 00:16:27,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 6: [2023-05-11 00:16:27,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 6: [2023-05-11 00:16:27,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 4: [2023-05-11 00:16:27,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +12: [2023-05-11 00:16:27,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +12: [2023-05-11 00:16:27,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 6: [2023-05-11 00:16:27,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +12: [2023-05-11 00:16:27,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +12: [2023-05-11 00:16:27,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +22: [2023-05-11 00:16:27,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +27: [2023-05-11 00:16:27,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +22: [2023-05-11 00:16:27,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +29: [2023-05-11 00:16:27,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +23: [2023-05-11 00:16:27,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +23: [2023-05-11 00:16:27,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +29: [2023-05-11 00:16:27,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +29: [2023-05-11 00:16:27,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 9: [2023-05-11 00:16:27,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +22: [2023-05-11 00:16:27,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +23: [2023-05-11 00:16:27,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +23: [2023-05-11 00:16:27,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +22: [2023-05-11 00:16:27,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 6: [2023-05-11 00:16:27,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 6: [2023-05-11 00:16:27,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +29: [2023-05-11 00:16:27,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 6: [2023-05-11 00:16:27,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 6: [2023-05-11 00:16:27,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +28: [2023-05-11 00:16:27,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +28: [2023-05-11 00:16:27,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 9: [2023-05-11 00:16:27,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +28: [2023-05-11 00:16:27,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +28: [2023-05-11 00:16:27,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 4: [2023-05-11 00:16:27,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +23: [2023-05-11 00:16:27,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +23: [2023-05-11 00:16:27,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +23: [2023-05-11 00:16:27,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +23: [2023-05-11 00:16:27,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +12: [2023-05-11 00:16:27,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +12: [2023-05-11 00:16:27,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +22: [2023-05-11 00:16:27,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 4: [2023-05-11 00:16:27,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +12: [2023-05-11 00:16:27,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +12: [2023-05-11 00:16:27,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 4: [2023-05-11 00:16:27,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +28: [2023-05-11 00:16:27,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +28: [2023-05-11 00:16:27,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +28: [2023-05-11 00:16:27,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +28: [2023-05-11 00:16:27,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 4: [2023-05-11 00:16:27,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +26: [2023-05-11 00:16:27,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +26: [2023-05-11 00:16:27,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +26: [2023-05-11 00:16:27,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +26: [2023-05-11 00:16:27,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +26: [2023-05-11 00:16:27,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +26: [2023-05-11 00:16:27,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 6: [2023-05-11 00:16:27,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +26: [2023-05-11 00:16:27,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 8: [2023-05-11 00:16:27,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 8: [2023-05-11 00:16:27,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +26: [2023-05-11 00:16:27,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +13: [2023-05-11 00:16:27,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +13: [2023-05-11 00:16:27,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +11: [2023-05-11 00:16:27,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +11: [2023-05-11 00:16:27,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +11: [2023-05-11 00:16:27,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +11: [2023-05-11 00:16:27,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +13: [2023-05-11 00:16:27,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +25: [2023-05-11 00:16:27,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +25: [2023-05-11 00:16:27,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +25: [2023-05-11 00:16:27,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +25: [2023-05-11 00:16:27,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +25: [2023-05-11 00:16:27,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +25: [2023-05-11 00:16:27,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +19: [2023-05-11 00:16:27,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +19: [2023-05-11 00:16:27,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +19: [2023-05-11 00:16:27,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +19: [2023-05-11 00:16:27,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +19: [2023-05-11 00:16:27,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +19: [2023-05-11 00:16:27,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +20: [2023-05-11 00:16:27,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +20: [2023-05-11 00:16:27,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +13: [2023-05-11 00:16:27,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +20: [2023-05-11 00:16:27,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +20: [2023-05-11 00:16:27,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +20: [2023-05-11 00:16:27,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +20: [2023-05-11 00:16:27,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +19: [2023-05-11 00:16:27,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +13: [2023-05-11 00:16:27,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +20: [2023-05-11 00:16:27,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +13: [2023-05-11 00:16:27,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +13: [2023-05-11 00:16:27,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +25: [2023-05-11 00:16:27,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +11: [2023-05-11 00:16:27,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +11: [2023-05-11 00:16:27,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +19: [2023-05-11 00:16:27,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +20: [2023-05-11 00:16:27,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +13: [2023-05-11 00:16:27,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +25: [2023-05-11 00:16:27,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +11: [2023-05-11 00:16:27,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +11: [2023-05-11 00:16:27,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +28: [2023-05-11 00:16:27,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +28: [2023-05-11 00:16:27,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 2: [2023-05-11 00:16:27,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +11: [2023-05-11 00:16:27,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +19: [2023-05-11 00:16:27,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +25: [2023-05-11 00:16:27,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +13: [2023-05-11 00:16:27,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +11: [2023-05-11 00:16:27,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 8: [2023-05-11 00:16:27,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +25: [2023-05-11 00:16:27,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +20: [2023-05-11 00:16:27,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +25: [2023-05-11 00:16:27,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +20: [2023-05-11 00:16:27,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +19: [2023-05-11 00:16:27,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +19: [2023-05-11 00:16:27,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +28: [2023-05-11 00:16:27,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +28: [2023-05-11 00:16:27,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +28: [2023-05-11 00:16:27,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +28: [2023-05-11 00:16:27,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +13: [2023-05-11 00:16:27,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +26: [2023-05-11 00:16:27,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +26: [2023-05-11 00:16:27,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +26: [2023-05-11 00:16:27,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +25: [2023-05-11 00:16:27,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +26: [2023-05-11 00:16:27,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +28: [2023-05-11 00:16:27,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 2: [2023-05-11 00:16:27,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +20: [2023-05-11 00:16:27,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +11: [2023-05-11 00:16:27,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +28: [2023-05-11 00:16:27,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +25: [2023-05-11 00:16:27,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +19: [2023-05-11 00:16:27,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +20: [2023-05-11 00:16:27,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +11: [2023-05-11 00:16:27,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +25: [2023-05-11 00:16:27,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +20: [2023-05-11 00:16:27,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +19: [2023-05-11 00:16:27,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +19: [2023-05-11 00:16:27,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +20: [2023-05-11 00:16:27,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +26: [2023-05-11 00:16:27,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +26: [2023-05-11 00:16:27,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +26: [2023-05-11 00:16:27,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +26: [2023-05-11 00:16:27,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +25: [2023-05-11 00:16:27,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +13: [2023-05-11 00:16:27,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +11: [2023-05-11 00:16:27,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +19: [2023-05-11 00:16:27,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +13: [2023-05-11 00:16:27,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +13: [2023-05-11 00:16:27,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +20: [2023-05-11 00:16:27,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 8: [2023-05-11 00:16:27,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 8: [2023-05-11 00:16:27,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +11: [2023-05-11 00:16:27,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +23: [2023-05-11 00:16:27,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +23: [2023-05-11 00:16:27,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +23: [2023-05-11 00:16:27,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +23: [2023-05-11 00:16:27,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 8: [2023-05-11 00:16:27,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +11: [2023-05-11 00:16:27,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +25: [2023-05-11 00:16:27,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +17: [2023-05-11 00:16:27,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +17: [2023-05-11 00:16:27,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +19: [2023-05-11 00:16:27,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +17: [2023-05-11 00:16:27,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +13: [2023-05-11 00:16:27,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +13: [2023-05-11 00:16:27,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +17: [2023-05-11 00:16:27,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +20: [2023-05-11 00:16:27,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +13: [2023-05-11 00:16:27,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +11: [2023-05-11 00:16:27,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +17: [2023-05-11 00:16:27,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +17: [2023-05-11 00:16:27,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +23: [2023-05-11 00:16:27,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +17: [2023-05-11 00:16:27,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +17: [2023-05-11 00:16:27,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +11: [2023-05-11 00:16:27,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +11: [2023-05-11 00:16:27,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +11: [2023-05-11 00:16:27,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +23: [2023-05-11 00:16:27,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +23: [2023-05-11 00:16:27,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +11: [2023-05-11 00:16:27,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +23: [2023-05-11 00:16:27,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +17: [2023-05-11 00:16:27,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 7: [2023-05-11 00:16:27,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +17: [2023-05-11 00:16:27,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 7: [2023-05-11 00:16:27,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 0: [2023-05-11 00:16:27,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +27: [2023-05-11 00:16:27,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +27: [2023-05-11 00:16:27,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +27: [2023-05-11 00:16:27,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 0: [2023-05-11 00:16:27,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +27: [2023-05-11 00:16:27,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +11: [2023-05-11 00:16:27,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +24: [2023-05-11 00:16:27,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +24: [2023-05-11 00:16:27,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +11: [2023-05-11 00:16:27,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +24: [2023-05-11 00:16:27,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +11: [2023-05-11 00:16:27,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +24: [2023-05-11 00:16:27,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +11: [2023-05-11 00:16:27,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +30: [2023-05-11 00:16:27,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +30: [2023-05-11 00:16:27,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +30: [2023-05-11 00:16:27,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +30: [2023-05-11 00:16:27,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +17: [2023-05-11 00:16:27,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +23: [2023-05-11 00:16:27,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +17: [2023-05-11 00:16:27,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +23: [2023-05-11 00:16:27,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +21: [2023-05-11 00:16:27,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +21: [2023-05-11 00:16:27,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +21: [2023-05-11 00:16:27,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +17: [2023-05-11 00:16:27,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +21: [2023-05-11 00:16:27,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +19: [2023-05-11 00:16:27,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +19: [2023-05-11 00:16:27,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +19: [2023-05-11 00:16:27,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +19: [2023-05-11 00:16:27,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +17: [2023-05-11 00:16:27,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 3: [2023-05-11 00:16:27,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 3: [2023-05-11 00:16:27,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 3: [2023-05-11 00:16:27,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +18: [2023-05-11 00:16:27,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +18: [2023-05-11 00:16:27,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +18: [2023-05-11 00:16:27,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 3: [2023-05-11 00:16:27,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +18: [2023-05-11 00:16:27,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 7: [2023-05-11 00:16:27,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +17: [2023-05-11 00:16:27,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +17: [2023-05-11 00:16:27,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +27: [2023-05-11 00:16:27,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +27: [2023-05-11 00:16:27,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +27: [2023-05-11 00:16:27,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +24: [2023-05-11 00:16:27,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +27: [2023-05-11 00:16:27,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +24: [2023-05-11 00:16:27,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +21: [2023-05-11 00:16:27,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +21: [2023-05-11 00:16:27,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +11: [2023-05-11 00:16:27,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +17: [2023-05-11 00:16:27,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +24: [2023-05-11 00:16:27,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +21: [2023-05-11 00:16:27,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +11: [2023-05-11 00:16:27,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +19: [2023-05-11 00:16:27,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +24: [2023-05-11 00:16:27,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +30: [2023-05-11 00:16:27,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +18: [2023-05-11 00:16:27,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +30: [2023-05-11 00:16:27,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +30: [2023-05-11 00:16:27,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +30: [2023-05-11 00:16:27,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +17: [2023-05-11 00:16:27,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +19: [2023-05-11 00:16:27,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +23: [2023-05-11 00:16:27,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +26: [2023-05-11 00:16:27,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +26: [2023-05-11 00:16:27,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 3: [2023-05-11 00:16:27,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 8: [2023-05-11 00:16:27,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 8: [2023-05-11 00:16:27,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +21: [2023-05-11 00:16:27,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +17: [2023-05-11 00:16:27,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +17: [2023-05-11 00:16:27,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +11: [2023-05-11 00:16:27,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +11: [2023-05-11 00:16:27,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +11: [2023-05-11 00:16:27,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +11: [2023-05-11 00:16:27,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +19: [2023-05-11 00:16:27,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +26: [2023-05-11 00:16:27,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 0: [2023-05-11 00:16:27,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +26: [2023-05-11 00:16:27,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +18: [2023-05-11 00:16:27,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 4: [2023-05-11 00:16:27,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 4: [2023-05-11 00:16:27,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +23: [2023-05-11 00:16:27,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +19: [2023-05-11 00:16:27,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 4: [2023-05-11 00:16:27,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 4: [2023-05-11 00:16:27,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 7: [2023-05-11 00:16:27,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +23: [2023-05-11 00:16:27,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +18: [2023-05-11 00:16:27,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +18: [2023-05-11 00:16:27,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +25: [2023-05-11 00:16:27,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +23: [2023-05-11 00:16:27,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +25: [2023-05-11 00:16:27,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +25: [2023-05-11 00:16:27,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +25: [2023-05-11 00:16:27,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 0: [2023-05-11 00:16:27,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +27: [2023-05-11 00:16:27,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +27: [2023-05-11 00:16:27,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +24: [2023-05-11 00:16:27,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +26: [2023-05-11 00:16:27,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +23: [2023-05-11 00:16:27,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +24: [2023-05-11 00:16:27,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +21: [2023-05-11 00:16:27,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +24: [2023-05-11 00:16:27,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +24: [2023-05-11 00:16:27,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +21: [2023-05-11 00:16:27,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 8: [2023-05-11 00:16:27,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 5: [2023-05-11 00:16:27,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 5: [2023-05-11 00:16:27,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 8: [2023-05-11 00:16:27,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +23: [2023-05-11 00:16:27,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +24: [2023-05-11 00:16:27,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +24: [2023-05-11 00:16:27,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +26: [2023-05-11 00:16:27,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +17: [2023-05-11 00:16:27,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +17: [2023-05-11 00:16:27,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +26: [2023-05-11 00:16:27,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +17: [2023-05-11 00:16:27,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +21: [2023-05-11 00:16:27,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +21: [2023-05-11 00:16:27,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +21: [2023-05-11 00:16:27,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +21: [2023-05-11 00:16:27,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +17: [2023-05-11 00:16:27,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +11: [2023-05-11 00:16:27,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 8: [2023-05-11 00:16:27,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +18: [2023-05-11 00:16:27,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +18: [2023-05-11 00:16:27,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +24: [2023-05-11 00:16:27,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +24: [2023-05-11 00:16:27,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +26: [2023-05-11 00:16:27,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +14: [2023-05-11 00:16:27,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +14: [2023-05-11 00:16:27,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +14: [2023-05-11 00:16:27,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +14: [2023-05-11 00:16:27,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 4: [2023-05-11 00:16:27,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +27: [2023-05-11 00:16:27,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +27: [2023-05-11 00:16:27,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +27: [2023-05-11 00:16:27,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +27: [2023-05-11 00:16:27,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +11: [2023-05-11 00:16:27,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 4: [2023-05-11 00:16:27,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +25: [2023-05-11 00:16:27,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +25: [2023-05-11 00:16:27,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +19: [2023-05-11 00:16:27,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 3: [2023-05-11 00:16:27,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +19: [2023-05-11 00:16:27,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +17: [2023-05-11 00:16:27,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +17: [2023-05-11 00:16:27,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +25: [2023-05-11 00:16:27,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +19: [2023-05-11 00:16:27,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +19: [2023-05-11 00:16:27,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 4: [2023-05-11 00:16:27,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 1: [2023-05-11 00:16:27,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 1: [2023-05-11 00:16:27,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 0: [2023-05-11 00:16:27,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 0: [2023-05-11 00:16:27,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 0: [2023-05-11 00:16:27,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 0: [2023-05-11 00:16:27,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 1: [2023-05-11 00:16:27,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +17: [2023-05-11 00:16:27,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +17: [2023-05-11 00:16:27,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 5: [2023-05-11 00:16:27,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +30: [2023-05-11 00:16:27,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +25: [2023-05-11 00:16:27,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +17: [2023-05-11 00:16:27,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +17: [2023-05-11 00:16:27,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 5: [2023-05-11 00:16:27,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +30: [2023-05-11 00:16:27,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +17: [2023-05-11 00:16:27,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +15: [2023-05-11 00:16:27,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +15: [2023-05-11 00:16:27,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +15: [2023-05-11 00:16:27,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +17: [2023-05-11 00:16:27,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +21: [2023-05-11 00:16:27,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +14: [2023-05-11 00:16:27,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 2: [2023-05-11 00:16:27,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 2: [2023-05-11 00:16:27,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 2: [2023-05-11 00:16:27,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +15: [2023-05-11 00:16:27,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +21: [2023-05-11 00:16:27,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +30: [2023-05-11 00:16:27,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +30: [2023-05-11 00:16:27,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +30: [2023-05-11 00:16:27,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +30: [2023-05-11 00:16:27,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +30: [2023-05-11 00:16:27,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +30: [2023-05-11 00:16:27,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +26: [2023-05-11 00:16:27,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +26: [2023-05-11 00:16:27,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +26: [2023-05-11 00:16:27,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 8: [2023-05-11 00:16:27,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +26: [2023-05-11 00:16:27,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +26: [2023-05-11 00:16:27,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 8: [2023-05-11 00:16:27,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +26: [2023-05-11 00:16:27,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 8: [2023-05-11 00:16:27,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +14: [2023-05-11 00:16:27,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 7: [2023-05-11 00:16:27,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 7: [2023-05-11 00:16:27,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +14: [2023-05-11 00:16:27,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +19: [2023-05-11 00:16:27,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +14: [2023-05-11 00:16:27,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +25: [2023-05-11 00:16:27,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +25: [2023-05-11 00:16:27,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +19: [2023-05-11 00:16:27,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +19: [2023-05-11 00:16:27,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 1: [2023-05-11 00:16:27,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +15: [2023-05-11 00:16:27,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +25: [2023-05-11 00:16:27,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +25: [2023-05-11 00:16:27,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +27: [2023-05-11 00:16:27,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +18: [2023-05-11 00:16:27,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +25: [2023-05-11 00:16:27,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +25: [2023-05-11 00:16:27,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 1: [2023-05-11 00:16:27,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +15: [2023-05-11 00:16:27,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +15: [2023-05-11 00:16:27,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +19: [2023-05-11 00:16:27,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +15: [2023-05-11 00:16:27,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +18: [2023-05-11 00:16:27,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +18: [2023-05-11 00:16:27,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +27: [2023-05-11 00:16:27,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 5: [2023-05-11 00:16:27,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +18: [2023-05-11 00:16:27,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +16: [2023-05-11 00:16:27,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +16: [2023-05-11 00:16:27,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +18: [2023-05-11 00:16:27,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 4: [2023-05-11 00:16:27,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +14: [2023-05-11 00:16:27,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 4: [2023-05-11 00:16:27,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +18: [2023-05-11 00:16:27,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +14: [2023-05-11 00:16:27,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +16: [2023-05-11 00:16:27,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +16: [2023-05-11 00:16:27,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 2: [2023-05-11 00:16:27,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 1: [2023-05-11 00:16:27,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 8: [2023-05-11 00:16:27,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 1: [2023-05-11 00:16:27,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 1: [2023-05-11 00:16:27,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +26: [2023-05-11 00:16:27,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 1: [2023-05-11 00:16:27,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 1: [2023-05-11 00:16:27,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 1: [2023-05-11 00:16:27,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 8: [2023-05-11 00:16:27,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 8: [2023-05-11 00:16:27,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 0: [2023-05-11 00:16:27,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +26: [2023-05-11 00:16:27,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +16: [2023-05-11 00:16:27,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +16: [2023-05-11 00:16:27,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +16: [2023-05-11 00:16:27,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +16: [2023-05-11 00:16:27,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +14: [2023-05-11 00:16:27,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +25: [2023-05-11 00:16:27,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +22: [2023-05-11 00:16:27,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +22: [2023-05-11 00:16:27,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +22: [2023-05-11 00:16:27,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +22: [2023-05-11 00:16:27,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 5: [2023-05-11 00:16:27,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +14: [2023-05-11 00:16:27,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +20: [2023-05-11 00:16:27,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +20: [2023-05-11 00:16:27,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +20: [2023-05-11 00:16:27,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 5: [2023-05-11 00:16:27,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +20: [2023-05-11 00:16:27,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. + 5: [2023-05-11 00:16:27,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 4: [2023-05-11 00:16:27,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +13: [2023-05-11 00:16:27,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +13: [2023-05-11 00:16:27,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +13: [2023-05-11 00:16:27,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +13: [2023-05-11 00:16:27,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt. +25: [2023-05-11 00:16:27,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 4: [2023-05-11 00:16:27,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 5: [2023-05-11 00:16:27,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +14: [2023-05-11 00:16:27,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +14: [2023-05-11 00:16:27,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 4: [2023-05-11 00:16:27,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +14: [2023-05-11 00:16:27,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +14: [2023-05-11 00:16:27,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +20: [2023-05-11 00:16:27,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 4: [2023-05-11 00:16:27,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +13: [2023-05-11 00:16:27,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +20: [2023-05-11 00:16:27,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +20: [2023-05-11 00:16:27,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +13: [2023-05-11 00:16:27,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +13: [2023-05-11 00:16:27,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +20: [2023-05-11 00:16:27,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 0: [2023-05-11 00:16:27,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +22: [2023-05-11 00:16:27,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +13: [2023-05-11 00:16:27,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +10: [2023-05-11 00:16:27,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +10: [2023-05-11 00:16:27,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +22: [2023-05-11 00:16:27,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +22: [2023-05-11 00:16:27,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +22: [2023-05-11 00:16:27,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt... +10: [2023-05-11 00:16:27,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +10: [2023-05-11 00:16:27,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +18: [2023-05-11 00:16:27,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +18: [2023-05-11 00:16:27,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +18: [2023-05-11 00:16:27,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +18: [2023-05-11 00:16:27,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +18: [2023-05-11 00:16:27,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +18: [2023-05-11 00:16:27,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +18: [2023-05-11 00:16:27,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +20: [2023-05-11 00:16:27,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +13: [2023-05-11 00:16:27,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +20: [2023-05-11 00:16:27,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +13: [2023-05-11 00:16:27,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +18: [2023-05-11 00:16:27,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +10: [2023-05-11 00:16:27,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 5: [2023-05-11 00:16:27,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +20: [2023-05-11 00:16:27,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +20: [2023-05-11 00:16:27,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... + 2: [2023-05-11 00:16:27,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +10: [2023-05-11 00:16:27,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +22: [2023-05-11 00:16:27,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +22: [2023-05-11 00:16:27,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +13: [2023-05-11 00:16:27,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +13: [2023-05-11 00:16:27,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +10: [2023-05-11 00:16:27,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +10: [2023-05-11 00:16:27,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +31: [2023-05-11 00:16:27,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +31: [2023-05-11 00:16:27,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +31: [2023-05-11 00:16:27,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +31: [2023-05-11 00:16:27,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +18: [2023-05-11 00:16:27,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +18: [2023-05-11 00:16:27,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +18: [2023-05-11 00:16:27,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +13: [2023-05-11 00:16:27,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +20: [2023-05-11 00:16:27,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +15: [2023-05-11 00:16:27,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +15: [2023-05-11 00:16:27,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +15: [2023-05-11 00:16:27,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +15: [2023-05-11 00:16:27,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +15: [2023-05-11 00:16:27,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +15: [2023-05-11 00:16:27,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +15: [2023-05-11 00:16:27,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +15: [2023-05-11 00:16:27,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +15: [2023-05-11 00:16:27,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +15: [2023-05-11 00:16:27,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +15: [2023-05-11 00:16:27,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +15: [2023-05-11 00:16:27,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +15: [2023-05-11 00:16:27,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +15: [2023-05-11 00:16:27,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +20: [2023-05-11 00:16:27,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +13: [2023-05-11 00:16:27,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +31: [2023-05-11 00:16:27,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +15: [2023-05-11 00:16:27,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +15: [2023-05-11 00:16:27,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +18: [2023-05-11 00:16:27,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +13: [2023-05-11 00:16:27,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +22: [2023-05-11 00:16:27,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +20: [2023-05-11 00:16:27,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +22: [2023-05-11 00:16:27,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +31: [2023-05-11 00:16:27,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +22: [2023-05-11 00:16:27,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt. +18: [2023-05-11 00:16:27,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +31: [2023-05-11 00:16:27,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +31: [2023-05-11 00:16:27,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +13: [2023-05-11 00:16:27,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +20: [2023-05-11 00:16:27,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +22: [2023-05-11 00:16:27,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +18: [2023-05-11 00:16:27,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +22: [2023-05-11 00:16:27,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +22: [2023-05-11 00:16:27,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt... +18: [2023-05-11 00:16:27,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +15: [2023-05-11 00:16:27,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 9: [2023-05-11 00:16:27,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 9: [2023-05-11 00:16:27,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 9: [2023-05-11 00:16:27,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +15: [2023-05-11 00:16:27,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +15: [2023-05-11 00:16:27,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +18: [2023-05-11 00:16:27,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +15: [2023-05-11 00:16:27,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +15: [2023-05-11 00:16:27,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +15: [2023-05-11 00:16:27,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +29: [2023-05-11 00:16:27,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +29: [2023-05-11 00:16:27,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +15: [2023-05-11 00:16:27,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +29: [2023-05-11 00:16:27,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +29: [2023-05-11 00:16:27,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +24: [2023-05-11 00:16:27,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +24: [2023-05-11 00:16:27,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +24: [2023-05-11 00:16:27,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +24: [2023-05-11 00:16:27,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +24: [2023-05-11 00:16:27,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +24: [2023-05-11 00:16:27,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +24: [2023-05-11 00:16:27,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +24: [2023-05-11 00:16:27,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +29: [2023-05-11 00:16:27,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +15: [2023-05-11 00:16:27,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +29: [2023-05-11 00:16:27,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +29: [2023-05-11 00:16:27,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +29: [2023-05-11 00:16:27,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +24: [2023-05-11 00:16:27,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +24: [2023-05-11 00:16:27,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +24: [2023-05-11 00:16:27,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +24: [2023-05-11 00:16:27,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +24: [2023-05-11 00:16:27,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +24: [2023-05-11 00:16:27,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +24: [2023-05-11 00:16:27,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +24: [2023-05-11 00:16:27,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +12: [2023-05-11 00:16:27,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +12: [2023-05-11 00:16:27,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +29: [2023-05-11 00:16:27,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +29: [2023-05-11 00:16:27,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +29: [2023-05-11 00:16:27,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +29: [2023-05-11 00:16:27,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +29: [2023-05-11 00:16:27,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +29: [2023-05-11 00:16:27,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +12: [2023-05-11 00:16:27,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +16: [2023-05-11 00:16:27,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +29: [2023-05-11 00:16:27,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +16: [2023-05-11 00:16:27,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +16: [2023-05-11 00:16:27,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +29: [2023-05-11 00:16:27,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +29: [2023-05-11 00:16:27,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +16: [2023-05-11 00:16:27,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +29: [2023-05-11 00:16:27,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +29: [2023-05-11 00:16:27,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +29: [2023-05-11 00:16:27,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +29: [2023-05-11 00:16:27,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +29: [2023-05-11 00:16:27,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +16: [2023-05-11 00:16:27,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +16: [2023-05-11 00:16:27,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +16: [2023-05-11 00:16:27,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +16: [2023-05-11 00:16:27,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +16: [2023-05-11 00:16:27,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +16: [2023-05-11 00:16:27,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +16: [2023-05-11 00:16:27,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +16: [2023-05-11 00:16:27,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +16: [2023-05-11 00:16:27,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +16: [2023-05-11 00:16:27,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +16: [2023-05-11 00:16:27,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +16: [2023-05-11 00:16:27,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +12: [2023-05-11 00:16:27,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +29: [2023-05-11 00:16:27,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +29: [2023-05-11 00:16:27,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +10: [2023-05-11 00:16:27,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +10: [2023-05-11 00:16:27,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +10: [2023-05-11 00:16:27,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +10: [2023-05-11 00:16:27,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +10: [2023-05-11 00:16:27,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +10: [2023-05-11 00:16:27,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +10: [2023-05-11 00:16:27,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +10: [2023-05-11 00:16:27,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +10: [2023-05-11 00:16:27,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +10: [2023-05-11 00:16:27,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +10: [2023-05-11 00:16:27,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +10: [2023-05-11 00:16:27,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +10: [2023-05-11 00:16:27,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +10: [2023-05-11 00:16:27,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +10: [2023-05-11 00:16:27,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +30: [2023-05-11 00:16:27,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +30: [2023-05-11 00:16:27,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +30: [2023-05-11 00:16:27,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +30: [2023-05-11 00:16:27,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +10: [2023-05-11 00:16:27,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +30: [2023-05-11 00:16:27,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +30: [2023-05-11 00:16:27,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +30: [2023-05-11 00:16:27,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +28: [2023-05-11 00:16:27,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +28: [2023-05-11 00:16:27,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +28: [2023-05-11 00:16:27,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +28: [2023-05-11 00:16:27,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +30: [2023-05-11 00:16:27,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +28: [2023-05-11 00:16:27,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +28: [2023-05-11 00:16:27,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +28: [2023-05-11 00:16:27,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +28: [2023-05-11 00:16:27,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +16: [2023-05-11 00:16:27,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +29: [2023-05-11 00:16:27,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +10: [2023-05-11 00:16:27,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +16: [2023-05-11 00:16:27,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +29: [2023-05-11 00:16:27,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +10: [2023-05-11 00:16:27,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +30: [2023-05-11 00:16:27,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +16: [2023-05-11 00:16:27,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +16: [2023-05-11 00:16:27,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +30: [2023-05-11 00:16:27,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +30: [2023-05-11 00:16:27,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +28: [2023-05-11 00:16:27,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +28: [2023-05-11 00:16:27,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +16: [2023-05-11 00:16:27,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +29: [2023-05-11 00:16:27,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +12: [2023-05-11 00:16:27,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +12: [2023-05-11 00:16:27,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +10: [2023-05-11 00:16:27,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +10: [2023-05-11 00:16:27,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +16: [2023-05-11 00:16:27,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +21: [2023-05-11 00:16:27,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +21: [2023-05-11 00:16:27,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +21: [2023-05-11 00:16:27,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +21: [2023-05-11 00:16:27,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +21: [2023-05-11 00:16:27,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +21: [2023-05-11 00:16:27,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +21: [2023-05-11 00:16:27,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +29: [2023-05-11 00:16:27,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +12: [2023-05-11 00:16:27,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +30: [2023-05-11 00:16:27,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +16: [2023-05-11 00:16:27,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +21: [2023-05-11 00:16:27,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +16: [2023-05-11 00:16:27,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +12: [2023-05-11 00:16:27,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +30: [2023-05-11 00:16:27,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +30: [2023-05-11 00:16:27,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +10: [2023-05-11 00:16:27,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +28: [2023-05-11 00:16:27,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +29: [2023-05-11 00:16:27,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +29: [2023-05-11 00:16:27,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +10: [2023-05-11 00:16:27,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +28: [2023-05-11 00:16:27,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +12: [2023-05-11 00:16:27,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +12: [2023-05-11 00:16:27,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +28: [2023-05-11 00:16:27,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +30: [2023-05-11 00:16:27,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +28: [2023-05-11 00:16:27,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +29: [2023-05-11 00:16:27,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 6: [2023-05-11 00:16:27,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +29: [2023-05-11 00:16:27,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +10: [2023-05-11 00:16:27,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +30: [2023-05-11 00:16:27,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +21: [2023-05-11 00:16:27,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +28: [2023-05-11 00:16:27,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +28: [2023-05-11 00:16:27,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +10: [2023-05-11 00:16:27,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +21: [2023-05-11 00:16:27,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +21: [2023-05-11 00:16:27,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 6: [2023-05-11 00:16:27,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +21: [2023-05-11 00:16:27,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 6: [2023-05-11 00:16:27,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +23: [2023-05-11 00:16:27,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +21: [2023-05-11 00:16:27,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +23: [2023-05-11 00:16:27,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +23: [2023-05-11 00:16:27,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +12: [2023-05-11 00:16:27,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +23: [2023-05-11 00:16:27,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +21: [2023-05-11 00:16:27,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +21: [2023-05-11 00:16:27,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +21: [2023-05-11 00:16:27,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +12: [2023-05-11 00:16:27,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +28: [2023-05-11 00:16:27,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +28: [2023-05-11 00:16:27,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +28: [2023-05-11 00:16:27,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +23: [2023-05-11 00:16:27,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +28: [2023-05-11 00:16:27,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +23: [2023-05-11 00:16:27,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +12: [2023-05-11 00:16:27,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +23: [2023-05-11 00:16:27,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +23: [2023-05-11 00:16:27,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +28: [2023-05-11 00:16:27,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +28: [2023-05-11 00:16:27,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +28: [2023-05-11 00:16:27,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +28: [2023-05-11 00:16:27,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +28: [2023-05-11 00:16:27,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +28: [2023-05-11 00:16:27,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +28: [2023-05-11 00:16:27,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +28: [2023-05-11 00:16:27,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +28: [2023-05-11 00:16:27,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +28: [2023-05-11 00:16:27,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +28: [2023-05-11 00:16:27,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +28: [2023-05-11 00:16:27,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 9: [2023-05-11 00:16:27,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 3: [2023-05-11 00:16:27,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 9: [2023-05-11 00:16:27,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 3: [2023-05-11 00:16:27,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +25: [2023-05-11 00:16:27,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +25: [2023-05-11 00:16:27,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +25: [2023-05-11 00:16:27,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +25: [2023-05-11 00:16:27,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +25: [2023-05-11 00:16:27,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +25: [2023-05-11 00:16:27,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +25: [2023-05-11 00:16:27,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +25: [2023-05-11 00:16:27,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 7: [2023-05-11 00:16:27,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 7: [2023-05-11 00:16:27,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 7: [2023-05-11 00:16:27,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 7: [2023-05-11 00:16:27,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 3: [2023-05-11 00:16:27,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 5: [2023-05-11 00:16:27,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 5: [2023-05-11 00:16:27,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 5: [2023-05-11 00:16:27,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 4: [2023-05-11 00:16:27,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 4: [2023-05-11 00:16:27,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 4: [2023-05-11 00:16:27,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 4: [2023-05-11 00:16:27,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +25: [2023-05-11 00:16:27,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +25: [2023-05-11 00:16:27,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +29: [2023-05-11 00:16:27,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +29: [2023-05-11 00:16:27,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +29: [2023-05-11 00:16:27,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +29: [2023-05-11 00:16:27,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 7: [2023-05-11 00:16:27,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 6: [2023-05-11 00:16:27,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 6: [2023-05-11 00:16:27,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +14: [2023-05-11 00:16:27,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +14: [2023-05-11 00:16:27,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 6: [2023-05-11 00:16:27,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +25: [2023-05-11 00:16:27,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 6: [2023-05-11 00:16:27,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +14: [2023-05-11 00:16:27,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 6: [2023-05-11 00:16:27,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +14: [2023-05-11 00:16:27,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 4: [2023-05-11 00:16:27,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 4: [2023-05-11 00:16:27,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 4: [2023-05-11 00:16:27,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 4: [2023-05-11 00:16:27,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 6: [2023-05-11 00:16:27,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 6: [2023-05-11 00:16:27,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 6: [2023-05-11 00:16:27,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 6: [2023-05-11 00:16:27,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 9: [2023-05-11 00:16:27,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +25: [2023-05-11 00:16:27,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +27: [2023-05-11 00:16:27,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +27: [2023-05-11 00:16:27,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +27: [2023-05-11 00:16:27,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +27: [2023-05-11 00:16:27,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +27: [2023-05-11 00:16:27,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +27: [2023-05-11 00:16:27,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +25: [2023-05-11 00:16:27,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +14: [2023-05-11 00:16:27,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +14: [2023-05-11 00:16:27,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +14: [2023-05-11 00:16:27,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +14: [2023-05-11 00:16:27,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +20: [2023-05-11 00:16:27,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +20: [2023-05-11 00:16:27,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +27: [2023-05-11 00:16:27,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +20: [2023-05-11 00:16:27,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +20: [2023-05-11 00:16:27,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +20: [2023-05-11 00:16:27,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +20: [2023-05-11 00:16:27,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +20: [2023-05-11 00:16:27,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +27: [2023-05-11 00:16:27,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +25: [2023-05-11 00:16:27,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +31: [2023-05-11 00:16:27,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +31: [2023-05-11 00:16:27,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +20: [2023-05-11 00:16:27,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +29: [2023-05-11 00:16:27,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +29: [2023-05-11 00:16:27,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +31: [2023-05-11 00:16:27,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +23: [2023-05-11 00:16:27,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +23: [2023-05-11 00:16:27,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +23: [2023-05-11 00:16:27,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +23: [2023-05-11 00:16:27,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +29: [2023-05-11 00:16:27,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +29: [2023-05-11 00:16:27,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +31: [2023-05-11 00:16:27,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +31: [2023-05-11 00:16:27,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +31: [2023-05-11 00:16:27,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +31: [2023-05-11 00:16:27,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +31: [2023-05-11 00:16:27,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +31: [2023-05-11 00:16:27,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +31: [2023-05-11 00:16:27,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +31: [2023-05-11 00:16:27,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +31: [2023-05-11 00:16:27,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +14: [2023-05-11 00:16:27,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +31: [2023-05-11 00:16:27,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +31: [2023-05-11 00:16:27,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 9: [2023-05-11 00:16:27,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +25: [2023-05-11 00:16:27,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +23: [2023-05-11 00:16:27,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +23: [2023-05-11 00:16:27,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +23: [2023-05-11 00:16:27,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +23: [2023-05-11 00:16:27,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +23: [2023-05-11 00:16:27,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +23: [2023-05-11 00:16:27,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +23: [2023-05-11 00:16:27,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +23: [2023-05-11 00:16:27,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +23: [2023-05-11 00:16:27,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +23: [2023-05-11 00:16:27,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +23: [2023-05-11 00:16:27,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +23: [2023-05-11 00:16:27,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +31: [2023-05-11 00:16:27,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +31: [2023-05-11 00:16:27,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 5: [2023-05-11 00:16:27,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 5: [2023-05-11 00:16:27,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +25: [2023-05-11 00:16:27,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +15: [2023-05-11 00:16:27,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +15: [2023-05-11 00:16:27,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +15: [2023-05-11 00:16:27,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +15: [2023-05-11 00:16:27,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +27: [2023-05-11 00:16:27,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +27: [2023-05-11 00:16:27,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 6: [2023-05-11 00:16:27,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 6: [2023-05-11 00:16:27,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +14: [2023-05-11 00:16:27,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +14: [2023-05-11 00:16:27,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 6: [2023-05-11 00:16:27,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +20: [2023-05-11 00:16:27,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +23: [2023-05-11 00:16:27,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +20: [2023-05-11 00:16:27,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +20: [2023-05-11 00:16:27,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +31: [2023-05-11 00:16:27,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +21: [2023-05-11 00:16:27,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +21: [2023-05-11 00:16:27,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +21: [2023-05-11 00:16:27,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +21: [2023-05-11 00:16:27,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +15: [2023-05-11 00:16:27,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +23: [2023-05-11 00:16:27,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +15: [2023-05-11 00:16:27,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +31: [2023-05-11 00:16:27,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +31: [2023-05-11 00:16:27,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +15: [2023-05-11 00:16:27,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +15: [2023-05-11 00:16:27,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +14: [2023-05-11 00:16:27,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +14: [2023-05-11 00:16:27,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +27: [2023-05-11 00:16:27,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +23: [2023-05-11 00:16:27,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +27: [2023-05-11 00:16:27,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +14: [2023-05-11 00:16:27,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 6: [2023-05-11 00:16:27,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +10: [2023-05-11 00:16:27,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +10: [2023-05-11 00:16:27,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +10: [2023-05-11 00:16:27,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +23: [2023-05-11 00:16:27,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +20: [2023-05-11 00:16:27,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +27: [2023-05-11 00:16:27,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +27: [2023-05-11 00:16:27,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +10: [2023-05-11 00:16:27,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +20: [2023-05-11 00:16:27,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +20: [2023-05-11 00:16:27,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +20: [2023-05-11 00:16:27,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +21: [2023-05-11 00:16:27,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +23: [2023-05-11 00:16:27,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +31: [2023-05-11 00:16:27,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +31: [2023-05-11 00:16:27,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +23: [2023-05-11 00:16:27,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +21: [2023-05-11 00:16:27,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +21: [2023-05-11 00:16:27,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +21: [2023-05-11 00:16:27,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +31: [2023-05-11 00:16:27,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +14: [2023-05-11 00:16:27,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +14: [2023-05-11 00:16:27,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +31: [2023-05-11 00:16:27,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +23: [2023-05-11 00:16:27,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 8: [2023-05-11 00:16:27,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 8: [2023-05-11 00:16:27,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 8: [2023-05-11 00:16:27,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +16: [2023-05-11 00:16:27,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +16: [2023-05-11 00:16:27,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +16: [2023-05-11 00:16:27,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +16: [2023-05-11 00:16:27,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +10: [2023-05-11 00:16:27,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +10: [2023-05-11 00:16:27,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +27: [2023-05-11 00:16:27,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +27: [2023-05-11 00:16:27,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +20: [2023-05-11 00:16:27,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +10: [2023-05-11 00:16:27,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +10: [2023-05-11 00:16:27,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +23: [2023-05-11 00:16:27,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +18: [2023-05-11 00:16:27,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +18: [2023-05-11 00:16:27,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +18: [2023-05-11 00:16:27,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +18: [2023-05-11 00:16:27,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +31: [2023-05-11 00:16:27,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +18: [2023-05-11 00:16:27,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +18: [2023-05-11 00:16:27,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +18: [2023-05-11 00:16:27,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +21: [2023-05-11 00:16:27,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +21: [2023-05-11 00:16:27,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +24: [2023-05-11 00:16:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +24: [2023-05-11 00:16:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +24: [2023-05-11 00:16:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +16: [2023-05-11 00:16:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +24: [2023-05-11 00:16:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +14: [2023-05-11 00:16:27,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 8: [2023-05-11 00:16:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +11: [2023-05-11 00:16:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +11: [2023-05-11 00:16:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +11: [2023-05-11 00:16:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +11: [2023-05-11 00:16:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +14: [2023-05-11 00:16:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 8: [2023-05-11 00:16:27,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +14: [2023-05-11 00:16:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +14: [2023-05-11 00:16:27,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +18: [2023-05-11 00:16:27,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +20: [2023-05-11 00:16:27,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +20: [2023-05-11 00:16:27,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +20: [2023-05-11 00:16:27,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 8: [2023-05-11 00:16:27,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +20: [2023-05-11 00:16:27,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +16: [2023-05-11 00:16:27,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +16: [2023-05-11 00:16:27,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +21: [2023-05-11 00:16:27,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +21: [2023-05-11 00:16:27,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +16: [2023-05-11 00:16:27,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +24: [2023-05-11 00:16:27,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +14: [2023-05-11 00:16:27,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +20: [2023-05-11 00:16:27,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +14: [2023-05-11 00:16:27,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +14: [2023-05-11 00:16:27,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +21: [2023-05-11 00:16:27,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +21: [2023-05-11 00:16:27,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +14: [2023-05-11 00:16:27,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +20: [2023-05-11 00:16:27,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +20: [2023-05-11 00:16:27,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +20: [2023-05-11 00:16:27,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +18: [2023-05-11 00:16:27,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +11: [2023-05-11 00:16:27,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +24: [2023-05-11 00:16:27,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +24: [2023-05-11 00:16:27,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +11: [2023-05-11 00:16:27,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +24: [2023-05-11 00:16:27,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +11: [2023-05-11 00:16:27,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +21: [2023-05-11 00:16:27,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +21: [2023-05-11 00:16:27,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +11: [2023-05-11 00:16:27,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +18: [2023-05-11 00:16:27,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +18: [2023-05-11 00:16:27,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +24: [2023-05-11 00:16:27,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +24: [2023-05-11 00:16:27,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +20: [2023-05-11 00:16:27,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +18: [2023-05-11 00:16:27,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 2: [2023-05-11 00:16:27,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +18: [2023-05-11 00:16:27,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +24: [2023-05-11 00:16:27,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +24: [2023-05-11 00:16:27,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +20: [2023-05-11 00:16:27,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 2: [2023-05-11 00:16:27,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 2: [2023-05-11 00:16:27,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +18: [2023-05-11 00:16:27,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +14: [2023-05-11 00:16:27,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +14: [2023-05-11 00:16:27,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +18: [2023-05-11 00:16:27,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +14: [2023-05-11 00:16:27,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +14: [2023-05-11 00:16:27,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +14: [2023-05-11 00:16:27,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +20: [2023-05-11 00:16:27,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +14: [2023-05-11 00:16:27,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +20: [2023-05-11 00:16:27,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +18: [2023-05-11 00:16:27,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +24: [2023-05-11 00:16:27,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 1: [2023-05-11 00:16:27,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 1: [2023-05-11 00:16:27,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 1: [2023-05-11 00:16:27,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 1: [2023-05-11 00:16:27,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +24: [2023-05-11 00:16:27,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +24: [2023-05-11 00:16:27,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +18: [2023-05-11 00:16:27,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +24: [2023-05-11 00:16:27,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +18: [2023-05-11 00:16:27,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +14: [2023-05-11 00:16:27,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +14: [2023-05-11 00:16:27,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +22: [2023-05-11 00:16:27,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +20: [2023-05-11 00:16:27,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +20: [2023-05-11 00:16:27,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +20: [2023-05-11 00:16:27,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +20: [2023-05-11 00:16:27,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 2: [2023-05-11 00:16:27,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +17: [2023-05-11 00:16:27,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +17: [2023-05-11 00:16:27,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +17: [2023-05-11 00:16:27,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +17: [2023-05-11 00:16:27,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +30: [2023-05-11 00:16:27,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 2: [2023-05-11 00:16:27,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +30: [2023-05-11 00:16:27,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +30: [2023-05-11 00:16:27,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 2: [2023-05-11 00:16:27,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +30: [2023-05-11 00:16:27,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +13: [2023-05-11 00:16:27,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +13: [2023-05-11 00:16:27,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 0: [2023-05-11 00:16:27,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 0: [2023-05-11 00:16:27,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +27: [2023-05-11 00:16:27,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +27: [2023-05-11 00:16:27,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +27: [2023-05-11 00:16:27,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +27: [2023-05-11 00:16:27,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +22: [2023-05-11 00:16:27,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +18: [2023-05-11 00:16:27,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 0: [2023-05-11 00:16:27,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 2: [2023-05-11 00:16:27,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +13: [2023-05-11 00:16:27,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +18: [2023-05-11 00:16:27,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +18: [2023-05-11 00:16:27,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +22: [2023-05-11 00:16:27,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 1: [2023-05-11 00:16:27,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +13: [2023-05-11 00:16:27,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +22: [2023-05-11 00:16:27,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +18: [2023-05-11 00:16:27,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +29: [2023-05-11 00:16:27,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +29: [2023-05-11 00:16:27,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +29: [2023-05-11 00:16:27,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +29: [2023-05-11 00:16:27,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +29: [2023-05-11 00:16:27,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +29: [2023-05-11 00:16:27,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +29: [2023-05-11 00:16:27,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +29: [2023-05-11 00:16:27,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +29: [2023-05-11 00:16:27,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +29: [2023-05-11 00:16:27,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +18: [2023-05-11 00:16:27,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +29: [2023-05-11 00:16:27,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +29: [2023-05-11 00:16:27,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +29: [2023-05-11 00:16:27,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +29: [2023-05-11 00:16:27,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +29: [2023-05-11 00:16:27,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +29: [2023-05-11 00:16:27,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +22: [2023-05-11 00:16:27,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +17: [2023-05-11 00:16:27,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +17: [2023-05-11 00:16:27,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +17: [2023-05-11 00:16:27,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +27: [2023-05-11 00:16:27,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +27: [2023-05-11 00:16:27,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +30: [2023-05-11 00:16:27,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +17: [2023-05-11 00:16:27,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +30: [2023-05-11 00:16:27,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +27: [2023-05-11 00:16:27,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +26: [2023-05-11 00:16:27,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 0: [2023-05-11 00:16:27,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +26: [2023-05-11 00:16:27,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +26: [2023-05-11 00:16:27,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 0: [2023-05-11 00:16:27,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +13: [2023-05-11 00:16:27,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +13: [2023-05-11 00:16:27,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 3: [2023-05-11 00:16:27,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 3: [2023-05-11 00:16:27,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +13: [2023-05-11 00:16:27,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +27: [2023-05-11 00:16:27,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +30: [2023-05-11 00:16:27,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +30: [2023-05-11 00:16:27,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +26: [2023-05-11 00:16:27,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +22: [2023-05-11 00:16:27,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +22: [2023-05-11 00:16:27,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +13: [2023-05-11 00:16:27,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 1: [2023-05-11 00:16:27,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 1: [2023-05-11 00:16:27,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +18: [2023-05-11 00:16:27,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +22: [2023-05-11 00:16:27,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +19: [2023-05-11 00:16:27,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. + 7: [2023-05-11 00:16:27,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 7: [2023-05-11 00:16:27,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 7: [2023-05-11 00:16:27,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +19: [2023-05-11 00:16:27,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +18: [2023-05-11 00:16:27,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +18: [2023-05-11 00:16:27,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 1: [2023-05-11 00:16:27,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 1: [2023-05-11 00:16:27,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 7: [2023-05-11 00:16:27,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +29: [2023-05-11 00:16:27,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +25: [2023-05-11 00:16:27,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +25: [2023-05-11 00:16:27,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +25: [2023-05-11 00:16:27,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +19: [2023-05-11 00:16:27,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +19: [2023-05-11 00:16:27,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +25: [2023-05-11 00:16:27,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt. +26: [2023-05-11 00:16:27,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 1: [2023-05-11 00:16:27,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 1: [2023-05-11 00:16:27,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +18: [2023-05-11 00:16:27,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +18: [2023-05-11 00:16:27,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 1: [2023-05-11 00:16:27,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 8: [2023-05-11 00:16:27,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +26: [2023-05-11 00:16:27,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +18: [2023-05-11 00:16:27,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +18: [2023-05-11 00:16:27,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +26: [2023-05-11 00:16:27,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 8: [2023-05-11 00:16:27,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +26: [2023-05-11 00:16:27,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +29: [2023-05-11 00:16:27,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +29: [2023-05-11 00:16:27,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +29: [2023-05-11 00:16:27,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +25: [2023-05-11 00:16:27,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +18: [2023-05-11 00:16:27,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +26: [2023-05-11 00:16:27,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +26: [2023-05-11 00:16:27,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +26: [2023-05-11 00:16:27,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 8: [2023-05-11 00:16:27,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 8: [2023-05-11 00:16:27,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 8: [2023-05-11 00:16:27,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 8: [2023-05-11 00:16:27,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 8: [2023-05-11 00:16:27,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 8: [2023-05-11 00:16:27,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 8: [2023-05-11 00:16:27,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +26: [2023-05-11 00:16:27,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +26: [2023-05-11 00:16:27,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +26: [2023-05-11 00:16:27,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 8: [2023-05-11 00:16:27,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +18: [2023-05-11 00:16:27,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 8: [2023-05-11 00:16:27,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 8: [2023-05-11 00:16:27,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +19: [2023-05-11 00:16:27,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +19: [2023-05-11 00:16:27,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +29: [2023-05-11 00:16:27,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +28: [2023-05-11 00:16:27,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +28: [2023-05-11 00:16:27,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +28: [2023-05-11 00:16:27,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +28: [2023-05-11 00:16:27,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +28: [2023-05-11 00:16:27,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +28: [2023-05-11 00:16:27,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 8: [2023-05-11 00:16:27,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +19: [2023-05-11 00:16:27,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +25: [2023-05-11 00:16:27,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +28: [2023-05-11 00:16:27,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +19: [2023-05-11 00:16:27,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 1: [2023-05-11 00:16:27,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +25: [2023-05-11 00:16:27,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +26: [2023-05-11 00:16:27,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +26: [2023-05-11 00:16:27,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 1: [2023-05-11 00:16:27,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 1: [2023-05-11 00:16:27,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +30: [2023-05-11 00:16:27,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +27: [2023-05-11 00:16:27,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +30: [2023-05-11 00:16:27,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 1: [2023-05-11 00:16:27,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +27: [2023-05-11 00:16:27,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +28: [2023-05-11 00:16:27,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +25: [2023-05-11 00:16:27,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +27: [2023-05-11 00:16:27,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +29: [2023-05-11 00:16:27,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +29: [2023-05-11 00:16:27,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +30: [2023-05-11 00:16:27,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +27: [2023-05-11 00:16:27,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +27: [2023-05-11 00:16:27,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +29: [2023-05-11 00:16:27,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +30: [2023-05-11 00:16:27,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 5: [2023-05-11 00:16:27,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 1: [2023-05-11 00:16:27,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +27: [2023-05-11 00:16:27,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +26: [2023-05-11 00:16:27,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +25: [2023-05-11 00:16:27,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +26: [2023-05-11 00:16:27,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 5: [2023-05-11 00:16:27,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +30: [2023-05-11 00:16:27,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 8: [2023-05-11 00:16:27,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +30: [2023-05-11 00:16:27,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +21: [2023-05-11 00:16:27,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +21: [2023-05-11 00:16:27,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +21: [2023-05-11 00:16:27,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +21: [2023-05-11 00:16:27,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +28: [2023-05-11 00:16:27,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +21: [2023-05-11 00:16:27,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +21: [2023-05-11 00:16:27,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +21: [2023-05-11 00:16:27,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +21: [2023-05-11 00:16:27,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +27: [2023-05-11 00:16:27,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +25: [2023-05-11 00:16:27,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +28: [2023-05-11 00:16:27,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +28: [2023-05-11 00:16:27,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +26: [2023-05-11 00:16:27,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +26: [2023-05-11 00:16:27,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 1: [2023-05-11 00:16:27,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +27: [2023-05-11 00:16:27,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +30: [2023-05-11 00:16:27,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +30: [2023-05-11 00:16:27,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 3: [2023-05-11 00:16:27,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 3: [2023-05-11 00:16:27,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 3: [2023-05-11 00:16:27,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +25: [2023-05-11 00:16:27,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +25: [2023-05-11 00:16:27,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 1: [2023-05-11 00:16:27,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 7: [2023-05-11 00:16:27,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 1: [2023-05-11 00:16:27,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 5: [2023-05-11 00:16:27,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +26: [2023-05-11 00:16:27,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +25: [2023-05-11 00:16:27,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 8: [2023-05-11 00:16:27,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +28: [2023-05-11 00:16:27,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +25: [2023-05-11 00:16:27,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 8: [2023-05-11 00:16:27,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +26: [2023-05-11 00:16:27,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 3: [2023-05-11 00:16:27,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 8: [2023-05-11 00:16:27,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +21: [2023-05-11 00:16:27,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +28: [2023-05-11 00:16:27,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +26: [2023-05-11 00:16:27,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +26: [2023-05-11 00:16:27,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +26: [2023-05-11 00:16:27,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +28: [2023-05-11 00:16:27,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 7: [2023-05-11 00:16:27,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 1: [2023-05-11 00:16:27,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +25: [2023-05-11 00:16:27,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +12: [2023-05-11 00:16:27,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +12: [2023-05-11 00:16:27,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +12: [2023-05-11 00:16:27,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +12: [2023-05-11 00:16:27,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +12: [2023-05-11 00:16:27,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +12: [2023-05-11 00:16:27,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +12: [2023-05-11 00:16:27,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +25: [2023-05-11 00:16:27,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +21: [2023-05-11 00:16:27,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +21: [2023-05-11 00:16:27,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +21: [2023-05-11 00:16:27,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 5: [2023-05-11 00:16:27,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +16: [2023-05-11 00:16:27,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +16: [2023-05-11 00:16:27,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +16: [2023-05-11 00:16:27,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +16: [2023-05-11 00:16:27,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +16: [2023-05-11 00:16:27,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +16: [2023-05-11 00:16:27,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +16: [2023-05-11 00:16:27,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +16: [2023-05-11 00:16:27,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +16: [2023-05-11 00:16:27,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +16: [2023-05-11 00:16:27,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +16: [2023-05-11 00:16:27,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +16: [2023-05-11 00:16:27,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +16: [2023-05-11 00:16:27,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +16: [2023-05-11 00:16:27,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +21: [2023-05-11 00:16:27,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +16: [2023-05-11 00:16:27,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +26: [2023-05-11 00:16:27,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +26: [2023-05-11 00:16:27,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +26: [2023-05-11 00:16:27,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +28: [2023-05-11 00:16:27,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +16: [2023-05-11 00:16:27,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +26: [2023-05-11 00:16:27,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 8: [2023-05-11 00:16:27,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 4: [2023-05-11 00:16:27,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 4: [2023-05-11 00:16:27,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +26: [2023-05-11 00:16:27,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +13: [2023-05-11 00:16:27,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +13: [2023-05-11 00:16:27,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +26: [2023-05-11 00:16:27,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +13: [2023-05-11 00:16:27,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +13: [2023-05-11 00:16:27,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +10: [2023-05-11 00:16:27,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +10: [2023-05-11 00:16:27,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +10: [2023-05-11 00:16:27,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +10: [2023-05-11 00:16:27,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +10: [2023-05-11 00:16:27,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +10: [2023-05-11 00:16:27,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +10: [2023-05-11 00:16:27,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +10: [2023-05-11 00:16:27,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +10: [2023-05-11 00:16:27,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +10: [2023-05-11 00:16:27,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +10: [2023-05-11 00:16:27,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +10: [2023-05-11 00:16:27,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +19: [2023-05-11 00:16:27,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +19: [2023-05-11 00:16:27,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +19: [2023-05-11 00:16:27,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +19: [2023-05-11 00:16:27,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +10: [2023-05-11 00:16:27,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +10: [2023-05-11 00:16:27,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +10: [2023-05-11 00:16:27,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +10: [2023-05-11 00:16:27,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +11: [2023-05-11 00:16:27,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +12: [2023-05-11 00:16:27,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +12: [2023-05-11 00:16:27,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +11: [2023-05-11 00:16:27,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +19: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +11: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +19: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +19: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +19: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +22: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +22: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +22: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +22: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +22: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +22: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +22: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +22: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +22: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +22: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 5: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +19: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +19: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +22: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +22: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +22: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +19: [2023-05-11 00:16:27,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +22: [2023-05-11 00:16:27,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +19: [2023-05-11 00:16:27,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +22: [2023-05-11 00:16:27,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +13: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +13: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +13: [2023-05-11 00:16:27,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +13: [2023-05-11 00:16:27,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +13: [2023-05-11 00:16:27,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +13: [2023-05-11 00:16:27,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +19: [2023-05-11 00:16:27,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +19: [2023-05-11 00:16:27,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 4: [2023-05-11 00:16:27,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +13: [2023-05-11 00:16:27,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +13: [2023-05-11 00:16:27,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +13: [2023-05-11 00:16:27,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +11: [2023-05-11 00:16:27,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +22: [2023-05-11 00:16:27,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +26: [2023-05-11 00:16:27,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +13: [2023-05-11 00:16:27,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +13: [2023-05-11 00:16:27,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +13: [2023-05-11 00:16:27,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +21: [2023-05-11 00:16:27,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +21: [2023-05-11 00:16:27,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 4: [2023-05-11 00:16:27,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +19: [2023-05-11 00:16:27,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +19: [2023-05-11 00:16:27,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +28: [2023-05-11 00:16:27,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +12: [2023-05-11 00:16:27,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +11: [2023-05-11 00:16:27,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +11: [2023-05-11 00:16:27,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +11: [2023-05-11 00:16:27,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +11: [2023-05-11 00:16:27,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +11: [2023-05-11 00:16:27,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +11: [2023-05-11 00:16:27,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +11: [2023-05-11 00:16:27,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +11: [2023-05-11 00:16:27,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +11: [2023-05-11 00:16:27,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +12: [2023-05-11 00:16:27,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 4: [2023-05-11 00:16:27,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 4: [2023-05-11 00:16:27,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +16: [2023-05-11 00:16:27,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +21: [2023-05-11 00:16:27,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +11: [2023-05-11 00:16:27,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +11: [2023-05-11 00:16:27,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +11: [2023-05-11 00:16:27,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +28: [2023-05-11 00:16:27,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +28: [2023-05-11 00:16:27,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +28: [2023-05-11 00:16:27,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +28: [2023-05-11 00:16:27,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +16: [2023-05-11 00:16:27,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +12: [2023-05-11 00:16:27,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +10: [2023-05-11 00:16:27,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +10: [2023-05-11 00:16:27,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +12: [2023-05-11 00:16:27,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +19: [2023-05-11 00:16:27,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +16: [2023-05-11 00:16:27,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +16: [2023-05-11 00:16:27,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +22: [2023-05-11 00:16:27,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +22: [2023-05-11 00:16:27,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +12: [2023-05-11 00:16:27,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +12: [2023-05-11 00:16:27,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +10: [2023-05-11 00:16:27,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 4: [2023-05-11 00:16:27,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +10: [2023-05-11 00:16:27,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +13: [2023-05-11 00:16:27,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +19: [2023-05-11 00:16:27,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +19: [2023-05-11 00:16:27,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +22: [2023-05-11 00:16:27,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 2: [2023-05-11 00:16:27,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 2: [2023-05-11 00:16:27,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +11: [2023-05-11 00:16:27,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +16: [2023-05-11 00:16:27,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 4: [2023-05-11 00:16:27,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +30: [2023-05-11 00:16:27,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +30: [2023-05-11 00:16:27,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +30: [2023-05-11 00:16:27,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +30: [2023-05-11 00:16:27,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +30: [2023-05-11 00:16:27,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +30: [2023-05-11 00:16:27,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +30: [2023-05-11 00:16:27,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +28: [2023-05-11 00:16:27,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +28: [2023-05-11 00:16:27,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +28: [2023-05-11 00:16:27,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +11: [2023-05-11 00:16:27,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +19: [2023-05-11 00:16:27,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +28: [2023-05-11 00:16:27,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +30: [2023-05-11 00:16:27,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +10: [2023-05-11 00:16:27,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +12: [2023-05-11 00:16:27,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +13: [2023-05-11 00:16:27,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +12: [2023-05-11 00:16:27,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +16: [2023-05-11 00:16:27,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +13: [2023-05-11 00:16:27,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +13: [2023-05-11 00:16:27,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +22: [2023-05-11 00:16:27,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 0: [2023-05-11 00:16:27,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 0: [2023-05-11 00:16:27,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 0: [2023-05-11 00:16:27,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +10: [2023-05-11 00:16:27,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +19: [2023-05-11 00:16:27,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 4: [2023-05-11 00:16:27,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +19: [2023-05-11 00:16:27,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +16: [2023-05-11 00:16:27,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +16: [2023-05-11 00:16:27,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +10: [2023-05-11 00:16:27,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... + 4: [2023-05-11 00:16:27,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +19: [2023-05-11 00:16:27,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +13: [2023-05-11 00:16:27,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 4: [2023-05-11 00:16:27,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +11: [2023-05-11 00:16:27,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +11: [2023-05-11 00:16:27,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +22: [2023-05-11 00:16:27,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +22: [2023-05-11 00:16:27,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +10: [2023-05-11 00:16:27,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +12: [2023-05-11 00:16:27,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +12: [2023-05-11 00:16:27,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +12: [2023-05-11 00:16:27,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +30: [2023-05-11 00:16:27,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +19: [2023-05-11 00:16:27,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +11: [2023-05-11 00:16:27,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +28: [2023-05-11 00:16:27,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +11: [2023-05-11 00:16:27,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +22: [2023-05-11 00:16:27,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +28: [2023-05-11 00:16:27,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +28: [2023-05-11 00:16:27,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +28: [2023-05-11 00:16:27,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +30: [2023-05-11 00:16:27,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +12: [2023-05-11 00:16:27,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +12: [2023-05-11 00:16:27,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +12: [2023-05-11 00:16:27,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +13: [2023-05-11 00:16:27,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +13: [2023-05-11 00:16:27,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +13: [2023-05-11 00:16:27,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +12: [2023-05-11 00:16:27,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +12: [2023-05-11 00:16:27,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +12: [2023-05-11 00:16:27,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 4: [2023-05-11 00:16:27,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +28: [2023-05-11 00:16:27,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +28: [2023-05-11 00:16:27,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +28: [2023-05-11 00:16:27,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +28: [2023-05-11 00:16:27,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +12: [2023-05-11 00:16:27,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +12: [2023-05-11 00:16:27,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +30: [2023-05-11 00:16:27,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 2: [2023-05-11 00:16:27,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +30: [2023-05-11 00:16:27,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +11: [2023-05-11 00:16:27,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 2: [2023-05-11 00:16:27,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +11: [2023-05-11 00:16:27,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +22: [2023-05-11 00:16:27,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 4: [2023-05-11 00:16:27,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 9: [2023-05-11 00:16:27,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +30: [2023-05-11 00:16:27,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 0: [2023-05-11 00:16:27,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +30: [2023-05-11 00:16:27,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +30: [2023-05-11 00:16:27,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 3: [2023-05-11 00:16:27,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +30: [2023-05-11 00:16:27,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 3: [2023-05-11 00:16:27,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 0: [2023-05-11 00:16:27,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 9: [2023-05-11 00:16:27,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 9: [2023-05-11 00:16:27,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 0: [2023-05-11 00:16:27,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +15: [2023-05-11 00:16:28,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +15: [2023-05-11 00:16:28,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +15: [2023-05-11 00:16:28,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +15: [2023-05-11 00:16:28,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +15: [2023-05-11 00:16:28,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +15: [2023-05-11 00:16:28,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +15: [2023-05-11 00:16:28,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +15: [2023-05-11 00:16:28,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +15: [2023-05-11 00:16:28,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +15: [2023-05-11 00:16:28,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +15: [2023-05-11 00:16:28,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +15: [2023-05-11 00:16:28,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +15: [2023-05-11 00:16:28,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +15: [2023-05-11 00:16:28,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +15: [2023-05-11 00:16:28,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +15: [2023-05-11 00:16:28,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 9: [2023-05-11 00:16:28,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 9: [2023-05-11 00:16:28,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 6: [2023-05-11 00:16:28,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 6: [2023-05-11 00:16:28,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 6: [2023-05-11 00:16:28,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 6: [2023-05-11 00:16:28,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +15: [2023-05-11 00:16:28,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +15: [2023-05-11 00:16:28,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +15: [2023-05-11 00:16:28,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +15: [2023-05-11 00:16:28,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +15: [2023-05-11 00:16:28,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +15: [2023-05-11 00:16:28,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +15: [2023-05-11 00:16:28,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +15: [2023-05-11 00:16:28,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +27: [2023-05-11 00:16:28,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +27: [2023-05-11 00:16:28,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +27: [2023-05-11 00:16:28,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +27: [2023-05-11 00:16:28,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +27: [2023-05-11 00:16:28,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +27: [2023-05-11 00:16:28,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +27: [2023-05-11 00:16:28,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +27: [2023-05-11 00:16:28,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +31: [2023-05-11 00:16:28,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +31: [2023-05-11 00:16:28,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +31: [2023-05-11 00:16:28,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +20: [2023-05-11 00:16:28,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +20: [2023-05-11 00:16:28,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +31: [2023-05-11 00:16:28,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +20: [2023-05-11 00:16:28,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +20: [2023-05-11 00:16:28,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +20: [2023-05-11 00:16:28,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +20: [2023-05-11 00:16:28,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +20: [2023-05-11 00:16:28,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +20: [2023-05-11 00:16:28,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +24: [2023-05-11 00:16:28,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +31: [2023-05-11 00:16:28,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +27: [2023-05-11 00:16:28,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +24: [2023-05-11 00:16:28,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +24: [2023-05-11 00:16:28,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +27: [2023-05-11 00:16:28,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +31: [2023-05-11 00:16:28,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +27: [2023-05-11 00:16:28,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +27: [2023-05-11 00:16:28,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +24: [2023-05-11 00:16:28,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +31: [2023-05-11 00:16:28,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +31: [2023-05-11 00:16:28,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +20: [2023-05-11 00:16:28,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +20: [2023-05-11 00:16:28,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +24: [2023-05-11 00:16:28,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +20: [2023-05-11 00:16:28,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +20: [2023-05-11 00:16:28,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +24: [2023-05-11 00:16:28,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +27: [2023-05-11 00:16:28,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 5: [2023-05-11 00:16:28,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 5: [2023-05-11 00:16:28,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 5: [2023-05-11 00:16:28,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +27: [2023-05-11 00:16:28,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +27: [2023-05-11 00:16:28,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +24: [2023-05-11 00:16:28,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +24: [2023-05-11 00:16:28,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +27: [2023-05-11 00:16:28,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +20: [2023-05-11 00:16:28,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +20: [2023-05-11 00:16:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +20: [2023-05-11 00:16:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +20: [2023-05-11 00:16:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +25: [2023-05-11 00:16:28,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +25: [2023-05-11 00:16:28,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +25: [2023-05-11 00:16:28,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +25: [2023-05-11 00:16:28,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +25: [2023-05-11 00:16:28,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +25: [2023-05-11 00:16:28,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 7: [2023-05-11 00:16:28,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 7: [2023-05-11 00:16:28,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 7: [2023-05-11 00:16:28,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +25: [2023-05-11 00:16:28,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +25: [2023-05-11 00:16:28,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 6: [2023-05-11 00:16:28,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 6: [2023-05-11 00:16:28,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 6: [2023-05-11 00:16:28,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +14: [2023-05-11 00:16:28,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +14: [2023-05-11 00:16:28,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +14: [2023-05-11 00:16:28,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +14: [2023-05-11 00:16:28,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +14: [2023-05-11 00:16:28,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +14: [2023-05-11 00:16:28,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +14: [2023-05-11 00:16:28,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +14: [2023-05-11 00:16:28,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +24: [2023-05-11 00:16:28,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +24: [2023-05-11 00:16:28,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +24: [2023-05-11 00:16:28,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +24: [2023-05-11 00:16:28,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +24: [2023-05-11 00:16:28,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +24: [2023-05-11 00:16:28,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +24: [2023-05-11 00:16:28,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +24: [2023-05-11 00:16:28,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +24: [2023-05-11 00:16:28,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +24: [2023-05-11 00:16:28,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +24: [2023-05-11 00:16:28,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +24: [2023-05-11 00:16:28,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +24: [2023-05-11 00:16:28,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +24: [2023-05-11 00:16:28,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +24: [2023-05-11 00:16:28,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 6: [2023-05-11 00:16:28,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +24: [2023-05-11 00:16:28,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 7: [2023-05-11 00:16:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +25: [2023-05-11 00:16:28,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +25: [2023-05-11 00:16:28,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 5: [2023-05-11 00:16:28,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 6: [2023-05-11 00:16:28,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +14: [2023-05-11 00:16:28,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +14: [2023-05-11 00:16:28,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +14: [2023-05-11 00:16:28,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +25: [2023-05-11 00:16:28,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +14: [2023-05-11 00:16:28,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +24: [2023-05-11 00:16:28,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +24: [2023-05-11 00:16:28,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +24: [2023-05-11 00:16:28,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +14: [2023-05-11 00:16:28,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +14: [2023-05-11 00:16:28,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +14: [2023-05-11 00:16:28,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +14: [2023-05-11 00:16:28,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 6: [2023-05-11 00:16:28,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +25: [2023-05-11 00:16:28,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +14: [2023-05-11 00:16:28,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +14: [2023-05-11 00:16:28,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +14: [2023-05-11 00:16:28,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +24: [2023-05-11 00:16:28,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +25: [2023-05-11 00:16:28,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +24: [2023-05-11 00:16:28,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +24: [2023-05-11 00:16:28,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +25: [2023-05-11 00:16:28,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +14: [2023-05-11 00:16:28,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +24: [2023-05-11 00:16:28,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +14: [2023-05-11 00:16:28,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +14: [2023-05-11 00:16:28,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +14: [2023-05-11 00:16:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +14: [2023-05-11 00:16:28,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +25: [2023-05-11 00:16:28,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +25: [2023-05-11 00:16:28,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +23: [2023-05-11 00:16:28,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +23: [2023-05-11 00:16:28,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +23: [2023-05-11 00:16:28,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +23: [2023-05-11 00:16:28,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +14: [2023-05-11 00:16:28,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +14: [2023-05-11 00:16:28,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +24: [2023-05-11 00:16:28,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +14: [2023-05-11 00:16:28,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +14: [2023-05-11 00:16:28,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +16: [2023-05-11 00:16:28,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +16: [2023-05-11 00:16:28,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +16: [2023-05-11 00:16:28,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +31: [2023-05-11 00:16:28,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +31: [2023-05-11 00:16:28,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +31: [2023-05-11 00:16:28,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +31: [2023-05-11 00:16:28,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +31: [2023-05-11 00:16:28,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +31: [2023-05-11 00:16:28,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +31: [2023-05-11 00:16:28,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +31: [2023-05-11 00:16:28,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +31: [2023-05-11 00:16:28,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +31: [2023-05-11 00:16:28,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +31: [2023-05-11 00:16:28,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +31: [2023-05-11 00:16:28,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +31: [2023-05-11 00:16:28,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +16: [2023-05-11 00:16:28,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +31: [2023-05-11 00:16:28,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +31: [2023-05-11 00:16:28,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +23: [2023-05-11 00:16:28,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +23: [2023-05-11 00:16:28,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +23: [2023-05-11 00:16:28,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +14: [2023-05-11 00:16:28,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +14: [2023-05-11 00:16:28,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +14: [2023-05-11 00:16:28,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +14: [2023-05-11 00:16:28,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +23: [2023-05-11 00:16:28,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +31: [2023-05-11 00:16:28,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +18: [2023-05-11 00:16:28,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +18: [2023-05-11 00:16:28,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +18: [2023-05-11 00:16:28,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +18: [2023-05-11 00:16:28,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +20: [2023-05-11 00:16:28,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +20: [2023-05-11 00:16:28,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +20: [2023-05-11 00:16:28,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +20: [2023-05-11 00:16:28,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +16: [2023-05-11 00:16:28,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +16: [2023-05-11 00:16:28,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +31: [2023-05-11 00:16:28,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +16: [2023-05-11 00:16:28,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +16: [2023-05-11 00:16:28,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +31: [2023-05-11 00:16:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +18: [2023-05-11 00:16:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +18: [2023-05-11 00:16:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +18: [2023-05-11 00:16:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +13: [2023-05-11 00:16:28,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +13: [2023-05-11 00:16:28,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +13: [2023-05-11 00:16:28,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +13: [2023-05-11 00:16:28,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +13: [2023-05-11 00:16:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +13: [2023-05-11 00:16:28,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +31: [2023-05-11 00:16:28,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +13: [2023-05-11 00:16:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +21: [2023-05-11 00:16:28,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +21: [2023-05-11 00:16:28,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +21: [2023-05-11 00:16:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +21: [2023-05-11 00:16:28,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 4: [2023-05-11 00:16:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +13: [2023-05-11 00:16:28,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +18: [2023-05-11 00:16:28,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +29: [2023-05-11 00:16:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +26: [2023-05-11 00:16:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +26: [2023-05-11 00:16:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +26: [2023-05-11 00:16:28,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +26: [2023-05-11 00:16:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +26: [2023-05-11 00:16:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +26: [2023-05-11 00:16:28,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +29: [2023-05-11 00:16:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +29: [2023-05-11 00:16:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +29: [2023-05-11 00:16:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +20: [2023-05-11 00:16:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +26: [2023-05-11 00:16:28,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +26: [2023-05-11 00:16:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +20: [2023-05-11 00:16:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +20: [2023-05-11 00:16:28,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +20: [2023-05-11 00:16:28,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +30: [2023-05-11 00:16:28,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +30: [2023-05-11 00:16:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +30: [2023-05-11 00:16:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +30: [2023-05-11 00:16:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 7: [2023-05-11 00:16:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 7: [2023-05-11 00:16:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +22: [2023-05-11 00:16:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +22: [2023-05-11 00:16:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 7: [2023-05-11 00:16:28,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +22: [2023-05-11 00:16:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +22: [2023-05-11 00:16:28,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +31: [2023-05-11 00:16:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 8: [2023-05-11 00:16:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 8: [2023-05-11 00:16:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 8: [2023-05-11 00:16:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 8: [2023-05-11 00:16:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 8: [2023-05-11 00:16:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 8: [2023-05-11 00:16:28,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 8: [2023-05-11 00:16:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 8: [2023-05-11 00:16:28,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +21: [2023-05-11 00:16:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +21: [2023-05-11 00:16:28,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +21: [2023-05-11 00:16:28,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +22: [2023-05-11 00:16:28,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +31: [2023-05-11 00:16:28,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +22: [2023-05-11 00:16:28,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +21: [2023-05-11 00:16:28,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +10: [2023-05-11 00:16:28,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +22: [2023-05-11 00:16:28,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +10: [2023-05-11 00:16:28,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +10: [2023-05-11 00:16:28,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +10: [2023-05-11 00:16:28,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 7: [2023-05-11 00:16:28,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +18: [2023-05-11 00:16:28,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +13: [2023-05-11 00:16:28,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +22: [2023-05-11 00:16:28,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +18: [2023-05-11 00:16:28,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +31: [2023-05-11 00:16:28,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +29: [2023-05-11 00:16:28,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +29: [2023-05-11 00:16:28,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +18: [2023-05-11 00:16:28,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +29: [2023-05-11 00:16:28,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +18: [2023-05-11 00:16:28,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 4: [2023-05-11 00:16:28,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +29: [2023-05-11 00:16:28,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +13: [2023-05-11 00:16:28,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +30: [2023-05-11 00:16:28,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +30: [2023-05-11 00:16:28,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +30: [2023-05-11 00:16:28,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +30: [2023-05-11 00:16:28,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +31: [2023-05-11 00:16:28,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +18: [2023-05-11 00:16:28,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +18: [2023-05-11 00:16:28,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 1: [2023-05-11 00:16:28,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +11: [2023-05-11 00:16:28,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +21: [2023-05-11 00:16:28,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +20: [2023-05-11 00:16:28,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +22: [2023-05-11 00:16:28,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 8: [2023-05-11 00:16:28,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +11: [2023-05-11 00:16:28,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +21: [2023-05-11 00:16:28,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +22: [2023-05-11 00:16:28,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +22: [2023-05-11 00:16:28,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +21: [2023-05-11 00:16:28,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +21: [2023-05-11 00:16:28,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +20: [2023-05-11 00:16:28,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +11: [2023-05-11 00:16:28,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +20: [2023-05-11 00:16:28,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +11: [2023-05-11 00:16:28,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +20: [2023-05-11 00:16:28,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +20: [2023-05-11 00:16:28,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +20: [2023-05-11 00:16:28,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +20: [2023-05-11 00:16:28,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +10: [2023-05-11 00:16:28,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +20: [2023-05-11 00:16:28,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +10: [2023-05-11 00:16:28,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 8: [2023-05-11 00:16:28,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 8: [2023-05-11 00:16:28,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +10: [2023-05-11 00:16:28,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +10: [2023-05-11 00:16:28,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 8: [2023-05-11 00:16:28,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +13: [2023-05-11 00:16:28,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 8: [2023-05-11 00:16:28,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 2: [2023-05-11 00:16:28,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 2: [2023-05-11 00:16:28,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 2: [2023-05-11 00:16:28,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 8: [2023-05-11 00:16:28,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 8: [2023-05-11 00:16:28,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +13: [2023-05-11 00:16:28,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 2: [2023-05-11 00:16:28,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +21: [2023-05-11 00:16:28,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +21: [2023-05-11 00:16:28,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +21: [2023-05-11 00:16:28,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +21: [2023-05-11 00:16:28,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 4: [2023-05-11 00:16:28,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +31: [2023-05-11 00:16:28,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +13: [2023-05-11 00:16:28,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +26: [2023-05-11 00:16:28,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +26: [2023-05-11 00:16:28,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +26: [2023-05-11 00:16:28,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +26: [2023-05-11 00:16:28,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +18: [2023-05-11 00:16:28,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +22: [2023-05-11 00:16:28,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +11: [2023-05-11 00:16:28,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +22: [2023-05-11 00:16:28,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 8: [2023-05-11 00:16:28,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +22: [2023-05-11 00:16:28,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +18: [2023-05-11 00:16:28,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 8: [2023-05-11 00:16:28,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +11: [2023-05-11 00:16:28,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +11: [2023-05-11 00:16:28,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 8: [2023-05-11 00:16:28,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +13: [2023-05-11 00:16:28,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +11: [2023-05-11 00:16:28,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 8: [2023-05-11 00:16:28,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 2: [2023-05-11 00:16:28,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 8: [2023-05-11 00:16:28,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 8: [2023-05-11 00:16:28,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 2: [2023-05-11 00:16:28,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 2: [2023-05-11 00:16:28,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +22: [2023-05-11 00:16:28,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 8: [2023-05-11 00:16:28,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 8: [2023-05-11 00:16:28,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 2: [2023-05-11 00:16:28,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 0: [2023-05-11 00:16:28,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 0: [2023-05-11 00:16:28,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 0: [2023-05-11 00:16:28,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 0: [2023-05-11 00:16:28,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 0: [2023-05-11 00:16:28,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +26: [2023-05-11 00:16:28,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 0: [2023-05-11 00:16:28,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +26: [2023-05-11 00:16:28,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +23: [2023-05-11 00:16:28,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 0: [2023-05-11 00:16:28,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +23: [2023-05-11 00:16:28,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 0: [2023-05-11 00:16:28,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +23: [2023-05-11 00:16:28,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +23: [2023-05-11 00:16:28,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +23: [2023-05-11 00:16:28,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +23: [2023-05-11 00:16:28,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +23: [2023-05-11 00:16:28,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +23: [2023-05-11 00:16:28,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +23: [2023-05-11 00:16:28,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +26: [2023-05-11 00:16:28,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +23: [2023-05-11 00:16:28,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +23: [2023-05-11 00:16:28,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +26: [2023-05-11 00:16:28,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +23: [2023-05-11 00:16:28,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +23: [2023-05-11 00:16:28,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +23: [2023-05-11 00:16:28,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +23: [2023-05-11 00:16:28,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +19: [2023-05-11 00:16:28,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +19: [2023-05-11 00:16:28,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +19: [2023-05-11 00:16:28,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +19: [2023-05-11 00:16:28,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +27: [2023-05-11 00:16:28,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +19: [2023-05-11 00:16:28,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +28: [2023-05-11 00:16:28,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +28: [2023-05-11 00:16:28,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 7: [2023-05-11 00:16:28,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +30: [2023-05-11 00:16:28,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +30: [2023-05-11 00:16:28,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +30: [2023-05-11 00:16:28,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +28: [2023-05-11 00:16:28,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +13: [2023-05-11 00:16:28,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +28: [2023-05-11 00:16:28,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +19: [2023-05-11 00:16:28,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +30: [2023-05-11 00:16:28,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +30: [2023-05-11 00:16:28,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +19: [2023-05-11 00:16:28,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +30: [2023-05-11 00:16:28,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +27: [2023-05-11 00:16:28,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +23: [2023-05-11 00:16:28,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +11: [2023-05-11 00:16:28,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +11: [2023-05-11 00:16:28,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +11: [2023-05-11 00:16:28,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +11: [2023-05-11 00:16:28,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 8: [2023-05-11 00:16:28,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +19: [2023-05-11 00:16:28,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +30: [2023-05-11 00:16:28,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +30: [2023-05-11 00:16:28,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +11: [2023-05-11 00:16:28,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +11: [2023-05-11 00:16:28,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +27: [2023-05-11 00:16:28,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +11: [2023-05-11 00:16:28,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +11: [2023-05-11 00:16:28,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +27: [2023-05-11 00:16:28,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +12: [2023-05-11 00:16:28,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +19: [2023-05-11 00:16:28,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +19: [2023-05-11 00:16:28,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +12: [2023-05-11 00:16:28,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +12: [2023-05-11 00:16:28,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +12: [2023-05-11 00:16:28,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +11: [2023-05-11 00:16:28,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +19: [2023-05-11 00:16:28,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +11: [2023-05-11 00:16:28,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +19: [2023-05-11 00:16:28,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +22: [2023-05-11 00:16:28,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +13: [2023-05-11 00:16:28,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +25: [2023-05-11 00:16:28,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +25: [2023-05-11 00:16:28,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +25: [2023-05-11 00:16:28,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 7: [2023-05-11 00:16:28,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +11: [2023-05-11 00:16:28,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +11: [2023-05-11 00:16:28,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +28: [2023-05-11 00:16:28,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +25: [2023-05-11 00:16:28,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 2: [2023-05-11 00:16:28,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 2: [2023-05-11 00:16:28,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 2: [2023-05-11 00:16:28,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 2: [2023-05-11 00:16:28,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 2: [2023-05-11 00:16:28,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 2: [2023-05-11 00:16:28,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +28: [2023-05-11 00:16:28,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +28: [2023-05-11 00:16:28,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +29: [2023-05-11 00:16:28,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +29: [2023-05-11 00:16:28,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +29: [2023-05-11 00:16:28,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +29: [2023-05-11 00:16:28,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +28: [2023-05-11 00:16:28,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 3: [2023-05-11 00:16:28,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +23: [2023-05-11 00:16:28,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +27: [2023-05-11 00:16:28,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +27: [2023-05-11 00:16:28,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +27: [2023-05-11 00:16:28,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +27: [2023-05-11 00:16:28,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +23: [2023-05-11 00:16:28,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +29: [2023-05-11 00:16:28,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +29: [2023-05-11 00:16:28,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +29: [2023-05-11 00:16:28,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +29: [2023-05-11 00:16:28,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +29: [2023-05-11 00:16:28,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +29: [2023-05-11 00:16:28,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 8: [2023-05-11 00:16:28,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 8: [2023-05-11 00:16:28,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +23: [2023-05-11 00:16:28,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +23: [2023-05-11 00:16:28,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +19: [2023-05-11 00:16:28,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 8: [2023-05-11 00:16:28,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 8: [2023-05-11 00:16:28,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 8: [2023-05-11 00:16:28,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 8: [2023-05-11 00:16:28,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 8: [2023-05-11 00:16:28,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 8: [2023-05-11 00:16:28,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +12: [2023-05-11 00:16:28,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +29: [2023-05-11 00:16:28,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +11: [2023-05-11 00:16:28,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +11: [2023-05-11 00:16:28,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 2: [2023-05-11 00:16:28,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +12: [2023-05-11 00:16:28,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +29: [2023-05-11 00:16:28,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 2: [2023-05-11 00:16:28,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +29: [2023-05-11 00:16:28,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +25: [2023-05-11 00:16:28,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +12: [2023-05-11 00:16:28,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +12: [2023-05-11 00:16:28,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +19: [2023-05-11 00:16:28,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +11: [2023-05-11 00:16:28,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +11: [2023-05-11 00:16:28,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +11: [2023-05-11 00:16:28,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +11: [2023-05-11 00:16:28,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +19: [2023-05-11 00:16:28,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +19: [2023-05-11 00:16:28,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +29: [2023-05-11 00:16:28,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +29: [2023-05-11 00:16:28,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +29: [2023-05-11 00:16:28,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +25: [2023-05-11 00:16:28,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +25: [2023-05-11 00:16:28,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +25: [2023-05-11 00:16:28,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 0: [2023-05-11 00:16:28,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 0: [2023-05-11 00:16:28,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 0: [2023-05-11 00:16:28,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 4: [2023-05-11 00:16:28,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 4: [2023-05-11 00:16:28,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 3: [2023-05-11 00:16:28,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 0: [2023-05-11 00:16:28,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +26: [2023-05-11 00:16:28,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +26: [2023-05-11 00:16:28,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +26: [2023-05-11 00:16:28,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +19: [2023-05-11 00:16:28,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 2: [2023-05-11 00:16:28,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +13: [2023-05-11 00:16:28,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +13: [2023-05-11 00:16:28,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +13: [2023-05-11 00:16:28,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +13: [2023-05-11 00:16:28,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +19: [2023-05-11 00:16:28,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +26: [2023-05-11 00:16:28,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 3: [2023-05-11 00:16:28,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +23: [2023-05-11 00:16:28,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +23: [2023-05-11 00:16:28,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +29: [2023-05-11 00:16:28,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 2: [2023-05-11 00:16:28,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 2: [2023-05-11 00:16:28,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 2: [2023-05-11 00:16:28,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 2: [2023-05-11 00:16:28,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +23: [2023-05-11 00:16:28,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 2: [2023-05-11 00:16:28,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 2: [2023-05-11 00:16:28,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +11: [2023-05-11 00:16:28,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +23: [2023-05-11 00:16:28,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +11: [2023-05-11 00:16:28,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +27: [2023-05-11 00:16:28,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +11: [2023-05-11 00:16:28,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +27: [2023-05-11 00:16:28,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +19: [2023-05-11 00:16:28,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +26: [2023-05-11 00:16:28,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +25: [2023-05-11 00:16:28,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +25: [2023-05-11 00:16:28,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +11: [2023-05-11 00:16:28,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +26: [2023-05-11 00:16:28,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +19: [2023-05-11 00:16:28,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 0: [2023-05-11 00:16:28,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 0: [2023-05-11 00:16:28,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 9: [2023-05-11 00:16:28,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 4: [2023-05-11 00:16:28,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 0: [2023-05-11 00:16:28,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 9: [2023-05-11 00:16:28,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 9: [2023-05-11 00:16:28,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +26: [2023-05-11 00:16:28,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +29: [2023-05-11 00:16:28,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +29: [2023-05-11 00:16:28,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +13: [2023-05-11 00:16:28,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +13: [2023-05-11 00:16:28,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 2: [2023-05-11 00:16:28,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 1: [2023-05-11 00:16:28,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 9: [2023-05-11 00:16:28,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 2: [2023-05-11 00:16:28,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 2: [2023-05-11 00:16:28,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +19: [2023-05-11 00:16:28,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 2: [2023-05-11 00:16:28,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +13: [2023-05-11 00:16:28,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +13: [2023-05-11 00:16:28,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +27: [2023-05-11 00:16:28,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +27: [2023-05-11 00:16:28,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +27: [2023-05-11 00:16:28,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +26: [2023-05-11 00:16:28,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +27: [2023-05-11 00:16:28,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +19: [2023-05-11 00:16:28,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 2: [2023-05-11 00:16:28,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +29: [2023-05-11 00:16:28,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +19: [2023-05-11 00:16:28,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 0: [2023-05-11 00:16:28,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 0: [2023-05-11 00:16:28,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 0: [2023-05-11 00:16:28,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +11: [2023-05-11 00:16:28,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +25: [2023-05-11 00:16:28,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 0: [2023-05-11 00:16:28,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +19: [2023-05-11 00:16:28,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +25: [2023-05-11 00:16:28,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +25: [2023-05-11 00:16:28,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +25: [2023-05-11 00:16:28,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 2: [2023-05-11 00:16:28,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 2: [2023-05-11 00:16:28,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +22: [2023-05-11 00:16:28,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +22: [2023-05-11 00:16:28,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +22: [2023-05-11 00:16:28,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +22: [2023-05-11 00:16:28,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +11: [2023-05-11 00:16:28,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 2: [2023-05-11 00:16:28,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 0: [2023-05-11 00:16:28,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 3: [2023-05-11 00:16:28,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 3: [2023-05-11 00:16:28,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 0: [2023-05-11 00:16:28,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 0: [2023-05-11 00:16:28,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 0: [2023-05-11 00:16:28,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +19: [2023-05-11 00:16:28,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +25: [2023-05-11 00:16:28,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +29: [2023-05-11 00:16:28,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +26: [2023-05-11 00:16:28,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +26: [2023-05-11 00:16:28,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +29: [2023-05-11 00:16:28,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +25: [2023-05-11 00:16:28,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 1: [2023-05-11 00:16:28,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +26: [2023-05-11 00:16:28,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +19: [2023-05-11 00:16:28,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +26: [2023-05-11 00:16:28,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 9: [2023-05-11 00:16:28,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +13: [2023-05-11 00:16:28,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +13: [2023-05-11 00:16:28,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 9: [2023-05-11 00:16:28,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +13: [2023-05-11 00:16:28,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +13: [2023-05-11 00:16:28,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 1: [2023-05-11 00:16:28,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +19: [2023-05-11 00:16:28,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +19: [2023-05-11 00:16:28,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +26: [2023-05-11 00:16:28,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +19: [2023-05-11 00:16:28,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +19: [2023-05-11 00:16:28,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +26: [2023-05-11 00:16:28,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 2: [2023-05-11 00:16:28,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +26: [2023-05-11 00:16:28,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +26: [2023-05-11 00:16:28,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +13: [2023-05-11 00:16:28,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +13: [2023-05-11 00:16:28,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +22: [2023-05-11 00:16:28,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 0: [2023-05-11 00:16:28,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +13: [2023-05-11 00:16:28,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +22: [2023-05-11 00:16:28,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +27: [2023-05-11 00:16:28,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +22: [2023-05-11 00:16:28,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +22: [2023-05-11 00:16:28,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +13: [2023-05-11 00:16:28,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +19: [2023-05-11 00:16:28,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +19: [2023-05-11 00:16:28,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 4: [2023-05-11 00:16:28,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +27: [2023-05-11 00:16:28,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 3: [2023-05-11 00:16:28,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 4: [2023-05-11 00:16:28,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 4: [2023-05-11 00:16:28,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 6: [2023-05-11 00:16:28,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 6: [2023-05-11 00:16:28,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 6: [2023-05-11 00:16:28,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 6: [2023-05-11 00:16:28,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +29: [2023-05-11 00:16:28,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +22: [2023-05-11 00:16:28,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +22: [2023-05-11 00:16:28,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 1: [2023-05-11 00:16:28,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 1: [2023-05-11 00:16:28,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 1: [2023-05-11 00:16:28,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 1: [2023-05-11 00:16:28,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +18: [2023-05-11 00:16:28,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +18: [2023-05-11 00:16:28,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +18: [2023-05-11 00:16:28,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +18: [2023-05-11 00:16:28,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 0: [2023-05-11 00:16:28,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 0: [2023-05-11 00:16:28,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 0: [2023-05-11 00:16:28,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 0: [2023-05-11 00:16:28,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +18: [2023-05-11 00:16:28,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 0: [2023-05-11 00:16:28,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +22: [2023-05-11 00:16:28,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +29: [2023-05-11 00:16:28,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +22: [2023-05-11 00:16:28,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 0: [2023-05-11 00:16:28,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 0: [2023-05-11 00:16:28,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 0: [2023-05-11 00:16:28,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +22: [2023-05-11 00:16:28,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +22: [2023-05-11 00:16:28,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 6: [2023-05-11 00:16:28,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +18: [2023-05-11 00:16:28,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +18: [2023-05-11 00:16:28,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +18: [2023-05-11 00:16:28,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +15: [2023-05-11 00:16:28,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +15: [2023-05-11 00:16:28,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +15: [2023-05-11 00:16:28,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +15: [2023-05-11 00:16:28,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 4: [2023-05-11 00:16:28,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 5: [2023-05-11 00:16:28,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +28: [2023-05-11 00:16:28,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +28: [2023-05-11 00:16:28,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +28: [2023-05-11 00:16:28,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +28: [2023-05-11 00:16:28,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +28: [2023-05-11 00:16:28,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +28: [2023-05-11 00:16:28,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +28: [2023-05-11 00:16:28,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +28: [2023-05-11 00:16:28,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +28: [2023-05-11 00:16:28,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +28: [2023-05-11 00:16:28,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +16: [2023-05-11 00:16:28,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +16: [2023-05-11 00:16:28,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +16: [2023-05-11 00:16:28,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +16: [2023-05-11 00:16:28,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +16: [2023-05-11 00:16:28,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +16: [2023-05-11 00:16:28,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +16: [2023-05-11 00:16:28,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +16: [2023-05-11 00:16:28,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +16: [2023-05-11 00:16:28,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +16: [2023-05-11 00:16:28,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +16: [2023-05-11 00:16:28,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +28: [2023-05-11 00:16:28,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +28: [2023-05-11 00:16:28,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +16: [2023-05-11 00:16:28,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +28: [2023-05-11 00:16:28,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +28: [2023-05-11 00:16:28,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +28: [2023-05-11 00:16:28,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +16: [2023-05-11 00:16:28,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +16: [2023-05-11 00:16:28,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +16: [2023-05-11 00:16:28,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +16: [2023-05-11 00:16:28,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +28: [2023-05-11 00:16:28,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +18: [2023-05-11 00:16:28,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +22: [2023-05-11 00:16:28,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +21: [2023-05-11 00:16:28,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +21: [2023-05-11 00:16:28,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +21: [2023-05-11 00:16:28,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +21: [2023-05-11 00:16:28,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +21: [2023-05-11 00:16:28,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +18: [2023-05-11 00:16:28,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +22: [2023-05-11 00:16:28,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +15: [2023-05-11 00:16:28,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +15: [2023-05-11 00:16:28,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +15: [2023-05-11 00:16:28,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +21: [2023-05-11 00:16:28,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +21: [2023-05-11 00:16:28,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +21: [2023-05-11 00:16:28,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +24: [2023-05-11 00:16:28,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +24: [2023-05-11 00:16:28,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +15: [2023-05-11 00:16:28,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +24: [2023-05-11 00:16:28,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +24: [2023-05-11 00:16:28,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +18: [2023-05-11 00:16:28,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +18: [2023-05-11 00:16:28,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +16: [2023-05-11 00:16:28,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +28: [2023-05-11 00:16:28,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +28: [2023-05-11 00:16:28,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +18: [2023-05-11 00:16:28,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +24: [2023-05-11 00:16:28,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +24: [2023-05-11 00:16:28,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +18: [2023-05-11 00:16:28,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +24: [2023-05-11 00:16:28,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +18: [2023-05-11 00:16:28,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +24: [2023-05-11 00:16:28,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +21: [2023-05-11 00:16:28,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +16: [2023-05-11 00:16:28,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +28: [2023-05-11 00:16:28,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +28: [2023-05-11 00:16:28,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +16: [2023-05-11 00:16:28,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +18: [2023-05-11 00:16:28,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +14: [2023-05-11 00:16:28,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +14: [2023-05-11 00:16:28,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +14: [2023-05-11 00:16:28,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +14: [2023-05-11 00:16:28,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +28: [2023-05-11 00:16:28,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +21: [2023-05-11 00:16:28,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +21: [2023-05-11 00:16:28,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +16: [2023-05-11 00:16:28,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +16: [2023-05-11 00:16:28,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +28: [2023-05-11 00:16:28,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +21: [2023-05-11 00:16:28,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +21: [2023-05-11 00:16:28,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +16: [2023-05-11 00:16:28,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +28: [2023-05-11 00:16:28,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +14: [2023-05-11 00:16:28,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +14: [2023-05-11 00:16:28,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +14: [2023-05-11 00:16:28,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +16: [2023-05-11 00:16:28,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +16: [2023-05-11 00:16:28,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +14: [2023-05-11 00:16:28,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +21: [2023-05-11 00:16:28,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 5: [2023-05-11 00:16:28,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 5: [2023-05-11 00:16:28,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 5: [2023-05-11 00:16:28,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +28: [2023-05-11 00:16:28,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +21: [2023-05-11 00:16:28,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +21: [2023-05-11 00:16:28,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +12: [2023-05-11 00:16:28,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +12: [2023-05-11 00:16:28,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +12: [2023-05-11 00:16:28,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +12: [2023-05-11 00:16:28,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +12: [2023-05-11 00:16:28,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +12: [2023-05-11 00:16:28,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +12: [2023-05-11 00:16:28,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +12: [2023-05-11 00:16:28,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +12: [2023-05-11 00:16:28,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +12: [2023-05-11 00:16:28,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +12: [2023-05-11 00:16:28,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +12: [2023-05-11 00:16:28,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +12: [2023-05-11 00:16:28,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +12: [2023-05-11 00:16:28,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +12: [2023-05-11 00:16:28,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +12: [2023-05-11 00:16:28,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +18: [2023-05-11 00:16:28,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +18: [2023-05-11 00:16:28,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 5: [2023-05-11 00:16:28,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 5: [2023-05-11 00:16:28,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +18: [2023-05-11 00:16:28,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +18: [2023-05-11 00:16:28,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +12: [2023-05-11 00:16:28,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +18: [2023-05-11 00:16:28,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +18: [2023-05-11 00:16:28,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +18: [2023-05-11 00:16:28,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +31: [2023-05-11 00:16:28,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +31: [2023-05-11 00:16:28,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +31: [2023-05-11 00:16:28,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +31: [2023-05-11 00:16:28,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +12: [2023-05-11 00:16:28,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +18: [2023-05-11 00:16:28,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +12: [2023-05-11 00:16:28,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +31: [2023-05-11 00:16:28,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +12: [2023-05-11 00:16:28,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +12: [2023-05-11 00:16:28,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +31: [2023-05-11 00:16:28,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +31: [2023-05-11 00:16:28,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +18: [2023-05-11 00:16:28,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +18: [2023-05-11 00:16:28,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +18: [2023-05-11 00:16:28,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +18: [2023-05-11 00:16:28,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +31: [2023-05-11 00:16:28,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 7: [2023-05-11 00:16:28,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 7: [2023-05-11 00:16:28,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 7: [2023-05-11 00:16:28,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 3: [2023-05-11 00:16:28,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +12: [2023-05-11 00:16:28,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +12: [2023-05-11 00:16:28,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 9: [2023-05-11 00:16:28,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 9: [2023-05-11 00:16:28,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +18: [2023-05-11 00:16:28,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +17: [2023-05-11 00:16:28,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +18: [2023-05-11 00:16:28,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +17: [2023-05-11 00:16:28,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +17: [2023-05-11 00:16:28,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +17: [2023-05-11 00:16:28,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 7: [2023-05-11 00:16:28,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +18: [2023-05-11 00:16:28,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +18: [2023-05-11 00:16:28,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 9: [2023-05-11 00:16:28,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +17: [2023-05-11 00:16:28,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +17: [2023-05-11 00:16:28,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +17: [2023-05-11 00:16:28,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +17: [2023-05-11 00:16:28,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +17: [2023-05-11 00:16:28,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +17: [2023-05-11 00:16:28,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... + 9: [2023-05-11 00:16:28,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +17: [2023-05-11 00:16:28,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +17: [2023-05-11 00:16:28,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +10: [2023-05-11 00:16:28,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +10: [2023-05-11 00:16:28,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +10: [2023-05-11 00:16:28,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +10: [2023-05-11 00:16:28,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +17: [2023-05-11 00:16:28,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +17: [2023-05-11 00:16:28,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +10: [2023-05-11 00:16:28,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +10: [2023-05-11 00:16:28,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +10: [2023-05-11 00:16:28,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +17: [2023-05-11 00:16:28,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt... +12: [2023-05-11 00:16:28,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +10: [2023-05-11 00:16:28,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +10: [2023-05-11 00:16:28,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +10: [2023-05-11 00:16:28,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +17: [2023-05-11 00:16:28,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt... +10: [2023-05-11 00:16:28,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +10: [2023-05-11 00:16:28,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +10: [2023-05-11 00:16:28,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +10: [2023-05-11 00:16:28,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +10: [2023-05-11 00:16:28,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +10: [2023-05-11 00:16:28,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +30: [2023-05-11 00:16:28,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +30: [2023-05-11 00:16:28,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +30: [2023-05-11 00:16:28,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +30: [2023-05-11 00:16:28,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +10: [2023-05-11 00:16:28,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +10: [2023-05-11 00:16:28,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +17: [2023-05-11 00:16:28,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +17: [2023-05-11 00:16:28,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. +17: [2023-05-11 00:16:28,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +21: [2023-05-11 00:16:28,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +21: [2023-05-11 00:16:28,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +21: [2023-05-11 00:16:28,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +21: [2023-05-11 00:16:28,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 9: [2023-05-11 00:16:28,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +10: [2023-05-11 00:16:28,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +10: [2023-05-11 00:16:28,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +30: [2023-05-11 00:16:28,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +30: [2023-05-11 00:16:28,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +30: [2023-05-11 00:16:28,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +30: [2023-05-11 00:16:28,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +17: [2023-05-11 00:16:28,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +17: [2023-05-11 00:16:28,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 8: [2023-05-11 00:16:28,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 8: [2023-05-11 00:16:28,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 8: [2023-05-11 00:16:28,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 8: [2023-05-11 00:16:28,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 9: [2023-05-11 00:16:28,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +17: [2023-05-11 00:16:28,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +21: [2023-05-11 00:16:28,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +21: [2023-05-11 00:16:28,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +21: [2023-05-11 00:16:28,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +21: [2023-05-11 00:16:28,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +17: [2023-05-11 00:16:28,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +10: [2023-05-11 00:16:28,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +10: [2023-05-11 00:16:28,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 8: [2023-05-11 00:16:28,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 8: [2023-05-11 00:16:28,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 8: [2023-05-11 00:16:28,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +15: [2023-05-11 00:16:28,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +15: [2023-05-11 00:16:28,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +15: [2023-05-11 00:16:28,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 8: [2023-05-11 00:16:28,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +30: [2023-05-11 00:16:28,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +30: [2023-05-11 00:16:28,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +15: [2023-05-11 00:16:28,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +20: [2023-05-11 00:16:28,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +20: [2023-05-11 00:16:28,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +20: [2023-05-11 00:16:28,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +20: [2023-05-11 00:16:28,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +23: [2023-05-11 00:16:28,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +23: [2023-05-11 00:16:28,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +17: [2023-05-11 00:16:28,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +23: [2023-05-11 00:16:28,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +23: [2023-05-11 00:16:28,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +30: [2023-05-11 00:16:28,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +15: [2023-05-11 00:16:28,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +15: [2023-05-11 00:16:28,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +15: [2023-05-11 00:16:28,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +15: [2023-05-11 00:16:28,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +15: [2023-05-11 00:16:28,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +15: [2023-05-11 00:16:28,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +30: [2023-05-11 00:16:28,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +21: [2023-05-11 00:16:28,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +15: [2023-05-11 00:16:28,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +21: [2023-05-11 00:16:28,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +15: [2023-05-11 00:16:28,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +30: [2023-05-11 00:16:28,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +10: [2023-05-11 00:16:28,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +15: [2023-05-11 00:16:28,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +10: [2023-05-11 00:16:28,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +15: [2023-05-11 00:16:28,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +15: [2023-05-11 00:16:28,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +30: [2023-05-11 00:16:28,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +15: [2023-05-11 00:16:28,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +30: [2023-05-11 00:16:28,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +30: [2023-05-11 00:16:28,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +27: [2023-05-11 00:16:28,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +27: [2023-05-11 00:16:28,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +27: [2023-05-11 00:16:28,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +27: [2023-05-11 00:16:28,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +20: [2023-05-11 00:16:28,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 2: [2023-05-11 00:16:28,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 2: [2023-05-11 00:16:28,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 2: [2023-05-11 00:16:28,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +30: [2023-05-11 00:16:28,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +30: [2023-05-11 00:16:28,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +30: [2023-05-11 00:16:28,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +30: [2023-05-11 00:16:28,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 2: [2023-05-11 00:16:28,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +20: [2023-05-11 00:16:28,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +20: [2023-05-11 00:16:28,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +23: [2023-05-11 00:16:28,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +20: [2023-05-11 00:16:28,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +15: [2023-05-11 00:16:28,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +23: [2023-05-11 00:16:28,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +30: [2023-05-11 00:16:28,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +23: [2023-05-11 00:16:28,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +23: [2023-05-11 00:16:28,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +30: [2023-05-11 00:16:28,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +30: [2023-05-11 00:16:28,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +29: [2023-05-11 00:16:28,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +29: [2023-05-11 00:16:28,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +29: [2023-05-11 00:16:28,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +27: [2023-05-11 00:16:28,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +29: [2023-05-11 00:16:28,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +21: [2023-05-11 00:16:28,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +21: [2023-05-11 00:16:28,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +27: [2023-05-11 00:16:28,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +21: [2023-05-11 00:16:28,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +27: [2023-05-11 00:16:28,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +21: [2023-05-11 00:16:28,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 2: [2023-05-11 00:16:28,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +27: [2023-05-11 00:16:28,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 2: [2023-05-11 00:16:28,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +21: [2023-05-11 00:16:28,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +15: [2023-05-11 00:16:28,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +30: [2023-05-11 00:16:28,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 2: [2023-05-11 00:16:28,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 2: [2023-05-11 00:16:28,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +21: [2023-05-11 00:16:28,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +30: [2023-05-11 00:16:28,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +16: [2023-05-11 00:16:28,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +16: [2023-05-11 00:16:28,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +30: [2023-05-11 00:16:28,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +16: [2023-05-11 00:16:28,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +16: [2023-05-11 00:16:28,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +30: [2023-05-11 00:16:28,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +30: [2023-05-11 00:16:28,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +29: [2023-05-11 00:16:28,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +29: [2023-05-11 00:16:28,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +15: [2023-05-11 00:16:28,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +29: [2023-05-11 00:16:28,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +29: [2023-05-11 00:16:28,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +13: [2023-05-11 00:16:28,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +13: [2023-05-11 00:16:28,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +13: [2023-05-11 00:16:28,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +13: [2023-05-11 00:16:28,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +15: [2023-05-11 00:16:28,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +30: [2023-05-11 00:16:28,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +16: [2023-05-11 00:16:28,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +16: [2023-05-11 00:16:28,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +15: [2023-05-11 00:16:28,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +30: [2023-05-11 00:16:28,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +16: [2023-05-11 00:16:28,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +16: [2023-05-11 00:16:28,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +30: [2023-05-11 00:16:28,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +19: [2023-05-11 00:16:28,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +19: [2023-05-11 00:16:28,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +19: [2023-05-11 00:16:28,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +19: [2023-05-11 00:16:28,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +15: [2023-05-11 00:16:28,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 4: [2023-05-11 00:16:28,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 4: [2023-05-11 00:16:28,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +13: [2023-05-11 00:16:28,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +13: [2023-05-11 00:16:28,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +15: [2023-05-11 00:16:28,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +30: [2023-05-11 00:16:28,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +13: [2023-05-11 00:16:28,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +13: [2023-05-11 00:16:28,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +15: [2023-05-11 00:16:28,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 3: [2023-05-11 00:16:28,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 3: [2023-05-11 00:16:28,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 3: [2023-05-11 00:16:28,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +26: [2023-05-11 00:16:28,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +26: [2023-05-11 00:16:28,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +26: [2023-05-11 00:16:28,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +26: [2023-05-11 00:16:28,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +19: [2023-05-11 00:16:28,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +16: [2023-05-11 00:16:28,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +16: [2023-05-11 00:16:28,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +16: [2023-05-11 00:16:28,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +16: [2023-05-11 00:16:28,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +19: [2023-05-11 00:16:28,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +19: [2023-05-11 00:16:28,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +16: [2023-05-11 00:16:28,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +19: [2023-05-11 00:16:28,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +16: [2023-05-11 00:16:28,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +16: [2023-05-11 00:16:28,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +11: [2023-05-11 00:16:28,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +11: [2023-05-11 00:16:28,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +16: [2023-05-11 00:16:28,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +11: [2023-05-11 00:16:28,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +11: [2023-05-11 00:16:28,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +16: [2023-05-11 00:16:28,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +16: [2023-05-11 00:16:28,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +16: [2023-05-11 00:16:28,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +16: [2023-05-11 00:16:28,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 0: [2023-05-11 00:16:28,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +25: [2023-05-11 00:16:28,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +25: [2023-05-11 00:16:28,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +25: [2023-05-11 00:16:28,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 0: [2023-05-11 00:16:28,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 0: [2023-05-11 00:16:28,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 0: [2023-05-11 00:16:28,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +16: [2023-05-11 00:16:28,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +16: [2023-05-11 00:16:28,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +16: [2023-05-11 00:16:28,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +16: [2023-05-11 00:16:28,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 4: [2023-05-11 00:16:28,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +25: [2023-05-11 00:16:28,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 4: [2023-05-11 00:16:28,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +26: [2023-05-11 00:16:28,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +11: [2023-05-11 00:16:28,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +26: [2023-05-11 00:16:28,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +26: [2023-05-11 00:16:28,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +28: [2023-05-11 00:16:28,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +11: [2023-05-11 00:16:28,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +26: [2023-05-11 00:16:28,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +11: [2023-05-11 00:16:28,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +25: [2023-05-11 00:16:28,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +11: [2023-05-11 00:16:28,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +28: [2023-05-11 00:16:28,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 0: [2023-05-11 00:16:28,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 0: [2023-05-11 00:16:28,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +16: [2023-05-11 00:16:28,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 0: [2023-05-11 00:16:28,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +25: [2023-05-11 00:16:28,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 0: [2023-05-11 00:16:28,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +25: [2023-05-11 00:16:28,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +16: [2023-05-11 00:16:28,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +28: [2023-05-11 00:16:28,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +28: [2023-05-11 00:16:28,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +25: [2023-05-11 00:16:28,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +16: [2023-05-11 00:16:28,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +16: [2023-05-11 00:16:28,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +28: [2023-05-11 00:16:28,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 1: [2023-05-11 00:16:28,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 1: [2023-05-11 00:16:28,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 1: [2023-05-11 00:16:28,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +20: [2023-05-11 00:16:28,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +20: [2023-05-11 00:16:28,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +22: [2023-05-11 00:16:28,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +22: [2023-05-11 00:16:28,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +22: [2023-05-11 00:16:28,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +28: [2023-05-11 00:16:28,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +20: [2023-05-11 00:16:28,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +20: [2023-05-11 00:16:28,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +14: [2023-05-11 00:16:28,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +14: [2023-05-11 00:16:28,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +14: [2023-05-11 00:16:28,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +14: [2023-05-11 00:16:28,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +20: [2023-05-11 00:16:28,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +20: [2023-05-11 00:16:28,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +20: [2023-05-11 00:16:28,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +20: [2023-05-11 00:16:28,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +20: [2023-05-11 00:16:28,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +20: [2023-05-11 00:16:28,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +14: [2023-05-11 00:16:28,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +14: [2023-05-11 00:16:28,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +14: [2023-05-11 00:16:28,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +14: [2023-05-11 00:16:28,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +14: [2023-05-11 00:16:28,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +14: [2023-05-11 00:16:28,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +14: [2023-05-11 00:16:28,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +14: [2023-05-11 00:16:28,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +14: [2023-05-11 00:16:28,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +22: [2023-05-11 00:16:28,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. + 3: [2023-05-11 00:16:28,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +20: [2023-05-11 00:16:28,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +12: [2023-05-11 00:16:28,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +12: [2023-05-11 00:16:28,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +14: [2023-05-11 00:16:28,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +20: [2023-05-11 00:16:28,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +20: [2023-05-11 00:16:28,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +20: [2023-05-11 00:16:28,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +28: [2023-05-11 00:16:28,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +20: [2023-05-11 00:16:28,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +20: [2023-05-11 00:16:28,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +28: [2023-05-11 00:16:28,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +14: [2023-05-11 00:16:28,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +14: [2023-05-11 00:16:28,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +16: [2023-05-11 00:16:28,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +16: [2023-05-11 00:16:28,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +12: [2023-05-11 00:16:28,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +16: [2023-05-11 00:16:28,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +12: [2023-05-11 00:16:28,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 3: [2023-05-11 00:16:28,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +16: [2023-05-11 00:16:28,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 5: [2023-05-11 00:16:28,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 5: [2023-05-11 00:16:28,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +22: [2023-05-11 00:16:28,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +22: [2023-05-11 00:16:28,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +14: [2023-05-11 00:16:28,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +12: [2023-05-11 00:16:28,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +22: [2023-05-11 00:16:28,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +22: [2023-05-11 00:16:28,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +12: [2023-05-11 00:16:28,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +25: [2023-05-11 00:16:28,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +25: [2023-05-11 00:16:28,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +25: [2023-05-11 00:16:28,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +25: [2023-05-11 00:16:28,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +25: [2023-05-11 00:16:28,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +25: [2023-05-11 00:16:28,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +25: [2023-05-11 00:16:28,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +25: [2023-05-11 00:16:28,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 3: [2023-05-11 00:16:28,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +18: [2023-05-11 00:16:28,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +25: [2023-05-11 00:16:28,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +25: [2023-05-11 00:16:28,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +29: [2023-05-11 00:16:28,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +29: [2023-05-11 00:16:28,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 1: [2023-05-11 00:16:28,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +25: [2023-05-11 00:16:28,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +25: [2023-05-11 00:16:28,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +29: [2023-05-11 00:16:28,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +29: [2023-05-11 00:16:28,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +25: [2023-05-11 00:16:28,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +18: [2023-05-11 00:16:28,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +12: [2023-05-11 00:16:28,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +29: [2023-05-11 00:16:28,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +29: [2023-05-11 00:16:28,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 3: [2023-05-11 00:16:28,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +29: [2023-05-11 00:16:28,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +29: [2023-05-11 00:16:28,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +29: [2023-05-11 00:16:28,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +29: [2023-05-11 00:16:28,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +12: [2023-05-11 00:16:28,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +29: [2023-05-11 00:16:28,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +29: [2023-05-11 00:16:28,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +29: [2023-05-11 00:16:28,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +29: [2023-05-11 00:16:28,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +29: [2023-05-11 00:16:28,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +29: [2023-05-11 00:16:28,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +18: [2023-05-11 00:16:28,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +25: [2023-05-11 00:16:28,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +18: [2023-05-11 00:16:28,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +14: [2023-05-11 00:16:28,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +14: [2023-05-11 00:16:28,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +18: [2023-05-11 00:16:28,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +18: [2023-05-11 00:16:28,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +20: [2023-05-11 00:16:28,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +20: [2023-05-11 00:16:28,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +25: [2023-05-11 00:16:28,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +25: [2023-05-11 00:16:28,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +14: [2023-05-11 00:16:28,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +25: [2023-05-11 00:16:28,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +14: [2023-05-11 00:16:28,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +18: [2023-05-11 00:16:28,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +18: [2023-05-11 00:16:28,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +20: [2023-05-11 00:16:28,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +20: [2023-05-11 00:16:28,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +25: [2023-05-11 00:16:28,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +29: [2023-05-11 00:16:28,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +14: [2023-05-11 00:16:28,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +17: [2023-05-11 00:16:28,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +17: [2023-05-11 00:16:28,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +17: [2023-05-11 00:16:28,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. + 5: [2023-05-11 00:16:28,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 6: [2023-05-11 00:16:28,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 6: [2023-05-11 00:16:28,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +20: [2023-05-11 00:16:28,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +20: [2023-05-11 00:16:28,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +25: [2023-05-11 00:16:28,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +14: [2023-05-11 00:16:28,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +17: [2023-05-11 00:16:28,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt. +29: [2023-05-11 00:16:28,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +29: [2023-05-11 00:16:28,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +14: [2023-05-11 00:16:28,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +29: [2023-05-11 00:16:28,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +20: [2023-05-11 00:16:28,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +20: [2023-05-11 00:16:28,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +12: [2023-05-11 00:16:28,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +12: [2023-05-11 00:16:28,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +12: [2023-05-11 00:16:28,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +12: [2023-05-11 00:16:28,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +12: [2023-05-11 00:16:28,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +12: [2023-05-11 00:16:28,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +12: [2023-05-11 00:16:28,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +12: [2023-05-11 00:16:28,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +12: [2023-05-11 00:16:28,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +12: [2023-05-11 00:16:28,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +21: [2023-05-11 00:16:28,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +21: [2023-05-11 00:16:28,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +29: [2023-05-11 00:16:28,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +12: [2023-05-11 00:16:28,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +12: [2023-05-11 00:16:28,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +21: [2023-05-11 00:16:28,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +21: [2023-05-11 00:16:28,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +21: [2023-05-11 00:16:28,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +21: [2023-05-11 00:16:28,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +25: [2023-05-11 00:16:28,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +12: [2023-05-11 00:16:28,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +12: [2023-05-11 00:16:28,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +12: [2023-05-11 00:16:28,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +12: [2023-05-11 00:16:28,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +21: [2023-05-11 00:16:28,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +21: [2023-05-11 00:16:28,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +25: [2023-05-11 00:16:28,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +17: [2023-05-11 00:16:28,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +17: [2023-05-11 00:16:28,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 5: [2023-05-11 00:16:28,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 9: [2023-05-11 00:16:28,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +17: [2023-05-11 00:16:28,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +17: [2023-05-11 00:16:28,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 9: [2023-05-11 00:16:28,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 9: [2023-05-11 00:16:28,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 9: [2023-05-11 00:16:28,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 6: [2023-05-11 00:16:28,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +25: [2023-05-11 00:16:28,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +25: [2023-05-11 00:16:28,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +10: [2023-05-11 00:16:28,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +10: [2023-05-11 00:16:28,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +10: [2023-05-11 00:16:28,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +29: [2023-05-11 00:16:28,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +10: [2023-05-11 00:16:28,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 5: [2023-05-11 00:16:28,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 9: [2023-05-11 00:16:28,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +29: [2023-05-11 00:16:28,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +29: [2023-05-11 00:16:28,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +12: [2023-05-11 00:16:28,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +21: [2023-05-11 00:16:28,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +25: [2023-05-11 00:16:28,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +10: [2023-05-11 00:16:28,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +10: [2023-05-11 00:16:28,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +12: [2023-05-11 00:16:28,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +12: [2023-05-11 00:16:28,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +12: [2023-05-11 00:16:28,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +21: [2023-05-11 00:16:28,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +21: [2023-05-11 00:16:28,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +10: [2023-05-11 00:16:28,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +10: [2023-05-11 00:16:28,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +21: [2023-05-11 00:16:28,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +12: [2023-05-11 00:16:28,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +21: [2023-05-11 00:16:28,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +12: [2023-05-11 00:16:28,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +12: [2023-05-11 00:16:28,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +12: [2023-05-11 00:16:28,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +21: [2023-05-11 00:16:28,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +21: [2023-05-11 00:16:28,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +21: [2023-05-11 00:16:28,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 3: [2023-05-11 00:16:28,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +17: [2023-05-11 00:16:28,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +17: [2023-05-11 00:16:28,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +17: [2023-05-11 00:16:28,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +17: [2023-05-11 00:16:28,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +17: [2023-05-11 00:16:28,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +17: [2023-05-11 00:16:28,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +17: [2023-05-11 00:16:28,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +17: [2023-05-11 00:16:28,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... +17: [2023-05-11 00:16:28,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +17: [2023-05-11 00:16:28,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... + 9: [2023-05-11 00:16:28,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +17: [2023-05-11 00:16:28,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +17: [2023-05-11 00:16:28,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +17: [2023-05-11 00:16:28,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +17: [2023-05-11 00:16:28,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt... +28: [2023-05-11 00:16:28,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +28: [2023-05-11 00:16:28,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +28: [2023-05-11 00:16:28,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +28: [2023-05-11 00:16:28,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +28: [2023-05-11 00:16:28,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +28: [2023-05-11 00:16:28,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +17: [2023-05-11 00:16:28,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +28: [2023-05-11 00:16:28,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +28: [2023-05-11 00:16:28,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +28: [2023-05-11 00:16:28,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +28: [2023-05-11 00:16:28,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +28: [2023-05-11 00:16:28,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +28: [2023-05-11 00:16:28,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +28: [2023-05-11 00:16:28,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +28: [2023-05-11 00:16:28,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +28: [2023-05-11 00:16:28,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +28: [2023-05-11 00:16:28,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 6: [2023-05-11 00:16:28,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +17: [2023-05-11 00:16:28,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +18: [2023-05-11 00:16:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +18: [2023-05-11 00:16:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +18: [2023-05-11 00:16:28,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +18: [2023-05-11 00:16:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +18: [2023-05-11 00:16:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +18: [2023-05-11 00:16:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +18: [2023-05-11 00:16:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +18: [2023-05-11 00:16:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +18: [2023-05-11 00:16:28,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +18: [2023-05-11 00:16:28,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +18: [2023-05-11 00:16:28,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 9: [2023-05-11 00:16:28,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 1: [2023-05-11 00:16:28,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +18: [2023-05-11 00:16:28,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +18: [2023-05-11 00:16:28,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +18: [2023-05-11 00:16:28,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +18: [2023-05-11 00:16:28,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 1: [2023-05-11 00:16:28,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +18: [2023-05-11 00:16:28,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 1: [2023-05-11 00:16:28,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 1: [2023-05-11 00:16:28,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 1: [2023-05-11 00:16:28,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 1: [2023-05-11 00:16:28,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +19: [2023-05-11 00:16:28,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +19: [2023-05-11 00:16:28,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +19: [2023-05-11 00:16:28,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +19: [2023-05-11 00:16:28,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +19: [2023-05-11 00:16:28,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +19: [2023-05-11 00:16:28,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +19: [2023-05-11 00:16:28,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +19: [2023-05-11 00:16:28,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +19: [2023-05-11 00:16:28,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +19: [2023-05-11 00:16:28,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +19: [2023-05-11 00:16:28,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +19: [2023-05-11 00:16:28,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +19: [2023-05-11 00:16:28,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +19: [2023-05-11 00:16:28,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +17: [2023-05-11 00:16:28,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +19: [2023-05-11 00:16:28,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +19: [2023-05-11 00:16:28,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +21: [2023-05-11 00:16:28,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +10: [2023-05-11 00:16:28,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +10: [2023-05-11 00:16:28,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +10: [2023-05-11 00:16:28,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +10: [2023-05-11 00:16:28,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +10: [2023-05-11 00:16:28,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +10: [2023-05-11 00:16:28,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +10: [2023-05-11 00:16:28,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +10: [2023-05-11 00:16:28,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +10: [2023-05-11 00:16:28,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +10: [2023-05-11 00:16:28,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +30: [2023-05-11 00:16:28,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +30: [2023-05-11 00:16:28,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +30: [2023-05-11 00:16:28,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +30: [2023-05-11 00:16:28,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +30: [2023-05-11 00:16:28,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +17: [2023-05-11 00:16:28,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +17: [2023-05-11 00:16:28,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +30: [2023-05-11 00:16:28,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +30: [2023-05-11 00:16:28,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +30: [2023-05-11 00:16:28,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +28: [2023-05-11 00:16:28,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +28: [2023-05-11 00:16:28,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +10: [2023-05-11 00:16:28,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +10: [2023-05-11 00:16:28,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +10: [2023-05-11 00:16:28,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +10: [2023-05-11 00:16:28,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +10: [2023-05-11 00:16:28,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +10: [2023-05-11 00:16:28,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +21: [2023-05-11 00:16:28,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 1: [2023-05-11 00:16:28,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +21: [2023-05-11 00:16:28,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +21: [2023-05-11 00:16:28,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +28: [2023-05-11 00:16:28,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +28: [2023-05-11 00:16:28,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +27: [2023-05-11 00:16:28,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +27: [2023-05-11 00:16:28,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +27: [2023-05-11 00:16:28,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +27: [2023-05-11 00:16:28,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +27: [2023-05-11 00:16:28,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +27: [2023-05-11 00:16:28,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +31: [2023-05-11 00:16:28,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +31: [2023-05-11 00:16:28,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +27: [2023-05-11 00:16:28,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +27: [2023-05-11 00:16:28,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +24: [2023-05-11 00:16:28,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +31: [2023-05-11 00:16:28,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +24: [2023-05-11 00:16:28,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +24: [2023-05-11 00:16:28,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +24: [2023-05-11 00:16:28,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +24: [2023-05-11 00:16:28,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +24: [2023-05-11 00:16:28,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +24: [2023-05-11 00:16:28,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +24: [2023-05-11 00:16:28,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +24: [2023-05-11 00:16:28,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +24: [2023-05-11 00:16:28,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +19: [2023-05-11 00:16:28,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +27: [2023-05-11 00:16:28,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +27: [2023-05-11 00:16:28,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +27: [2023-05-11 00:16:28,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +24: [2023-05-11 00:16:28,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +31: [2023-05-11 00:16:28,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +27: [2023-05-11 00:16:28,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +27: [2023-05-11 00:16:28,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +31: [2023-05-11 00:16:28,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +31: [2023-05-11 00:16:28,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +24: [2023-05-11 00:16:28,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +18: [2023-05-11 00:16:28,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +18: [2023-05-11 00:16:28,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +24: [2023-05-11 00:16:28,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +24: [2023-05-11 00:16:28,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +31: [2023-05-11 00:16:28,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +18: [2023-05-11 00:16:28,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +31: [2023-05-11 00:16:28,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +24: [2023-05-11 00:16:28,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +18: [2023-05-11 00:16:28,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +31: [2023-05-11 00:16:28,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +27: [2023-05-11 00:16:28,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +27: [2023-05-11 00:16:28,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +31: [2023-05-11 00:16:28,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +21: [2023-05-11 00:16:28,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +31: [2023-05-11 00:16:28,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +19: [2023-05-11 00:16:28,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +31: [2023-05-11 00:16:28,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +27: [2023-05-11 00:16:28,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +31: [2023-05-11 00:16:28,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +24: [2023-05-11 00:16:28,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +17: [2023-05-11 00:16:28,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +31: [2023-05-11 00:16:28,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +31: [2023-05-11 00:16:28,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +31: [2023-05-11 00:16:28,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +17: [2023-05-11 00:16:28,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +17: [2023-05-11 00:16:28,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 7: [2023-05-11 00:16:28,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 7: [2023-05-11 00:16:28,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +28: [2023-05-11 00:16:28,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +28: [2023-05-11 00:16:28,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +10: [2023-05-11 00:16:28,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +10: [2023-05-11 00:16:28,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +30: [2023-05-11 00:16:28,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +30: [2023-05-11 00:16:28,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +21: [2023-05-11 00:16:28,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +21: [2023-05-11 00:16:28,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +21: [2023-05-11 00:16:28,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +10: [2023-05-11 00:16:28,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +28: [2023-05-11 00:16:28,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +18: [2023-05-11 00:16:28,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +28: [2023-05-11 00:16:28,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +30: [2023-05-11 00:16:28,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +30: [2023-05-11 00:16:28,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +15: [2023-05-11 00:16:28,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +15: [2023-05-11 00:16:28,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +15: [2023-05-11 00:16:28,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +15: [2023-05-11 00:16:28,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +15: [2023-05-11 00:16:28,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +18: [2023-05-11 00:16:28,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +18: [2023-05-11 00:16:28,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +27: [2023-05-11 00:16:28,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +18: [2023-05-11 00:16:28,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +19: [2023-05-11 00:16:28,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +24: [2023-05-11 00:16:28,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +15: [2023-05-11 00:16:28,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +15: [2023-05-11 00:16:28,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +15: [2023-05-11 00:16:28,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +19: [2023-05-11 00:16:28,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +27: [2023-05-11 00:16:28,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +15: [2023-05-11 00:16:28,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +10: [2023-05-11 00:16:28,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +17: [2023-05-11 00:16:28,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt. +15: [2023-05-11 00:16:28,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 1: [2023-05-11 00:16:28,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +10: [2023-05-11 00:16:28,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +15: [2023-05-11 00:16:28,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +31: [2023-05-11 00:16:28,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +15: [2023-05-11 00:16:28,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +21: [2023-05-11 00:16:28,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +21: [2023-05-11 00:16:28,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 7: [2023-05-11 00:16:28,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +30: [2023-05-11 00:16:28,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +10: [2023-05-11 00:16:28,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +30: [2023-05-11 00:16:28,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +30: [2023-05-11 00:16:28,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +21: [2023-05-11 00:16:28,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +21: [2023-05-11 00:16:28,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +15: [2023-05-11 00:16:28,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +19: [2023-05-11 00:16:28,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +10: [2023-05-11 00:16:28,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +30: [2023-05-11 00:16:28,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +24: [2023-05-11 00:16:28,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +24: [2023-05-11 00:16:28,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +24: [2023-05-11 00:16:28,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +27: [2023-05-11 00:16:28,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +27: [2023-05-11 00:16:28,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +19: [2023-05-11 00:16:28,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +24: [2023-05-11 00:16:28,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +27: [2023-05-11 00:16:28,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +21: [2023-05-11 00:16:28,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +21: [2023-05-11 00:16:28,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +21: [2023-05-11 00:16:28,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +21: [2023-05-11 00:16:28,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 1: [2023-05-11 00:16:28,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +10: [2023-05-11 00:16:28,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +17: [2023-05-11 00:16:28,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +31: [2023-05-11 00:16:28,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +15: [2023-05-11 00:16:28,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +15: [2023-05-11 00:16:28,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +15: [2023-05-11 00:16:28,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +15: [2023-05-11 00:16:28,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +30: [2023-05-11 00:16:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 7: [2023-05-11 00:16:28,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +30: [2023-05-11 00:16:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +30: [2023-05-11 00:16:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +19: [2023-05-11 00:16:28,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +30: [2023-05-11 00:16:28,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +27: [2023-05-11 00:16:28,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +31: [2023-05-11 00:16:28,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +31: [2023-05-11 00:16:28,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +24: [2023-05-11 00:16:28,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +27: [2023-05-11 00:16:28,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +31: [2023-05-11 00:16:28,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +24: [2023-05-11 00:16:28,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +24: [2023-05-11 00:16:28,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +19: [2023-05-11 00:16:28,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +15: [2023-05-11 00:16:28,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +27: [2023-05-11 00:16:28,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +15: [2023-05-11 00:16:28,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +15: [2023-05-11 00:16:28,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +15: [2023-05-11 00:16:28,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +30: [2023-05-11 00:16:28,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +31: [2023-05-11 00:16:28,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +31: [2023-05-11 00:16:28,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +15: [2023-05-11 00:16:28,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +15: [2023-05-11 00:16:28,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +15: [2023-05-11 00:16:28,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +30: [2023-05-11 00:16:28,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +30: [2023-05-11 00:16:28,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +30: [2023-05-11 00:16:28,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +31: [2023-05-11 00:16:28,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +15: [2023-05-11 00:16:28,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +15: [2023-05-11 00:16:28,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +15: [2023-05-11 00:16:28,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +15: [2023-05-11 00:16:28,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +15: [2023-05-11 00:16:28,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +15: [2023-05-11 00:16:28,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +15: [2023-05-11 00:16:28,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 4: [2023-05-11 00:16:28,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +13: [2023-05-11 00:16:28,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +13: [2023-05-11 00:16:28,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +13: [2023-05-11 00:16:28,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +13: [2023-05-11 00:16:28,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 4: [2023-05-11 00:16:28,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +13: [2023-05-11 00:16:28,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +13: [2023-05-11 00:16:28,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +13: [2023-05-11 00:16:28,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +13: [2023-05-11 00:16:28,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 4: [2023-05-11 00:16:28,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +13: [2023-05-11 00:16:28,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +15: [2023-05-11 00:16:28,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +13: [2023-05-11 00:16:28,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +13: [2023-05-11 00:16:28,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +13: [2023-05-11 00:16:28,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +13: [2023-05-11 00:16:28,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +13: [2023-05-11 00:16:28,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +13: [2023-05-11 00:16:28,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +13: [2023-05-11 00:16:28,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 4: [2023-05-11 00:16:28,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +30: [2023-05-11 00:16:28,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +30: [2023-05-11 00:16:28,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +30: [2023-05-11 00:16:28,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +30: [2023-05-11 00:16:28,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 4: [2023-05-11 00:16:28,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +30: [2023-05-11 00:16:28,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +30: [2023-05-11 00:16:28,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 4: [2023-05-11 00:16:28,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +30: [2023-05-11 00:16:28,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +30: [2023-05-11 00:16:28,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +13: [2023-05-11 00:16:28,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +13: [2023-05-11 00:16:28,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +13: [2023-05-11 00:16:28,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +14: [2023-05-11 00:16:28,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +14: [2023-05-11 00:16:28,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +14: [2023-05-11 00:16:28,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +14: [2023-05-11 00:16:28,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +14: [2023-05-11 00:16:28,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +14: [2023-05-11 00:16:28,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +14: [2023-05-11 00:16:28,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +14: [2023-05-11 00:16:28,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +13: [2023-05-11 00:16:28,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +13: [2023-05-11 00:16:28,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +13: [2023-05-11 00:16:28,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +13: [2023-05-11 00:16:28,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +20: [2023-05-11 00:16:28,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +20: [2023-05-11 00:16:28,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +17: [2023-05-11 00:16:28,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +17: [2023-05-11 00:16:28,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +17: [2023-05-11 00:16:28,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +17: [2023-05-11 00:16:28,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt. +20: [2023-05-11 00:16:28,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +20: [2023-05-11 00:16:28,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +14: [2023-05-11 00:16:28,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 5: [2023-05-11 00:16:28,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 5: [2023-05-11 00:16:28,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 6: [2023-05-11 00:16:28,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 6: [2023-05-11 00:16:28,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +29: [2023-05-11 00:16:28,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +29: [2023-05-11 00:16:28,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +29: [2023-05-11 00:16:28,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +29: [2023-05-11 00:16:28,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +13: [2023-05-11 00:16:28,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +17: [2023-05-11 00:16:28,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +17: [2023-05-11 00:16:28,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +17: [2023-05-11 00:16:28,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 6: [2023-05-11 00:16:28,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +20: [2023-05-11 00:16:28,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +17: [2023-05-11 00:16:28,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +20: [2023-05-11 00:16:28,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +14: [2023-05-11 00:16:28,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +14: [2023-05-11 00:16:28,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +14: [2023-05-11 00:16:28,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +20: [2023-05-11 00:16:28,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +20: [2023-05-11 00:16:28,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +14: [2023-05-11 00:16:28,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +29: [2023-05-11 00:16:28,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +14: [2023-05-11 00:16:28,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +14: [2023-05-11 00:16:28,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +29: [2023-05-11 00:16:28,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +14: [2023-05-11 00:16:28,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +29: [2023-05-11 00:16:28,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +29: [2023-05-11 00:16:28,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +12: [2023-05-11 00:16:28,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +12: [2023-05-11 00:16:28,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +12: [2023-05-11 00:16:28,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +12: [2023-05-11 00:16:28,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 6: [2023-05-11 00:16:28,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +14: [2023-05-11 00:16:28,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +14: [2023-05-11 00:16:28,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +10: [2023-05-11 00:16:28,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +10: [2023-05-11 00:16:28,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +10: [2023-05-11 00:16:28,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +10: [2023-05-11 00:16:28,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 9: [2023-05-11 00:16:28,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 9: [2023-05-11 00:16:28,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 9: [2023-05-11 00:16:28,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 9: [2023-05-11 00:16:28,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +14: [2023-05-11 00:16:28,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +14: [2023-05-11 00:16:28,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 6: [2023-05-11 00:16:28,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +16: [2023-05-11 00:16:28,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +16: [2023-05-11 00:16:28,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +16: [2023-05-11 00:16:28,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +16: [2023-05-11 00:16:28,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 6: [2023-05-11 00:16:28,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +25: [2023-05-11 00:16:28,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +25: [2023-05-11 00:16:28,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 6: [2023-05-11 00:16:28,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +12: [2023-05-11 00:16:28,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +14: [2023-05-11 00:16:28,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +12: [2023-05-11 00:16:28,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +12: [2023-05-11 00:16:28,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +12: [2023-05-11 00:16:28,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +25: [2023-05-11 00:16:28,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +25: [2023-05-11 00:16:28,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +14: [2023-05-11 00:16:28,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +10: [2023-05-11 00:16:28,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +14: [2023-05-11 00:16:28,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +14: [2023-05-11 00:16:28,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +16: [2023-05-11 00:16:28,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +16: [2023-05-11 00:16:28,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +16: [2023-05-11 00:16:28,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 9: [2023-05-11 00:16:28,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +25: [2023-05-11 00:16:28,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 3: [2023-05-11 00:16:28,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +16: [2023-05-11 00:16:28,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +10: [2023-05-11 00:16:28,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +10: [2023-05-11 00:16:28,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +10: [2023-05-11 00:16:28,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +25: [2023-05-11 00:16:28,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +25: [2023-05-11 00:16:28,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +14: [2023-05-11 00:16:28,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +25: [2023-05-11 00:16:28,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +14: [2023-05-11 00:16:28,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +14: [2023-05-11 00:16:28,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +14: [2023-05-11 00:16:28,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 3: [2023-05-11 00:16:28,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +28: [2023-05-11 00:16:28,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +28: [2023-05-11 00:16:28,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +28: [2023-05-11 00:16:28,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +28: [2023-05-11 00:16:28,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 3: [2023-05-11 00:16:28,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +20: [2023-05-11 00:16:28,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +20: [2023-05-11 00:16:28,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +20: [2023-05-11 00:16:28,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +20: [2023-05-11 00:16:28,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +20: [2023-05-11 00:16:28,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +25: [2023-05-11 00:16:28,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +25: [2023-05-11 00:16:28,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +25: [2023-05-11 00:16:28,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +25: [2023-05-11 00:16:28,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +18: [2023-05-11 00:16:28,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +18: [2023-05-11 00:16:28,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +18: [2023-05-11 00:16:28,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +14: [2023-05-11 00:16:28,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +14: [2023-05-11 00:16:28,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +14: [2023-05-11 00:16:28,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +14: [2023-05-11 00:16:28,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 6: [2023-05-11 00:16:28,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +25: [2023-05-11 00:16:28,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +18: [2023-05-11 00:16:28,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +25: [2023-05-11 00:16:28,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +25: [2023-05-11 00:16:28,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +25: [2023-05-11 00:16:28,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +28: [2023-05-11 00:16:28,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +25: [2023-05-11 00:16:28,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +25: [2023-05-11 00:16:28,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +25: [2023-05-11 00:16:28,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +25: [2023-05-11 00:16:28,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 6: [2023-05-11 00:16:28,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +28: [2023-05-11 00:16:28,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +28: [2023-05-11 00:16:28,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +28: [2023-05-11 00:16:28,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +25: [2023-05-11 00:16:28,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +25: [2023-05-11 00:16:28,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 1: [2023-05-11 00:16:28,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +18: [2023-05-11 00:16:28,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +18: [2023-05-11 00:16:28,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +18: [2023-05-11 00:16:28,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 3: [2023-05-11 00:16:28,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 1: [2023-05-11 00:16:28,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +18: [2023-05-11 00:16:28,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +25: [2023-05-11 00:16:28,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +19: [2023-05-11 00:16:28,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +19: [2023-05-11 00:16:28,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +31: [2023-05-11 00:16:28,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 1: [2023-05-11 00:16:28,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +25: [2023-05-11 00:16:28,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +25: [2023-05-11 00:16:28,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +31: [2023-05-11 00:16:28,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +19: [2023-05-11 00:16:28,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +25: [2023-05-11 00:16:28,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +27: [2023-05-11 00:16:28,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +27: [2023-05-11 00:16:28,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +31: [2023-05-11 00:16:28,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +31: [2023-05-11 00:16:28,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 7: [2023-05-11 00:16:28,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 7: [2023-05-11 00:16:28,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +19: [2023-05-11 00:16:28,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 7: [2023-05-11 00:16:28,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 1: [2023-05-11 00:16:28,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 1: [2023-05-11 00:16:28,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 1: [2023-05-11 00:16:28,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +27: [2023-05-11 00:16:28,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 7: [2023-05-11 00:16:28,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 3: [2023-05-11 00:16:28,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 1: [2023-05-11 00:16:28,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 1: [2023-05-11 00:16:28,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +24: [2023-05-11 00:16:28,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +24: [2023-05-11 00:16:28,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +24: [2023-05-11 00:16:28,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +27: [2023-05-11 00:16:28,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 1: [2023-05-11 00:16:28,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +24: [2023-05-11 00:16:28,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +25: [2023-05-11 00:16:28,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +27: [2023-05-11 00:16:28,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +19: [2023-05-11 00:16:28,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +19: [2023-05-11 00:16:28,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +27: [2023-05-11 00:16:28,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +31: [2023-05-11 00:16:28,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +31: [2023-05-11 00:16:28,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +31: [2023-05-11 00:16:28,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +19: [2023-05-11 00:16:28,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +27: [2023-05-11 00:16:28,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +19: [2023-05-11 00:16:28,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +31: [2023-05-11 00:16:28,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +12: [2023-05-11 00:16:28,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +12: [2023-05-11 00:16:28,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +29: [2023-05-11 00:16:28,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +29: [2023-05-11 00:16:28,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +29: [2023-05-11 00:16:28,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +29: [2023-05-11 00:16:28,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +29: [2023-05-11 00:16:28,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +25: [2023-05-11 00:16:28,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +29: [2023-05-11 00:16:28,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +29: [2023-05-11 00:16:28,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +29: [2023-05-11 00:16:28,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +29: [2023-05-11 00:16:28,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +29: [2023-05-11 00:16:28,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +29: [2023-05-11 00:16:28,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +29: [2023-05-11 00:16:28,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 7: [2023-05-11 00:16:28,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +24: [2023-05-11 00:16:28,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +24: [2023-05-11 00:16:28,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +25: [2023-05-11 00:16:28,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +20: [2023-05-11 00:16:28,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +20: [2023-05-11 00:16:28,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +20: [2023-05-11 00:16:28,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +20: [2023-05-11 00:16:28,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +20: [2023-05-11 00:16:28,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +20: [2023-05-11 00:16:28,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +20: [2023-05-11 00:16:28,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +20: [2023-05-11 00:16:28,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +20: [2023-05-11 00:16:28,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +20: [2023-05-11 00:16:28,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +20: [2023-05-11 00:16:28,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +20: [2023-05-11 00:16:28,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +20: [2023-05-11 00:16:28,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +20: [2023-05-11 00:16:28,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +20: [2023-05-11 00:16:28,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +20: [2023-05-11 00:16:28,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +20: [2023-05-11 00:16:28,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +20: [2023-05-11 00:16:28,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +20: [2023-05-11 00:16:28,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +12: [2023-05-11 00:16:28,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +12: [2023-05-11 00:16:28,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +12: [2023-05-11 00:16:28,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +12: [2023-05-11 00:16:28,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +27: [2023-05-11 00:16:28,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +12: [2023-05-11 00:16:28,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +12: [2023-05-11 00:16:28,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +12: [2023-05-11 00:16:28,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +12: [2023-05-11 00:16:28,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +29: [2023-05-11 00:16:28,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +12: [2023-05-11 00:16:28,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +24: [2023-05-11 00:16:28,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +12: [2023-05-11 00:16:28,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +12: [2023-05-11 00:16:28,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +12: [2023-05-11 00:16:28,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +12: [2023-05-11 00:16:28,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +29: [2023-05-11 00:16:28,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +29: [2023-05-11 00:16:28,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +12: [2023-05-11 00:16:28,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +29: [2023-05-11 00:16:28,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +25: [2023-05-11 00:16:28,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +24: [2023-05-11 00:16:28,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +25: [2023-05-11 00:16:28,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +16: [2023-05-11 00:16:28,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +16: [2023-05-11 00:16:28,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +16: [2023-05-11 00:16:28,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +16: [2023-05-11 00:16:28,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +29: [2023-05-11 00:16:28,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +19: [2023-05-11 00:16:28,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +19: [2023-05-11 00:16:28,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +19: [2023-05-11 00:16:28,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +19: [2023-05-11 00:16:28,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +16: [2023-05-11 00:16:28,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +16: [2023-05-11 00:16:28,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +16: [2023-05-11 00:16:28,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +16: [2023-05-11 00:16:28,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +16: [2023-05-11 00:16:28,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +16: [2023-05-11 00:16:28,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +16: [2023-05-11 00:16:28,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +25: [2023-05-11 00:16:28,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +19: [2023-05-11 00:16:28,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +19: [2023-05-11 00:16:28,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +16: [2023-05-11 00:16:28,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +19: [2023-05-11 00:16:28,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 1: [2023-05-11 00:16:28,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 1: [2023-05-11 00:16:28,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +16: [2023-05-11 00:16:28,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +16: [2023-05-11 00:16:28,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +29: [2023-05-11 00:16:28,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +16: [2023-05-11 00:16:28,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +16: [2023-05-11 00:16:28,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +19: [2023-05-11 00:16:28,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +30: [2023-05-11 00:16:28,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +30: [2023-05-11 00:16:28,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +30: [2023-05-11 00:16:28,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 1: [2023-05-11 00:16:28,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 1: [2023-05-11 00:16:28,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 1: [2023-05-11 00:16:28,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +19: [2023-05-11 00:16:28,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +19: [2023-05-11 00:16:28,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +30: [2023-05-11 00:16:28,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +19: [2023-05-11 00:16:28,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +19: [2023-05-11 00:16:28,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +19: [2023-05-11 00:16:28,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +19: [2023-05-11 00:16:28,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +24: [2023-05-11 00:16:28,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +24: [2023-05-11 00:16:28,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +24: [2023-05-11 00:16:28,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +24: [2023-05-11 00:16:28,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +24: [2023-05-11 00:16:28,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +24: [2023-05-11 00:16:28,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +24: [2023-05-11 00:16:28,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +24: [2023-05-11 00:16:28,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +24: [2023-05-11 00:16:28,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +24: [2023-05-11 00:16:28,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +24: [2023-05-11 00:16:28,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +24: [2023-05-11 00:16:28,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +24: [2023-05-11 00:16:28,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +24: [2023-05-11 00:16:28,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +21: [2023-05-11 00:16:28,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +21: [2023-05-11 00:16:28,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +21: [2023-05-11 00:16:28,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +21: [2023-05-11 00:16:28,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +19: [2023-05-11 00:16:28,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +19: [2023-05-11 00:16:28,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +19: [2023-05-11 00:16:28,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +12: [2023-05-11 00:16:28,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +30: [2023-05-11 00:16:28,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +29: [2023-05-11 00:16:28,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +16: [2023-05-11 00:16:28,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +24: [2023-05-11 00:16:28,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +24: [2023-05-11 00:16:28,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +29: [2023-05-11 00:16:28,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +12: [2023-05-11 00:16:28,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +12: [2023-05-11 00:16:28,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +30: [2023-05-11 00:16:28,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +30: [2023-05-11 00:16:28,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +30: [2023-05-11 00:16:28,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +12: [2023-05-11 00:16:28,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +19: [2023-05-11 00:16:28,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +28: [2023-05-11 00:16:28,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +28: [2023-05-11 00:16:28,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +28: [2023-05-11 00:16:28,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +28: [2023-05-11 00:16:28,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +24: [2023-05-11 00:16:28,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +28: [2023-05-11 00:16:28,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +21: [2023-05-11 00:16:28,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +28: [2023-05-11 00:16:28,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +28: [2023-05-11 00:16:28,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +28: [2023-05-11 00:16:28,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +28: [2023-05-11 00:16:28,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +28: [2023-05-11 00:16:28,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +28: [2023-05-11 00:16:28,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +28: [2023-05-11 00:16:28,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +12: [2023-05-11 00:16:28,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +16: [2023-05-11 00:16:28,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +28: [2023-05-11 00:16:28,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +28: [2023-05-11 00:16:28,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +28: [2023-05-11 00:16:28,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +28: [2023-05-11 00:16:28,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +19: [2023-05-11 00:16:28,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +13: [2023-05-11 00:16:28,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +13: [2023-05-11 00:16:28,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +13: [2023-05-11 00:16:28,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +21: [2023-05-11 00:16:28,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +21: [2023-05-11 00:16:28,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +19: [2023-05-11 00:16:28,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +15: [2023-05-11 00:16:28,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +15: [2023-05-11 00:16:28,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +21: [2023-05-11 00:16:28,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +15: [2023-05-11 00:16:28,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +15: [2023-05-11 00:16:28,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +29: [2023-05-11 00:16:28,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +12: [2023-05-11 00:16:28,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +12: [2023-05-11 00:16:28,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +12: [2023-05-11 00:16:28,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +16: [2023-05-11 00:16:28,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +13: [2023-05-11 00:16:28,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 7: [2023-05-11 00:16:28,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +29: [2023-05-11 00:16:28,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +19: [2023-05-11 00:16:28,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +24: [2023-05-11 00:16:28,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +24: [2023-05-11 00:16:28,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +24: [2023-05-11 00:16:28,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +28: [2023-05-11 00:16:28,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +16: [2023-05-11 00:16:28,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +19: [2023-05-11 00:16:28,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +19: [2023-05-11 00:16:28,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +13: [2023-05-11 00:16:28,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +15: [2023-05-11 00:16:28,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +27: [2023-05-11 00:16:28,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +27: [2023-05-11 00:16:28,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +27: [2023-05-11 00:16:28,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +27: [2023-05-11 00:16:28,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +27: [2023-05-11 00:16:28,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +27: [2023-05-11 00:16:28,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +27: [2023-05-11 00:16:28,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +16: [2023-05-11 00:16:28,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +27: [2023-05-11 00:16:28,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +27: [2023-05-11 00:16:28,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +27: [2023-05-11 00:16:28,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +27: [2023-05-11 00:16:28,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +27: [2023-05-11 00:16:28,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +27: [2023-05-11 00:16:28,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +27: [2023-05-11 00:16:28,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 7: [2023-05-11 00:16:28,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +15: [2023-05-11 00:16:28,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +13: [2023-05-11 00:16:28,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +13: [2023-05-11 00:16:28,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +15: [2023-05-11 00:16:28,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +15: [2023-05-11 00:16:28,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +16: [2023-05-11 00:16:28,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +13: [2023-05-11 00:16:28,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +24: [2023-05-11 00:16:28,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +24: [2023-05-11 00:16:28,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +29: [2023-05-11 00:16:28,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +28: [2023-05-11 00:16:28,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +16: [2023-05-11 00:16:28,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +27: [2023-05-11 00:16:28,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +27: [2023-05-11 00:16:28,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +24: [2023-05-11 00:16:28,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +19: [2023-05-11 00:16:28,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +29: [2023-05-11 00:16:28,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +16: [2023-05-11 00:16:28,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +28: [2023-05-11 00:16:28,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +27: [2023-05-11 00:16:28,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +27: [2023-05-11 00:16:28,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +24: [2023-05-11 00:16:28,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +27: [2023-05-11 00:16:28,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +21: [2023-05-11 00:16:28,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +21: [2023-05-11 00:16:28,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +21: [2023-05-11 00:16:28,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +21: [2023-05-11 00:16:28,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +28: [2023-05-11 00:16:28,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +27: [2023-05-11 00:16:28,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +28: [2023-05-11 00:16:28,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 4: [2023-05-11 00:16:28,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +21: [2023-05-11 00:16:28,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +21: [2023-05-11 00:16:28,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +21: [2023-05-11 00:16:28,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +21: [2023-05-11 00:16:28,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +21: [2023-05-11 00:16:28,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +21: [2023-05-11 00:16:28,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +30: [2023-05-11 00:16:28,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +30: [2023-05-11 00:16:28,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +30: [2023-05-11 00:16:28,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +21: [2023-05-11 00:16:28,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +21: [2023-05-11 00:16:28,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +21: [2023-05-11 00:16:28,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +30: [2023-05-11 00:16:28,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +30: [2023-05-11 00:16:28,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +30: [2023-05-11 00:16:28,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +21: [2023-05-11 00:16:28,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 5: [2023-05-11 00:16:28,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 5: [2023-05-11 00:16:28,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +21: [2023-05-11 00:16:28,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +21: [2023-05-11 00:16:28,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 5: [2023-05-11 00:16:28,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +30: [2023-05-11 00:16:28,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +30: [2023-05-11 00:16:28,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +30: [2023-05-11 00:16:28,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +30: [2023-05-11 00:16:28,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +30: [2023-05-11 00:16:28,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 7: [2023-05-11 00:16:28,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 3: [2023-05-11 00:16:28,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +30: [2023-05-11 00:16:28,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +30: [2023-05-11 00:16:28,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +30: [2023-05-11 00:16:28,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 6: [2023-05-11 00:16:28,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 6: [2023-05-11 00:16:28,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +30: [2023-05-11 00:16:28,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +30: [2023-05-11 00:16:28,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 4: [2023-05-11 00:16:28,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +28: [2023-05-11 00:16:28,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 6: [2023-05-11 00:16:28,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +10: [2023-05-11 00:16:28,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +10: [2023-05-11 00:16:28,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +10: [2023-05-11 00:16:28,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +10: [2023-05-11 00:16:28,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +10: [2023-05-11 00:16:28,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +10: [2023-05-11 00:16:28,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +10: [2023-05-11 00:16:28,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +10: [2023-05-11 00:16:28,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +10: [2023-05-11 00:16:28,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +10: [2023-05-11 00:16:28,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +10: [2023-05-11 00:16:28,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +27: [2023-05-11 00:16:28,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +10: [2023-05-11 00:16:28,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +27: [2023-05-11 00:16:28,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +27: [2023-05-11 00:16:28,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +28: [2023-05-11 00:16:28,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +27: [2023-05-11 00:16:28,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +10: [2023-05-11 00:16:28,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +28: [2023-05-11 00:16:28,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +10: [2023-05-11 00:16:28,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +10: [2023-05-11 00:16:28,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +10: [2023-05-11 00:16:28,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 4: [2023-05-11 00:16:28,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +21: [2023-05-11 00:16:28,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +30: [2023-05-11 00:16:28,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 6: [2023-05-11 00:16:28,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 6: [2023-05-11 00:16:28,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 7: [2023-05-11 00:16:28,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 9: [2023-05-11 00:16:28,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 9: [2023-05-11 00:16:28,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 9: [2023-05-11 00:16:28,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 9: [2023-05-11 00:16:28,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 9: [2023-05-11 00:16:28,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +25: [2023-05-11 00:16:28,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +25: [2023-05-11 00:16:28,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +10: [2023-05-11 00:16:28,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +25: [2023-05-11 00:16:28,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +25: [2023-05-11 00:16:28,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +10: [2023-05-11 00:16:28,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +21: [2023-05-11 00:16:28,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +30: [2023-05-11 00:16:28,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +21: [2023-05-11 00:16:28,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 3: [2023-05-11 00:16:28,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 3: [2023-05-11 00:16:28,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 5: [2023-05-11 00:16:28,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 4: [2023-05-11 00:16:28,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +31: [2023-05-11 00:16:28,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +31: [2023-05-11 00:16:28,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +30: [2023-05-11 00:16:28,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +31: [2023-05-11 00:16:28,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +31: [2023-05-11 00:16:28,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +31: [2023-05-11 00:16:28,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +31: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 5: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +31: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +13: [2023-05-11 00:16:28,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +13: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +13: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +13: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +31: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +13: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +13: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +13: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +13: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +10: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +31: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +31: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +31: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +13: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +13: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +13: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +13: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +31: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +31: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +31: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +13: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +30: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +13: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +25: [2023-05-11 00:16:28,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +13: [2023-05-11 00:16:28,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +31: [2023-05-11 00:16:28,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +31: [2023-05-11 00:16:28,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +13: [2023-05-11 00:16:28,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +25: [2023-05-11 00:16:28,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 4: [2023-05-11 00:16:28,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 4: [2023-05-11 00:16:28,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +25: [2023-05-11 00:16:28,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +10: [2023-05-11 00:16:28,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +21: [2023-05-11 00:16:28,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +21: [2023-05-11 00:16:28,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +10: [2023-05-11 00:16:28,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +21: [2023-05-11 00:16:28,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +30: [2023-05-11 00:16:28,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +18: [2023-05-11 00:16:28,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +18: [2023-05-11 00:16:28,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +18: [2023-05-11 00:16:28,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +18: [2023-05-11 00:16:28,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +18: [2023-05-11 00:16:28,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +18: [2023-05-11 00:16:28,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +25: [2023-05-11 00:16:28,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +18: [2023-05-11 00:16:28,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +18: [2023-05-11 00:16:28,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +18: [2023-05-11 00:16:28,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +18: [2023-05-11 00:16:28,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +18: [2023-05-11 00:16:28,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +18: [2023-05-11 00:16:28,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 4: [2023-05-11 00:16:29,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +18: [2023-05-11 00:16:29,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +18: [2023-05-11 00:16:29,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +18: [2023-05-11 00:16:29,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +18: [2023-05-11 00:16:29,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +10: [2023-05-11 00:16:29,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +30: [2023-05-11 00:16:29,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +30: [2023-05-11 00:16:29,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +21: [2023-05-11 00:16:29,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +21: [2023-05-11 00:16:29,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +10: [2023-05-11 00:16:29,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +13: [2023-05-11 00:16:29,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +31: [2023-05-11 00:16:29,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +30: [2023-05-11 00:16:29,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +14: [2023-05-11 00:16:29,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +14: [2023-05-11 00:16:29,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +14: [2023-05-11 00:16:29,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +31: [2023-05-11 00:16:29,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +14: [2023-05-11 00:16:29,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +31: [2023-05-11 00:16:29,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +31: [2023-05-11 00:16:29,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +18: [2023-05-11 00:16:29,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +18: [2023-05-11 00:16:29,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +15: [2023-05-11 00:16:29,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +15: [2023-05-11 00:16:29,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +15: [2023-05-11 00:16:29,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +15: [2023-05-11 00:16:29,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +15: [2023-05-11 00:16:29,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +10: [2023-05-11 00:16:29,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +15: [2023-05-11 00:16:29,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +15: [2023-05-11 00:16:29,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +15: [2023-05-11 00:16:29,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +15: [2023-05-11 00:16:29,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +15: [2023-05-11 00:16:29,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +15: [2023-05-11 00:16:29,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +14: [2023-05-11 00:16:29,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +14: [2023-05-11 00:16:29,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +13: [2023-05-11 00:16:29,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +13: [2023-05-11 00:16:29,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +13: [2023-05-11 00:16:29,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +15: [2023-05-11 00:16:29,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +15: [2023-05-11 00:16:29,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +14: [2023-05-11 00:16:29,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +14: [2023-05-11 00:16:29,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +14: [2023-05-11 00:16:29,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +14: [2023-05-11 00:16:29,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +15: [2023-05-11 00:16:29,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +15: [2023-05-11 00:16:29,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +15: [2023-05-11 00:16:29,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 4: [2023-05-11 00:16:29,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +14: [2023-05-11 00:16:29,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +31: [2023-05-11 00:16:29,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +13: [2023-05-11 00:16:29,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +14: [2023-05-11 00:16:29,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +31: [2023-05-11 00:16:29,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 9: [2023-05-11 00:16:28,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +14: [2023-05-11 00:16:29,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +31: [2023-05-11 00:16:29,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +18: [2023-05-11 00:16:29,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +14: [2023-05-11 00:16:29,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +14: [2023-05-11 00:16:29,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +18: [2023-05-11 00:16:29,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +13: [2023-05-11 00:16:29,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +14: [2023-05-11 00:16:29,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +31: [2023-05-11 00:16:29,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +13: [2023-05-11 00:16:29,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +15: [2023-05-11 00:16:29,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +13: [2023-05-11 00:16:29,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +15: [2023-05-11 00:16:29,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +18: [2023-05-11 00:16:29,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +18: [2023-05-11 00:16:29,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +14: [2023-05-11 00:16:29,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 6: [2023-05-11 00:16:29,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 6: [2023-05-11 00:16:29,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +15: [2023-05-11 00:16:29,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +15: [2023-05-11 00:16:29,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +18: [2023-05-11 00:16:29,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 1: [2023-05-11 00:16:29,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 1: [2023-05-11 00:16:29,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 1: [2023-05-11 00:16:29,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +14: [2023-05-11 00:16:29,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +15: [2023-05-11 00:16:29,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 6: [2023-05-11 00:16:29,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +18: [2023-05-11 00:16:29,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +14: [2023-05-11 00:16:29,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +14: [2023-05-11 00:16:29,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 3: [2023-05-11 00:16:29,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 3: [2023-05-11 00:16:29,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 3: [2023-05-11 00:16:29,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 3: [2023-05-11 00:16:29,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +14: [2023-05-11 00:16:29,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +14: [2023-05-11 00:16:29,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +15: [2023-05-11 00:16:29,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +20: [2023-05-11 00:16:29,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +20: [2023-05-11 00:16:29,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +20: [2023-05-11 00:16:29,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +20: [2023-05-11 00:16:29,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +15: [2023-05-11 00:16:29,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +14: [2023-05-11 00:16:29,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +19: [2023-05-11 00:16:29,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +19: [2023-05-11 00:16:29,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +19: [2023-05-11 00:16:29,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +19: [2023-05-11 00:16:29,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +14: [2023-05-11 00:16:29,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +14: [2023-05-11 00:16:29,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +14: [2023-05-11 00:16:29,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +15: [2023-05-11 00:16:29,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +20: [2023-05-11 00:16:29,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +20: [2023-05-11 00:16:29,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +20: [2023-05-11 00:16:29,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +19: [2023-05-11 00:16:29,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +20: [2023-05-11 00:16:29,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +19: [2023-05-11 00:16:29,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +19: [2023-05-11 00:16:29,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +19: [2023-05-11 00:16:29,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +14: [2023-05-11 00:16:29,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +14: [2023-05-11 00:16:29,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +14: [2023-05-11 00:16:29,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 6: [2023-05-11 00:16:29,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +14: [2023-05-11 00:16:29,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +14: [2023-05-11 00:16:29,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +14: [2023-05-11 00:16:29,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 3: [2023-05-11 00:16:29,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 6: [2023-05-11 00:16:29,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +27: [2023-05-11 00:16:29,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +27: [2023-05-11 00:16:29,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +27: [2023-05-11 00:16:29,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +27: [2023-05-11 00:16:29,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 3: [2023-05-11 00:16:29,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 3: [2023-05-11 00:16:29,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +27: [2023-05-11 00:16:29,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +27: [2023-05-11 00:16:29,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +27: [2023-05-11 00:16:29,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 7: [2023-05-11 00:16:29,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +27: [2023-05-11 00:16:29,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 4: [2023-05-11 00:16:29,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 4: [2023-05-11 00:16:29,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 4: [2023-05-11 00:16:29,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 7: [2023-05-11 00:16:29,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +24: [2023-05-11 00:16:29,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +24: [2023-05-11 00:16:29,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +24: [2023-05-11 00:16:29,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +24: [2023-05-11 00:16:29,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 4: [2023-05-11 00:16:29,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 7: [2023-05-11 00:16:29,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +31: [2023-05-11 00:16:29,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +31: [2023-05-11 00:16:29,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +31: [2023-05-11 00:16:29,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 4: [2023-05-11 00:16:29,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +31: [2023-05-11 00:16:29,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +25: [2023-05-11 00:16:29,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +25: [2023-05-11 00:16:29,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +25: [2023-05-11 00:16:29,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +25: [2023-05-11 00:16:29,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +25: [2023-05-11 00:16:29,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +25: [2023-05-11 00:16:29,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +24: [2023-05-11 00:16:29,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +24: [2023-05-11 00:16:29,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +25: [2023-05-11 00:16:29,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +25: [2023-05-11 00:16:29,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +25: [2023-05-11 00:16:29,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +25: [2023-05-11 00:16:29,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +25: [2023-05-11 00:16:29,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +25: [2023-05-11 00:16:29,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +25: [2023-05-11 00:16:29,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +25: [2023-05-11 00:16:29,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +19: [2023-05-11 00:16:29,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +19: [2023-05-11 00:16:29,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +19: [2023-05-11 00:16:29,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +19: [2023-05-11 00:16:29,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +24: [2023-05-11 00:16:29,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +19: [2023-05-11 00:16:29,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +19: [2023-05-11 00:16:29,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +25: [2023-05-11 00:16:29,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +19: [2023-05-11 00:16:29,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +19: [2023-05-11 00:16:29,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +19: [2023-05-11 00:16:29,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +19: [2023-05-11 00:16:29,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +19: [2023-05-11 00:16:29,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +24: [2023-05-11 00:16:29,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +13: [2023-05-11 00:16:29,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +13: [2023-05-11 00:16:29,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +13: [2023-05-11 00:16:29,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 4: [2023-05-11 00:16:29,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +19: [2023-05-11 00:16:29,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +19: [2023-05-11 00:16:29,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +13: [2023-05-11 00:16:29,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +25: [2023-05-11 00:16:29,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +19: [2023-05-11 00:16:29,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +19: [2023-05-11 00:16:29,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +19: [2023-05-11 00:16:29,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 5: [2023-05-11 00:16:29,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 5: [2023-05-11 00:16:29,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 5: [2023-05-11 00:16:29,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 5: [2023-05-11 00:16:29,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +31: [2023-05-11 00:16:29,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +31: [2023-05-11 00:16:29,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +25: [2023-05-11 00:16:29,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +31: [2023-05-11 00:16:29,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +19: [2023-05-11 00:16:29,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +25: [2023-05-11 00:16:29,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +13: [2023-05-11 00:16:29,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +31: [2023-05-11 00:16:29,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +25: [2023-05-11 00:16:29,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +19: [2023-05-11 00:16:29,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +13: [2023-05-11 00:16:29,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +13: [2023-05-11 00:16:29,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +13: [2023-05-11 00:16:29,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +25: [2023-05-11 00:16:29,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +25: [2023-05-11 00:16:29,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +25: [2023-05-11 00:16:29,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +19: [2023-05-11 00:16:29,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +19: [2023-05-11 00:16:29,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +25: [2023-05-11 00:16:29,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +25: [2023-05-11 00:16:29,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 7: [2023-05-11 00:16:29,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 7: [2023-05-11 00:16:29,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +19: [2023-05-11 00:16:29,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 7: [2023-05-11 00:16:29,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +19: [2023-05-11 00:16:29,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +20: [2023-05-11 00:16:29,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +20: [2023-05-11 00:16:29,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +20: [2023-05-11 00:16:29,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +20: [2023-05-11 00:16:29,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +20: [2023-05-11 00:16:29,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +20: [2023-05-11 00:16:29,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +20: [2023-05-11 00:16:29,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +20: [2023-05-11 00:16:29,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +20: [2023-05-11 00:16:29,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +20: [2023-05-11 00:16:29,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +20: [2023-05-11 00:16:29,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +20: [2023-05-11 00:16:29,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +20: [2023-05-11 00:16:29,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +20: [2023-05-11 00:16:29,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +20: [2023-05-11 00:16:29,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +20: [2023-05-11 00:16:29,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +19: [2023-05-11 00:16:29,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 9: [2023-05-11 00:16:29,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 9: [2023-05-11 00:16:29,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 1: [2023-05-11 00:16:29,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +19: [2023-05-11 00:16:29,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 1: [2023-05-11 00:16:29,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 9: [2023-05-11 00:16:29,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 1: [2023-05-11 00:16:29,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 1: [2023-05-11 00:16:29,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 1: [2023-05-11 00:16:29,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 9: [2023-05-11 00:16:29,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +20: [2023-05-11 00:16:29,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +20: [2023-05-11 00:16:29,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +30: [2023-05-11 00:16:29,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +30: [2023-05-11 00:16:29,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +30: [2023-05-11 00:16:29,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +30: [2023-05-11 00:16:29,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +20: [2023-05-11 00:16:29,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +20: [2023-05-11 00:16:29,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +24: [2023-05-11 00:16:29,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +24: [2023-05-11 00:16:29,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +24: [2023-05-11 00:16:29,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +24: [2023-05-11 00:16:29,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 4: [2023-05-11 00:16:29,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +24: [2023-05-11 00:16:29,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +27: [2023-05-11 00:16:29,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +27: [2023-05-11 00:16:29,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +27: [2023-05-11 00:16:29,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +27: [2023-05-11 00:16:29,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +27: [2023-05-11 00:16:29,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +27: [2023-05-11 00:16:29,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +27: [2023-05-11 00:16:29,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +27: [2023-05-11 00:16:29,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +27: [2023-05-11 00:16:29,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +24: [2023-05-11 00:16:29,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +24: [2023-05-11 00:16:29,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +27: [2023-05-11 00:16:29,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +24: [2023-05-11 00:16:29,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +24: [2023-05-11 00:16:29,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +24: [2023-05-11 00:16:29,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +24: [2023-05-11 00:16:29,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +27: [2023-05-11 00:16:29,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +24: [2023-05-11 00:16:29,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +27: [2023-05-11 00:16:29,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 4: [2023-05-11 00:16:29,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 4: [2023-05-11 00:16:29,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 4: [2023-05-11 00:16:29,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +12: [2023-05-11 00:16:29,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +27: [2023-05-11 00:16:29,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +12: [2023-05-11 00:16:29,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +27: [2023-05-11 00:16:29,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +27: [2023-05-11 00:16:29,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +27: [2023-05-11 00:16:29,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +24: [2023-05-11 00:16:29,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +24: [2023-05-11 00:16:29,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +24: [2023-05-11 00:16:29,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +24: [2023-05-11 00:16:29,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +20: [2023-05-11 00:16:29,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +20: [2023-05-11 00:16:29,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +12: [2023-05-11 00:16:29,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +12: [2023-05-11 00:16:29,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +16: [2023-05-11 00:16:29,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +16: [2023-05-11 00:16:29,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +30: [2023-05-11 00:16:29,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +20: [2023-05-11 00:16:29,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +16: [2023-05-11 00:16:29,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +16: [2023-05-11 00:16:29,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +30: [2023-05-11 00:16:29,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +20: [2023-05-11 00:16:29,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +16: [2023-05-11 00:16:29,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +16: [2023-05-11 00:16:29,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +16: [2023-05-11 00:16:29,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +16: [2023-05-11 00:16:29,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +13: [2023-05-11 00:16:29,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +13: [2023-05-11 00:16:29,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +13: [2023-05-11 00:16:29,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +13: [2023-05-11 00:16:29,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +16: [2023-05-11 00:16:29,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +16: [2023-05-11 00:16:29,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +13: [2023-05-11 00:16:29,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +13: [2023-05-11 00:16:29,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +13: [2023-05-11 00:16:29,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +13: [2023-05-11 00:16:29,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +16: [2023-05-11 00:16:29,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +16: [2023-05-11 00:16:29,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +13: [2023-05-11 00:16:29,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +13: [2023-05-11 00:16:29,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +13: [2023-05-11 00:16:29,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +13: [2023-05-11 00:16:29,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +30: [2023-05-11 00:16:29,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +30: [2023-05-11 00:16:29,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +31: [2023-05-11 00:16:29,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +31: [2023-05-11 00:16:29,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +13: [2023-05-11 00:16:29,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +31: [2023-05-11 00:16:29,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +24: [2023-05-11 00:16:29,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +13: [2023-05-11 00:16:29,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +31: [2023-05-11 00:16:29,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +31: [2023-05-11 00:16:29,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +31: [2023-05-11 00:16:29,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +31: [2023-05-11 00:16:29,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +31: [2023-05-11 00:16:29,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +31: [2023-05-11 00:16:29,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +13: [2023-05-11 00:16:29,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +13: [2023-05-11 00:16:29,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +28: [2023-05-11 00:16:29,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +28: [2023-05-11 00:16:29,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +28: [2023-05-11 00:16:29,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 4: [2023-05-11 00:16:29,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +31: [2023-05-11 00:16:29,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +28: [2023-05-11 00:16:29,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +31: [2023-05-11 00:16:29,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +31: [2023-05-11 00:16:29,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +27: [2023-05-11 00:16:29,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +31: [2023-05-11 00:16:29,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +12: [2023-05-11 00:16:29,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +31: [2023-05-11 00:16:29,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +12: [2023-05-11 00:16:29,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +27: [2023-05-11 00:16:29,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +31: [2023-05-11 00:16:29,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +16: [2023-05-11 00:16:29,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +31: [2023-05-11 00:16:29,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +12: [2023-05-11 00:16:29,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +12: [2023-05-11 00:16:29,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +24: [2023-05-11 00:16:29,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +16: [2023-05-11 00:16:29,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +16: [2023-05-11 00:16:29,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +10: [2023-05-11 00:16:29,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +16: [2023-05-11 00:16:29,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +29: [2023-05-11 00:16:29,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +29: [2023-05-11 00:16:29,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +10: [2023-05-11 00:16:29,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +24: [2023-05-11 00:16:29,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +28: [2023-05-11 00:16:29,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +28: [2023-05-11 00:16:29,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +28: [2023-05-11 00:16:29,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +10: [2023-05-11 00:16:29,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +10: [2023-05-11 00:16:29,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +28: [2023-05-11 00:16:29,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +27: [2023-05-11 00:16:29,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +31: [2023-05-11 00:16:29,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +13: [2023-05-11 00:16:29,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +27: [2023-05-11 00:16:29,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +29: [2023-05-11 00:16:29,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +29: [2023-05-11 00:16:29,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +27: [2023-05-11 00:16:29,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +16: [2023-05-11 00:16:29,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +16: [2023-05-11 00:16:29,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +24: [2023-05-11 00:16:29,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +10: [2023-05-11 00:16:29,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +21: [2023-05-11 00:16:29,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +13: [2023-05-11 00:16:29,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +16: [2023-05-11 00:16:29,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +16: [2023-05-11 00:16:29,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +10: [2023-05-11 00:16:29,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +27: [2023-05-11 00:16:29,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +21: [2023-05-11 00:16:29,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +21: [2023-05-11 00:16:29,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +29: [2023-05-11 00:16:29,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +10: [2023-05-11 00:16:29,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +31: [2023-05-11 00:16:29,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +10: [2023-05-11 00:16:29,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +21: [2023-05-11 00:16:29,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +16: [2023-05-11 00:16:29,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +16: [2023-05-11 00:16:29,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +13: [2023-05-11 00:16:29,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +24: [2023-05-11 00:16:29,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +24: [2023-05-11 00:16:29,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +29: [2023-05-11 00:16:29,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +13: [2023-05-11 00:16:29,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +13: [2023-05-11 00:16:29,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +27: [2023-05-11 00:16:29,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +13: [2023-05-11 00:16:29,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +21: [2023-05-11 00:16:29,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +31: [2023-05-11 00:16:29,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +16: [2023-05-11 00:16:29,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +16: [2023-05-11 00:16:29,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +16: [2023-05-11 00:16:29,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +16: [2023-05-11 00:16:29,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +16: [2023-05-11 00:16:29,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 5: [2023-05-11 00:16:29,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 5: [2023-05-11 00:16:29,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 5: [2023-05-11 00:16:29,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +16: [2023-05-11 00:16:29,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +29: [2023-05-11 00:16:29,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +29: [2023-05-11 00:16:29,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +27: [2023-05-11 00:16:29,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +21: [2023-05-11 00:16:29,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +16: [2023-05-11 00:16:29,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +16: [2023-05-11 00:16:29,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +21: [2023-05-11 00:16:29,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +16: [2023-05-11 00:16:29,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +16: [2023-05-11 00:16:29,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 5: [2023-05-11 00:16:29,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +31: [2023-05-11 00:16:29,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +21: [2023-05-11 00:16:29,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +15: [2023-05-11 00:16:29,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +15: [2023-05-11 00:16:29,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +15: [2023-05-11 00:16:29,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +31: [2023-05-11 00:16:29,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +24: [2023-05-11 00:16:29,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +24: [2023-05-11 00:16:29,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +15: [2023-05-11 00:16:29,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 9: [2023-05-11 00:16:29,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 9: [2023-05-11 00:16:29,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 9: [2023-05-11 00:16:29,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 9: [2023-05-11 00:16:29,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 9: [2023-05-11 00:16:29,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 9: [2023-05-11 00:16:29,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 9: [2023-05-11 00:16:29,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 9: [2023-05-11 00:16:29,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 9: [2023-05-11 00:16:29,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 9: [2023-05-11 00:16:29,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 9: [2023-05-11 00:16:29,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +12: [2023-05-11 00:16:29,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +12: [2023-05-11 00:16:29,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 9: [2023-05-11 00:16:29,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +28: [2023-05-11 00:16:29,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +28: [2023-05-11 00:16:29,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +28: [2023-05-11 00:16:29,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +28: [2023-05-11 00:16:29,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +28: [2023-05-11 00:16:29,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +28: [2023-05-11 00:16:29,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +12: [2023-05-11 00:16:29,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +28: [2023-05-11 00:16:29,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +28: [2023-05-11 00:16:29,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +28: [2023-05-11 00:16:29,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +28: [2023-05-11 00:16:29,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +28: [2023-05-11 00:16:29,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +28: [2023-05-11 00:16:29,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +12: [2023-05-11 00:16:29,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +12: [2023-05-11 00:16:29,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +12: [2023-05-11 00:16:29,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +28: [2023-05-11 00:16:29,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +28: [2023-05-11 00:16:29,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +28: [2023-05-11 00:16:29,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +28: [2023-05-11 00:16:29,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +13: [2023-05-11 00:16:29,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +12: [2023-05-11 00:16:29,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +12: [2023-05-11 00:16:29,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +12: [2023-05-11 00:16:29,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +12: [2023-05-11 00:16:29,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 9: [2023-05-11 00:16:29,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +12: [2023-05-11 00:16:29,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +12: [2023-05-11 00:16:29,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +12: [2023-05-11 00:16:29,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +12: [2023-05-11 00:16:29,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +12: [2023-05-11 00:16:29,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +12: [2023-05-11 00:16:29,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +13: [2023-05-11 00:16:29,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +31: [2023-05-11 00:16:29,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +21: [2023-05-11 00:16:29,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +21: [2023-05-11 00:16:29,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +21: [2023-05-11 00:16:29,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +21: [2023-05-11 00:16:29,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +21: [2023-05-11 00:16:29,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +21: [2023-05-11 00:16:29,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +31: [2023-05-11 00:16:29,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +21: [2023-05-11 00:16:29,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +21: [2023-05-11 00:16:29,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +31: [2023-05-11 00:16:29,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +15: [2023-05-11 00:16:29,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +15: [2023-05-11 00:16:29,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +15: [2023-05-11 00:16:29,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +15: [2023-05-11 00:16:29,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +21: [2023-05-11 00:16:29,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +21: [2023-05-11 00:16:29,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +21: [2023-05-11 00:16:29,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +21: [2023-05-11 00:16:29,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +28: [2023-05-11 00:16:29,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +28: [2023-05-11 00:16:29,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +26: [2023-05-11 00:16:29,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +12: [2023-05-11 00:16:29,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +22: [2023-05-11 00:16:29,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +22: [2023-05-11 00:16:29,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +26: [2023-05-11 00:16:29,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +26: [2023-05-11 00:16:29,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +26: [2023-05-11 00:16:29,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +26: [2023-05-11 00:16:29,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +26: [2023-05-11 00:16:29,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +26: [2023-05-11 00:16:29,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +28: [2023-05-11 00:16:29,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +26: [2023-05-11 00:16:29,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +26: [2023-05-11 00:16:29,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +26: [2023-05-11 00:16:29,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +28: [2023-05-11 00:16:29,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +26: [2023-05-11 00:16:29,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +26: [2023-05-11 00:16:29,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +26: [2023-05-11 00:16:29,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +22: [2023-05-11 00:16:29,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +22: [2023-05-11 00:16:29,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +22: [2023-05-11 00:16:29,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +22: [2023-05-11 00:16:29,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +22: [2023-05-11 00:16:29,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +22: [2023-05-11 00:16:29,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +22: [2023-05-11 00:16:29,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +22: [2023-05-11 00:16:29,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +22: [2023-05-11 00:16:29,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +26: [2023-05-11 00:16:29,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +26: [2023-05-11 00:16:29,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +26: [2023-05-11 00:16:29,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +22: [2023-05-11 00:16:29,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 8: [2023-05-11 00:16:29,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 9: [2023-05-11 00:16:29,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 9: [2023-05-11 00:16:29,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +22: [2023-05-11 00:16:29,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 5: [2023-05-11 00:16:29,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +22: [2023-05-11 00:16:29,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +22: [2023-05-11 00:16:29,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +22: [2023-05-11 00:16:29,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +23: [2023-05-11 00:16:29,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +23: [2023-05-11 00:16:29,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 0: [2023-05-11 00:16:29,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +23: [2023-05-11 00:16:29,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +23: [2023-05-11 00:16:29,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 8: [2023-05-11 00:16:29,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +23: [2023-05-11 00:16:29,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +23: [2023-05-11 00:16:29,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +23: [2023-05-11 00:16:29,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +23: [2023-05-11 00:16:29,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +23: [2023-05-11 00:16:29,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +23: [2023-05-11 00:16:29,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +23: [2023-05-11 00:16:29,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 8: [2023-05-11 00:16:29,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +23: [2023-05-11 00:16:29,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +23: [2023-05-11 00:16:29,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +23: [2023-05-11 00:16:29,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 2: [2023-05-11 00:16:29,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 0: [2023-05-11 00:16:29,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 0: [2023-05-11 00:16:29,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 0: [2023-05-11 00:16:29,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +21: [2023-05-11 00:16:29,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +21: [2023-05-11 00:16:29,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 0: [2023-05-11 00:16:29,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +23: [2023-05-11 00:16:29,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +21: [2023-05-11 00:16:29,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +12: [2023-05-11 00:16:29,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +12: [2023-05-11 00:16:29,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +23: [2023-05-11 00:16:29,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +12: [2023-05-11 00:16:29,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +21: [2023-05-11 00:16:29,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 2: [2023-05-11 00:16:29,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 2: [2023-05-11 00:16:29,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +21: [2023-05-11 00:16:29,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +28: [2023-05-11 00:16:29,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +28: [2023-05-11 00:16:29,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +22: [2023-05-11 00:16:29,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +21: [2023-05-11 00:16:29,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +21: [2023-05-11 00:16:29,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 9: [2023-05-11 00:16:29,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +21: [2023-05-11 00:16:29,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +28: [2023-05-11 00:16:29,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +28: [2023-05-11 00:16:29,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +22: [2023-05-11 00:16:29,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +22: [2023-05-11 00:16:29,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +21: [2023-05-11 00:16:29,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +29: [2023-05-11 00:16:29,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +29: [2023-05-11 00:16:29,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +29: [2023-05-11 00:16:29,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +29: [2023-05-11 00:16:29,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +29: [2023-05-11 00:16:29,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +29: [2023-05-11 00:16:29,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +29: [2023-05-11 00:16:29,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +29: [2023-05-11 00:16:29,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +29: [2023-05-11 00:16:29,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +12: [2023-05-11 00:16:29,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +12: [2023-05-11 00:16:29,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +29: [2023-05-11 00:16:29,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +23: [2023-05-11 00:16:29,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +23: [2023-05-11 00:16:29,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +29: [2023-05-11 00:16:29,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +29: [2023-05-11 00:16:29,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +29: [2023-05-11 00:16:29,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +29: [2023-05-11 00:16:29,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +29: [2023-05-11 00:16:29,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +29: [2023-05-11 00:16:29,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +12: [2023-05-11 00:16:29,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +12: [2023-05-11 00:16:29,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +26: [2023-05-11 00:16:29,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +26: [2023-05-11 00:16:29,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +21: [2023-05-11 00:16:29,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +21: [2023-05-11 00:16:29,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 9: [2023-05-11 00:16:29,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +22: [2023-05-11 00:16:29,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +18: [2023-05-11 00:16:29,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +18: [2023-05-11 00:16:29,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 0: [2023-05-11 00:16:29,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +21: [2023-05-11 00:16:29,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +18: [2023-05-11 00:16:29,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +18: [2023-05-11 00:16:29,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +22: [2023-05-11 00:16:29,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +22: [2023-05-11 00:16:29,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +26: [2023-05-11 00:16:29,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 9: [2023-05-11 00:16:29,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +23: [2023-05-11 00:16:29,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +23: [2023-05-11 00:16:29,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +23: [2023-05-11 00:16:29,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +22: [2023-05-11 00:16:29,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +23: [2023-05-11 00:16:29,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +29: [2023-05-11 00:16:29,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +26: [2023-05-11 00:16:29,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 3: [2023-05-11 00:16:29,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 2: [2023-05-11 00:16:29,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +26: [2023-05-11 00:16:29,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +18: [2023-05-11 00:16:29,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +18: [2023-05-11 00:16:29,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +18: [2023-05-11 00:16:29,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +18: [2023-05-11 00:16:29,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 3: [2023-05-11 00:16:29,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +26: [2023-05-11 00:16:29,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +22: [2023-05-11 00:16:29,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +23: [2023-05-11 00:16:29,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +29: [2023-05-11 00:16:29,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +29: [2023-05-11 00:16:29,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +29: [2023-05-11 00:16:29,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +23: [2023-05-11 00:16:29,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +26: [2023-05-11 00:16:29,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +29: [2023-05-11 00:16:29,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 6: [2023-05-11 00:16:29,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 3: [2023-05-11 00:16:29,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 6: [2023-05-11 00:16:29,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +26: [2023-05-11 00:16:29,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +29: [2023-05-11 00:16:29,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +29: [2023-05-11 00:16:29,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +29: [2023-05-11 00:16:29,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +30: [2023-05-11 00:16:29,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +30: [2023-05-11 00:16:29,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +30: [2023-05-11 00:16:29,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +30: [2023-05-11 00:16:29,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +30: [2023-05-11 00:16:29,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +30: [2023-05-11 00:16:29,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +30: [2023-05-11 00:16:29,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +30: [2023-05-11 00:16:29,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +30: [2023-05-11 00:16:29,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +15: [2023-05-11 00:16:29,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +15: [2023-05-11 00:16:29,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 3: [2023-05-11 00:16:29,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +15: [2023-05-11 00:16:29,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +30: [2023-05-11 00:16:29,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +15: [2023-05-11 00:16:29,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +15: [2023-05-11 00:16:29,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +30: [2023-05-11 00:16:29,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +15: [2023-05-11 00:16:29,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +15: [2023-05-11 00:16:29,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +30: [2023-05-11 00:16:29,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +15: [2023-05-11 00:16:29,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +15: [2023-05-11 00:16:29,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +15: [2023-05-11 00:16:29,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 3: [2023-05-11 00:16:29,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +30: [2023-05-11 00:16:29,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +15: [2023-05-11 00:16:29,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +30: [2023-05-11 00:16:29,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +15: [2023-05-11 00:16:29,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +15: [2023-05-11 00:16:29,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +15: [2023-05-11 00:16:29,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +15: [2023-05-11 00:16:29,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +30: [2023-05-11 00:16:29,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +15: [2023-05-11 00:16:29,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +30: [2023-05-11 00:16:29,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 3: [2023-05-11 00:16:29,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +14: [2023-05-11 00:16:29,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +30: [2023-05-11 00:16:29,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +15: [2023-05-11 00:16:29,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +14: [2023-05-11 00:16:29,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +14: [2023-05-11 00:16:29,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +30: [2023-05-11 00:16:29,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +14: [2023-05-11 00:16:29,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +15: [2023-05-11 00:16:29,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +30: [2023-05-11 00:16:29,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +15: [2023-05-11 00:16:29,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +18: [2023-05-11 00:16:29,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +18: [2023-05-11 00:16:29,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +18: [2023-05-11 00:16:29,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +18: [2023-05-11 00:16:29,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +18: [2023-05-11 00:16:29,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +18: [2023-05-11 00:16:29,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +18: [2023-05-11 00:16:29,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +18: [2023-05-11 00:16:29,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 3: [2023-05-11 00:16:29,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +15: [2023-05-11 00:16:29,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +11: [2023-05-11 00:16:29,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +11: [2023-05-11 00:16:29,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +30: [2023-05-11 00:16:29,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +18: [2023-05-11 00:16:29,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +18: [2023-05-11 00:16:29,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +11: [2023-05-11 00:16:29,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 7: [2023-05-11 00:16:29,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +18: [2023-05-11 00:16:29,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +11: [2023-05-11 00:16:29,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +18: [2023-05-11 00:16:29,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 7: [2023-05-11 00:16:29,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +11: [2023-05-11 00:16:29,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +11: [2023-05-11 00:16:29,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +18: [2023-05-11 00:16:29,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +18: [2023-05-11 00:16:29,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +30: [2023-05-11 00:16:29,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +11: [2023-05-11 00:16:29,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +11: [2023-05-11 00:16:29,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +11: [2023-05-11 00:16:29,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +11: [2023-05-11 00:16:29,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +30: [2023-05-11 00:16:29,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +14: [2023-05-11 00:16:29,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +11: [2023-05-11 00:16:29,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +14: [2023-05-11 00:16:29,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +14: [2023-05-11 00:16:29,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +18: [2023-05-11 00:16:29,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +11: [2023-05-11 00:16:29,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +30: [2023-05-11 00:16:29,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +18: [2023-05-11 00:16:29,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +14: [2023-05-11 00:16:29,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +11: [2023-05-11 00:16:29,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +11: [2023-05-11 00:16:29,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 7: [2023-05-11 00:16:29,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 7: [2023-05-11 00:16:29,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 3: [2023-05-11 00:16:29,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +11: [2023-05-11 00:16:29,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... + 7: [2023-05-11 00:16:29,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +11: [2023-05-11 00:16:29,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +15: [2023-05-11 00:16:29,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +15: [2023-05-11 00:16:29,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +15: [2023-05-11 00:16:29,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +30: [2023-05-11 00:16:29,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +15: [2023-05-11 00:16:29,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +18: [2023-05-11 00:16:29,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +18: [2023-05-11 00:16:29,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +18: [2023-05-11 00:16:29,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +19: [2023-05-11 00:16:29,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +19: [2023-05-11 00:16:29,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 7: [2023-05-11 00:16:29,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +19: [2023-05-11 00:16:29,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +19: [2023-05-11 00:16:29,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +25: [2023-05-11 00:16:29,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +25: [2023-05-11 00:16:29,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +25: [2023-05-11 00:16:29,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +25: [2023-05-11 00:16:29,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +25: [2023-05-11 00:16:29,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +25: [2023-05-11 00:16:29,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +10: [2023-05-11 00:16:29,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +25: [2023-05-11 00:16:29,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +11: [2023-05-11 00:16:29,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +10: [2023-05-11 00:16:29,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +10: [2023-05-11 00:16:29,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +10: [2023-05-11 00:16:29,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +25: [2023-05-11 00:16:29,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +10: [2023-05-11 00:16:29,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +10: [2023-05-11 00:16:29,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +10: [2023-05-11 00:16:29,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +25: [2023-05-11 00:16:29,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +10: [2023-05-11 00:16:29,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +10: [2023-05-11 00:16:29,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +10: [2023-05-11 00:16:29,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +10: [2023-05-11 00:16:29,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +10: [2023-05-11 00:16:29,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +26: [2023-05-11 00:16:29,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +20: [2023-05-11 00:16:29,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +26: [2023-05-11 00:16:29,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +20: [2023-05-11 00:16:29,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +20: [2023-05-11 00:16:29,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +20: [2023-05-11 00:16:29,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +25: [2023-05-11 00:16:29,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +10: [2023-05-11 00:16:29,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +10: [2023-05-11 00:16:29,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +10: [2023-05-11 00:16:29,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +25: [2023-05-11 00:16:29,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +10: [2023-05-11 00:16:29,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +25: [2023-05-11 00:16:29,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 1: [2023-05-11 00:16:29,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +26: [2023-05-11 00:16:29,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 7: [2023-05-11 00:16:29,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +26: [2023-05-11 00:16:29,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 1: [2023-05-11 00:16:29,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +18: [2023-05-11 00:16:29,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +18: [2023-05-11 00:16:29,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +11: [2023-05-11 00:16:29,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +11: [2023-05-11 00:16:29,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +18: [2023-05-11 00:16:29,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +19: [2023-05-11 00:16:29,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +19: [2023-05-11 00:16:29,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +19: [2023-05-11 00:16:29,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +19: [2023-05-11 00:16:29,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 8: [2023-05-11 00:16:29,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 8: [2023-05-11 00:16:29,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +11: [2023-05-11 00:16:29,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +25: [2023-05-11 00:16:29,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +18: [2023-05-11 00:16:29,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +26: [2023-05-11 00:16:29,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +20: [2023-05-11 00:16:29,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +26: [2023-05-11 00:16:29,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +20: [2023-05-11 00:16:29,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +20: [2023-05-11 00:16:29,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +10: [2023-05-11 00:16:29,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +10: [2023-05-11 00:16:29,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +25: [2023-05-11 00:16:29,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +11: [2023-05-11 00:16:29,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +25: [2023-05-11 00:16:29,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +10: [2023-05-11 00:16:29,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +26: [2023-05-11 00:16:29,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +11: [2023-05-11 00:16:29,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +25: [2023-05-11 00:16:29,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 6: [2023-05-11 00:16:29,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 6: [2023-05-11 00:16:29,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 6: [2023-05-11 00:16:29,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +11: [2023-05-11 00:16:29,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +20: [2023-05-11 00:16:29,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +18: [2023-05-11 00:16:29,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +26: [2023-05-11 00:16:29,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +25: [2023-05-11 00:16:29,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 6: [2023-05-11 00:16:29,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +25: [2023-05-11 00:16:29,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +19: [2023-05-11 00:16:29,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +19: [2023-05-11 00:16:29,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +19: [2023-05-11 00:16:29,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +19: [2023-05-11 00:16:29,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 2: [2023-05-11 00:16:29,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 2: [2023-05-11 00:16:29,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 7: [2023-05-11 00:16:29,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +10: [2023-05-11 00:16:29,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +14: [2023-05-11 00:16:29,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +25: [2023-05-11 00:16:29,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +14: [2023-05-11 00:16:29,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +25: [2023-05-11 00:16:29,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +10: [2023-05-11 00:16:29,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +14: [2023-05-11 00:16:29,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +14: [2023-05-11 00:16:29,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +19: [2023-05-11 00:16:29,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +19: [2023-05-11 00:16:29,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +19: [2023-05-11 00:16:29,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +25: [2023-05-11 00:16:29,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +25: [2023-05-11 00:16:29,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +25: [2023-05-11 00:16:29,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +19: [2023-05-11 00:16:29,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +19: [2023-05-11 00:16:29,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 0: [2023-05-11 00:16:29,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +14: [2023-05-11 00:16:29,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +10: [2023-05-11 00:16:29,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +14: [2023-05-11 00:16:29,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +14: [2023-05-11 00:16:29,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +14: [2023-05-11 00:16:29,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +14: [2023-05-11 00:16:29,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +14: [2023-05-11 00:16:29,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +14: [2023-05-11 00:16:29,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +14: [2023-05-11 00:16:29,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +14: [2023-05-11 00:16:29,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +14: [2023-05-11 00:16:29,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +14: [2023-05-11 00:16:29,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +19: [2023-05-11 00:16:29,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +14: [2023-05-11 00:16:29,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +11: [2023-05-11 00:16:29,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 0: [2023-05-11 00:16:29,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 6: [2023-05-11 00:16:29,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 7: [2023-05-11 00:16:29,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 8: [2023-05-11 00:16:29,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +25: [2023-05-11 00:16:29,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +25: [2023-05-11 00:16:29,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +19: [2023-05-11 00:16:29,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +19: [2023-05-11 00:16:29,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +10: [2023-05-11 00:16:29,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +19: [2023-05-11 00:16:29,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +25: [2023-05-11 00:16:29,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +25: [2023-05-11 00:16:29,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +23: [2023-05-11 00:16:29,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +23: [2023-05-11 00:16:29,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +26: [2023-05-11 00:16:29,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +26: [2023-05-11 00:16:29,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +19: [2023-05-11 00:16:29,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +19: [2023-05-11 00:16:29,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +26: [2023-05-11 00:16:29,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +26: [2023-05-11 00:16:29,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 2: [2023-05-11 00:16:29,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +25: [2023-05-11 00:16:29,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +19: [2023-05-11 00:16:29,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +19: [2023-05-11 00:16:29,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 8: [2023-05-11 00:16:29,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 8: [2023-05-11 00:16:29,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 8: [2023-05-11 00:16:29,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +23: [2023-05-11 00:16:29,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +23: [2023-05-11 00:16:29,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +10: [2023-05-11 00:16:29,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +26: [2023-05-11 00:16:29,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +26: [2023-05-11 00:16:29,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +26: [2023-05-11 00:16:29,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +26: [2023-05-11 00:16:29,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +26: [2023-05-11 00:16:29,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +26: [2023-05-11 00:16:29,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +26: [2023-05-11 00:16:29,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +26: [2023-05-11 00:16:29,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +26: [2023-05-11 00:16:29,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +26: [2023-05-11 00:16:29,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +23: [2023-05-11 00:16:29,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +23: [2023-05-11 00:16:29,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +23: [2023-05-11 00:16:29,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +26: [2023-05-11 00:16:29,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 6: [2023-05-11 00:16:29,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +19: [2023-05-11 00:16:29,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +26: [2023-05-11 00:16:29,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +23: [2023-05-11 00:16:29,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +25: [2023-05-11 00:16:29,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +25: [2023-05-11 00:16:29,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +22: [2023-05-11 00:16:29,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +22: [2023-05-11 00:16:29,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +22: [2023-05-11 00:16:29,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +22: [2023-05-11 00:16:29,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +19: [2023-05-11 00:16:29,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +22: [2023-05-11 00:16:29,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +22: [2023-05-11 00:16:29,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +19: [2023-05-11 00:16:29,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +22: [2023-05-11 00:16:29,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +25: [2023-05-11 00:16:29,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +25: [2023-05-11 00:16:29,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +14: [2023-05-11 00:16:29,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +14: [2023-05-11 00:16:29,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +14: [2023-05-11 00:16:29,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +22: [2023-05-11 00:16:29,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +27: [2023-05-11 00:16:29,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +27: [2023-05-11 00:16:29,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +27: [2023-05-11 00:16:29,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +27: [2023-05-11 00:16:29,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +27: [2023-05-11 00:16:29,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +27: [2023-05-11 00:16:29,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +27: [2023-05-11 00:16:29,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +14: [2023-05-11 00:16:29,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +23: [2023-05-11 00:16:29,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +13: [2023-05-11 00:16:29,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +13: [2023-05-11 00:16:29,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +13: [2023-05-11 00:16:29,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +13: [2023-05-11 00:16:29,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +13: [2023-05-11 00:16:29,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +13: [2023-05-11 00:16:29,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +27: [2023-05-11 00:16:29,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 8: [2023-05-11 00:16:29,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +20: [2023-05-11 00:16:29,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +20: [2023-05-11 00:16:29,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +20: [2023-05-11 00:16:29,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +20: [2023-05-11 00:16:29,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +20: [2023-05-11 00:16:29,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +20: [2023-05-11 00:16:29,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +20: [2023-05-11 00:16:29,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +27: [2023-05-11 00:16:29,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 6: [2023-05-11 00:16:29,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +20: [2023-05-11 00:16:29,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +20: [2023-05-11 00:16:29,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +20: [2023-05-11 00:16:29,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +20: [2023-05-11 00:16:29,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +20: [2023-05-11 00:16:29,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 8: [2023-05-11 00:16:29,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +23: [2023-05-11 00:16:29,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +19: [2023-05-11 00:16:29,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +31: [2023-05-11 00:16:29,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +19: [2023-05-11 00:16:29,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +31: [2023-05-11 00:16:29,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +31: [2023-05-11 00:16:29,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +31: [2023-05-11 00:16:29,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +31: [2023-05-11 00:16:29,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +31: [2023-05-11 00:16:29,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +31: [2023-05-11 00:16:29,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +31: [2023-05-11 00:16:29,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +13: [2023-05-11 00:16:29,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +13: [2023-05-11 00:16:29,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 1: [2023-05-11 00:16:29,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 1: [2023-05-11 00:16:29,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 8: [2023-05-11 00:16:29,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +23: [2023-05-11 00:16:29,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +23: [2023-05-11 00:16:29,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +23: [2023-05-11 00:16:29,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +23: [2023-05-11 00:16:29,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 6: [2023-05-11 00:16:29,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +19: [2023-05-11 00:16:29,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +23: [2023-05-11 00:16:29,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 6: [2023-05-11 00:16:29,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +24: [2023-05-11 00:16:29,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +24: [2023-05-11 00:16:29,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +27: [2023-05-11 00:16:29,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +24: [2023-05-11 00:16:29,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +27: [2023-05-11 00:16:29,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +27: [2023-05-11 00:16:29,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 0: [2023-05-11 00:16:29,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +24: [2023-05-11 00:16:29,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +24: [2023-05-11 00:16:29,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 0: [2023-05-11 00:16:29,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 0: [2023-05-11 00:16:29,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +22: [2023-05-11 00:16:29,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +14: [2023-05-11 00:16:29,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +14: [2023-05-11 00:16:29,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +14: [2023-05-11 00:16:29,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +24: [2023-05-11 00:16:29,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +24: [2023-05-11 00:16:29,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +27: [2023-05-11 00:16:29,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +24: [2023-05-11 00:16:29,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +20: [2023-05-11 00:16:29,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +20: [2023-05-11 00:16:29,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +20: [2023-05-11 00:16:29,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +14: [2023-05-11 00:16:29,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +22: [2023-05-11 00:16:29,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +20: [2023-05-11 00:16:29,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +22: [2023-05-11 00:16:29,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +27: [2023-05-11 00:16:29,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +22: [2023-05-11 00:16:29,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 8: [2023-05-11 00:16:29,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +22: [2023-05-11 00:16:29,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +22: [2023-05-11 00:16:29,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +24: [2023-05-11 00:16:29,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +22: [2023-05-11 00:16:29,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 4: [2023-05-11 00:16:29,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 4: [2023-05-11 00:16:29,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 4: [2023-05-11 00:16:29,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 2: [2023-05-11 00:16:29,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +24: [2023-05-11 00:16:29,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +24: [2023-05-11 00:16:29,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +27: [2023-05-11 00:16:29,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +26: [2023-05-11 00:16:29,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +26: [2023-05-11 00:16:29,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +26: [2023-05-11 00:16:29,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +19: [2023-05-11 00:16:29,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +24: [2023-05-11 00:16:29,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +23: [2023-05-11 00:16:29,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +23: [2023-05-11 00:16:29,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +27: [2023-05-11 00:16:29,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +24: [2023-05-11 00:16:29,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +22: [2023-05-11 00:16:29,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +27: [2023-05-11 00:16:29,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +13: [2023-05-11 00:16:29,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +23: [2023-05-11 00:16:29,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +23: [2023-05-11 00:16:29,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +24: [2023-05-11 00:16:29,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +23: [2023-05-11 00:16:29,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +20: [2023-05-11 00:16:29,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +20: [2023-05-11 00:16:29,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +23: [2023-05-11 00:16:29,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +23: [2023-05-11 00:16:29,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +13: [2023-05-11 00:16:29,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +27: [2023-05-11 00:16:29,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +24: [2023-05-11 00:16:29,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +31: [2023-05-11 00:16:29,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 4: [2023-05-11 00:16:29,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +20: [2023-05-11 00:16:29,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +13: [2023-05-11 00:16:29,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +13: [2023-05-11 00:16:29,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 2: [2023-05-11 00:16:29,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +22: [2023-05-11 00:16:29,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +22: [2023-05-11 00:16:29,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +13: [2023-05-11 00:16:29,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +13: [2023-05-11 00:16:29,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 2: [2023-05-11 00:16:29,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 0: [2023-05-11 00:16:29,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +31: [2023-05-11 00:16:29,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +31: [2023-05-11 00:16:29,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +31: [2023-05-11 00:16:29,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +22: [2023-05-11 00:16:29,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +26: [2023-05-11 00:16:29,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +22: [2023-05-11 00:16:29,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +26: [2023-05-11 00:16:29,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +26: [2023-05-11 00:16:29,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +22: [2023-05-11 00:16:29,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +23: [2023-05-11 00:16:29,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +27: [2023-05-11 00:16:29,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +22: [2023-05-11 00:16:29,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +27: [2023-05-11 00:16:29,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +22: [2023-05-11 00:16:29,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +20: [2023-05-11 00:16:29,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +24: [2023-05-11 00:16:29,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +24: [2023-05-11 00:16:29,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +13: [2023-05-11 00:16:29,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +24: [2023-05-11 00:16:29,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +27: [2023-05-11 00:16:29,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +27: [2023-05-11 00:16:29,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +20: [2023-05-11 00:16:29,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +20: [2023-05-11 00:16:29,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +27: [2023-05-11 00:16:29,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +13: [2023-05-11 00:16:29,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +13: [2023-05-11 00:16:29,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +26: [2023-05-11 00:16:29,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +13: [2023-05-11 00:16:29,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +24: [2023-05-11 00:16:29,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +23: [2023-05-11 00:16:29,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +23: [2023-05-11 00:16:29,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 1: [2023-05-11 00:16:29,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +20: [2023-05-11 00:16:29,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +31: [2023-05-11 00:16:29,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +23: [2023-05-11 00:16:29,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +13: [2023-05-11 00:16:29,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +23: [2023-05-11 00:16:29,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +31: [2023-05-11 00:16:29,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +24: [2023-05-11 00:16:29,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +31: [2023-05-11 00:16:29,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +13: [2023-05-11 00:16:29,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +13: [2023-05-11 00:16:29,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +23: [2023-05-11 00:16:29,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +31: [2023-05-11 00:16:29,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +27: [2023-05-11 00:16:29,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +23: [2023-05-11 00:16:29,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +23: [2023-05-11 00:16:29,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +20: [2023-05-11 00:16:29,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +23: [2023-05-11 00:16:29,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +13: [2023-05-11 00:16:29,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +23: [2023-05-11 00:16:29,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +22: [2023-05-11 00:16:29,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +22: [2023-05-11 00:16:29,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +24: [2023-05-11 00:16:29,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +22: [2023-05-11 00:16:29,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +22: [2023-05-11 00:16:29,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +22: [2023-05-11 00:16:29,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +27: [2023-05-11 00:16:29,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +24: [2023-05-11 00:16:29,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +26: [2023-05-11 00:16:29,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +24: [2023-05-11 00:16:29,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +24: [2023-05-11 00:16:29,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +13: [2023-05-11 00:16:29,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +13: [2023-05-11 00:16:29,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +24: [2023-05-11 00:16:29,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +24: [2023-05-11 00:16:29,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +22: [2023-05-11 00:16:29,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +22: [2023-05-11 00:16:29,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +22: [2023-05-11 00:16:29,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +22: [2023-05-11 00:16:29,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +13: [2023-05-11 00:16:29,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +11: [2023-05-11 00:16:29,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +11: [2023-05-11 00:16:29,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 4: [2023-05-11 00:16:29,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +13: [2023-05-11 00:16:29,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 4: [2023-05-11 00:16:29,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +27: [2023-05-11 00:16:29,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +11: [2023-05-11 00:16:29,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +11: [2023-05-11 00:16:29,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +27: [2023-05-11 00:16:29,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +27: [2023-05-11 00:16:29,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +27: [2023-05-11 00:16:29,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +27: [2023-05-11 00:16:29,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +13: [2023-05-11 00:16:29,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +13: [2023-05-11 00:16:29,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +24: [2023-05-11 00:16:29,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +24: [2023-05-11 00:16:29,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +27: [2023-05-11 00:16:29,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +27: [2023-05-11 00:16:29,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +11: [2023-05-11 00:16:29,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +24: [2023-05-11 00:16:29,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +24: [2023-05-11 00:16:29,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +11: [2023-05-11 00:16:29,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +11: [2023-05-11 00:16:29,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +13: [2023-05-11 00:16:29,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +11: [2023-05-11 00:16:29,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +13: [2023-05-11 00:16:29,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 4: [2023-05-11 00:16:29,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +24: [2023-05-11 00:16:29,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +13: [2023-05-11 00:16:29,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +13: [2023-05-11 00:16:29,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 4: [2023-05-11 00:16:29,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 4: [2023-05-11 00:16:29,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +31: [2023-05-11 00:16:29,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +31: [2023-05-11 00:16:29,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +31: [2023-05-11 00:16:29,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +31: [2023-05-11 00:16:29,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 4: [2023-05-11 00:16:29,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +24: [2023-05-11 00:16:29,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +31: [2023-05-11 00:16:29,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +31: [2023-05-11 00:16:29,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +31: [2023-05-11 00:16:29,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +31: [2023-05-11 00:16:29,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +11: [2023-05-11 00:16:29,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +11: [2023-05-11 00:16:29,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +11: [2023-05-11 00:16:29,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +11: [2023-05-11 00:16:29,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +11: [2023-05-11 00:16:29,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +11: [2023-05-11 00:16:29,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +11: [2023-05-11 00:16:29,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +11: [2023-05-11 00:16:29,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +11: [2023-05-11 00:16:29,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +11: [2023-05-11 00:16:29,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +11: [2023-05-11 00:16:29,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +11: [2023-05-11 00:16:29,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +11: [2023-05-11 00:16:29,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +11: [2023-05-11 00:16:29,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +11: [2023-05-11 00:16:29,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +11: [2023-05-11 00:16:29,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +31: [2023-05-11 00:16:29,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +31: [2023-05-11 00:16:29,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +11: [2023-05-11 00:16:29,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +11: [2023-05-11 00:16:29,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +31: [2023-05-11 00:16:29,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +31: [2023-05-11 00:16:29,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +31: [2023-05-11 00:16:29,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +31: [2023-05-11 00:16:29,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +21: [2023-05-11 00:16:29,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +21: [2023-05-11 00:16:29,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +21: [2023-05-11 00:16:29,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +21: [2023-05-11 00:16:29,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +21: [2023-05-11 00:16:29,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +21: [2023-05-11 00:16:29,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +11: [2023-05-11 00:16:29,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +21: [2023-05-11 00:16:29,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +21: [2023-05-11 00:16:29,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +11: [2023-05-11 00:16:29,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +11: [2023-05-11 00:16:29,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +31: [2023-05-11 00:16:29,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +28: [2023-05-11 00:16:29,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +31: [2023-05-11 00:16:29,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +28: [2023-05-11 00:16:29,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +28: [2023-05-11 00:16:29,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +28: [2023-05-11 00:16:29,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +12: [2023-05-11 00:16:29,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +12: [2023-05-11 00:16:29,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +12: [2023-05-11 00:16:29,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +12: [2023-05-11 00:16:29,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +12: [2023-05-11 00:16:29,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +12: [2023-05-11 00:16:29,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +12: [2023-05-11 00:16:29,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +28: [2023-05-11 00:16:29,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +28: [2023-05-11 00:16:29,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +28: [2023-05-11 00:16:29,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +12: [2023-05-11 00:16:29,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +28: [2023-05-11 00:16:29,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +11: [2023-05-11 00:16:29,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +21: [2023-05-11 00:16:29,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +11: [2023-05-11 00:16:29,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 5: [2023-05-11 00:16:29,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 5: [2023-05-11 00:16:29,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 5: [2023-05-11 00:16:29,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +11: [2023-05-11 00:16:29,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 8: [2023-05-11 00:16:29,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 8: [2023-05-11 00:16:29,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 8: [2023-05-11 00:16:29,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 9: [2023-05-11 00:16:29,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 9: [2023-05-11 00:16:29,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 9: [2023-05-11 00:16:29,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 9: [2023-05-11 00:16:29,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +21: [2023-05-11 00:16:29,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +28: [2023-05-11 00:16:29,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +12: [2023-05-11 00:16:29,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +21: [2023-05-11 00:16:29,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +21: [2023-05-11 00:16:29,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +21: [2023-05-11 00:16:29,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +21: [2023-05-11 00:16:29,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +21: [2023-05-11 00:16:29,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +28: [2023-05-11 00:16:29,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +28: [2023-05-11 00:16:29,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 0: [2023-05-11 00:16:29,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 0: [2023-05-11 00:16:29,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 5: [2023-05-11 00:16:29,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +12: [2023-05-11 00:16:29,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +12: [2023-05-11 00:16:29,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +28: [2023-05-11 00:16:29,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +26: [2023-05-11 00:16:29,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 9: [2023-05-11 00:16:29,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +26: [2023-05-11 00:16:29,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +26: [2023-05-11 00:16:29,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +21: [2023-05-11 00:16:29,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +15: [2023-05-11 00:16:29,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +15: [2023-05-11 00:16:29,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +15: [2023-05-11 00:16:29,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +21: [2023-05-11 00:16:29,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +21: [2023-05-11 00:16:29,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +26: [2023-05-11 00:16:29,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +15: [2023-05-11 00:16:29,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 9: [2023-05-11 00:16:29,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +21: [2023-05-11 00:16:29,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 2: [2023-05-11 00:16:29,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 2: [2023-05-11 00:16:29,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 2: [2023-05-11 00:16:29,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +21: [2023-05-11 00:16:29,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +28: [2023-05-11 00:16:29,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +28: [2023-05-11 00:16:29,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +29: [2023-05-11 00:16:29,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +29: [2023-05-11 00:16:29,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +28: [2023-05-11 00:16:29,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +29: [2023-05-11 00:16:29,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 0: [2023-05-11 00:16:29,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +29: [2023-05-11 00:16:29,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +21: [2023-05-11 00:16:29,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +23: [2023-05-11 00:16:29,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +23: [2023-05-11 00:16:29,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +23: [2023-05-11 00:16:29,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +23: [2023-05-11 00:16:29,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +12: [2023-05-11 00:16:29,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +12: [2023-05-11 00:16:29,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +12: [2023-05-11 00:16:29,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +26: [2023-05-11 00:16:29,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +21: [2023-05-11 00:16:29,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +26: [2023-05-11 00:16:29,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +26: [2023-05-11 00:16:29,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +15: [2023-05-11 00:16:29,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +15: [2023-05-11 00:16:29,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +15: [2023-05-11 00:16:29,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +15: [2023-05-11 00:16:29,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +12: [2023-05-11 00:16:29,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +26: [2023-05-11 00:16:29,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +28: [2023-05-11 00:16:29,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +21: [2023-05-11 00:16:29,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +22: [2023-05-11 00:16:29,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +22: [2023-05-11 00:16:29,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +22: [2023-05-11 00:16:29,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +21: [2023-05-11 00:16:29,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +21: [2023-05-11 00:16:29,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +23: [2023-05-11 00:16:29,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +23: [2023-05-11 00:16:29,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +22: [2023-05-11 00:16:29,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +23: [2023-05-11 00:16:29,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +21: [2023-05-11 00:16:29,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +21: [2023-05-11 00:16:29,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +29: [2023-05-11 00:16:29,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +29: [2023-05-11 00:16:29,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +21: [2023-05-11 00:16:29,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +23: [2023-05-11 00:16:29,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +29: [2023-05-11 00:16:29,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +12: [2023-05-11 00:16:29,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +29: [2023-05-11 00:16:29,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +28: [2023-05-11 00:16:29,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +28: [2023-05-11 00:16:29,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +28: [2023-05-11 00:16:29,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +28: [2023-05-11 00:16:29,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +21: [2023-05-11 00:16:29,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +22: [2023-05-11 00:16:29,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +22: [2023-05-11 00:16:29,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +21: [2023-05-11 00:16:29,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +22: [2023-05-11 00:16:29,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +22: [2023-05-11 00:16:29,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +21: [2023-05-11 00:16:29,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +21: [2023-05-11 00:16:29,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +28: [2023-05-11 00:16:29,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +28: [2023-05-11 00:16:29,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +26: [2023-05-11 00:16:29,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +26: [2023-05-11 00:16:29,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +26: [2023-05-11 00:16:29,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +26: [2023-05-11 00:16:29,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +28: [2023-05-11 00:16:29,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +26: [2023-05-11 00:16:29,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +28: [2023-05-11 00:16:29,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +26: [2023-05-11 00:16:29,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +26: [2023-05-11 00:16:29,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +26: [2023-05-11 00:16:29,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +26: [2023-05-11 00:16:29,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +26: [2023-05-11 00:16:29,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +26: [2023-05-11 00:16:29,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +26: [2023-05-11 00:16:29,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +26: [2023-05-11 00:16:29,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 9: [2023-05-11 00:16:29,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 9: [2023-05-11 00:16:29,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 9: [2023-05-11 00:16:29,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 9: [2023-05-11 00:16:29,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 9: [2023-05-11 00:16:29,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 9: [2023-05-11 00:16:29,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 9: [2023-05-11 00:16:29,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 9: [2023-05-11 00:16:29,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +26: [2023-05-11 00:16:29,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +26: [2023-05-11 00:16:29,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +26: [2023-05-11 00:16:29,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 9: [2023-05-11 00:16:29,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 9: [2023-05-11 00:16:29,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 2: [2023-05-11 00:16:29,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 2: [2023-05-11 00:16:29,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 2: [2023-05-11 00:16:29,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +22: [2023-05-11 00:16:29,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +22: [2023-05-11 00:16:29,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +22: [2023-05-11 00:16:29,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 0: [2023-05-11 00:16:29,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +22: [2023-05-11 00:16:29,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +22: [2023-05-11 00:16:29,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +22: [2023-05-11 00:16:29,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +11: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 8: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +11: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +22: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +11: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +22: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +11: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 0: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +22: [2023-05-11 00:16:29,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +23: [2023-05-11 00:16:29,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +23: [2023-05-11 00:16:29,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +23: [2023-05-11 00:16:29,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +23: [2023-05-11 00:16:29,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +23: [2023-05-11 00:16:29,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +23: [2023-05-11 00:16:29,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +23: [2023-05-11 00:16:29,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +23: [2023-05-11 00:16:29,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 0: [2023-05-11 00:16:29,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 5: [2023-05-11 00:16:29,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 8: [2023-05-11 00:16:29,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 8: [2023-05-11 00:16:29,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 8: [2023-05-11 00:16:29,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 8: [2023-05-11 00:16:29,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +23: [2023-05-11 00:16:29,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 0: [2023-05-11 00:16:29,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +23: [2023-05-11 00:16:29,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +23: [2023-05-11 00:16:29,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +23: [2023-05-11 00:16:29,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 8: [2023-05-11 00:16:29,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +22: [2023-05-11 00:16:29,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +23: [2023-05-11 00:16:29,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +23: [2023-05-11 00:16:29,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +30: [2023-05-11 00:16:29,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +30: [2023-05-11 00:16:29,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +30: [2023-05-11 00:16:29,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +30: [2023-05-11 00:16:29,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +23: [2023-05-11 00:16:29,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +23: [2023-05-11 00:16:29,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +10: [2023-05-11 00:16:29,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +10: [2023-05-11 00:16:29,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +10: [2023-05-11 00:16:29,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +10: [2023-05-11 00:16:29,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +22: [2023-05-11 00:16:29,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +30: [2023-05-11 00:16:29,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +30: [2023-05-11 00:16:29,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +22: [2023-05-11 00:16:29,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 9: [2023-05-11 00:16:29,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 9: [2023-05-11 00:16:29,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +30: [2023-05-11 00:16:29,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +30: [2023-05-11 00:16:29,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +11: [2023-05-11 00:16:29,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +11: [2023-05-11 00:16:29,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +11: [2023-05-11 00:16:29,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +22: [2023-05-11 00:16:29,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +26: [2023-05-11 00:16:29,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +26: [2023-05-11 00:16:29,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +26: [2023-05-11 00:16:29,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +10: [2023-05-11 00:16:29,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +10: [2023-05-11 00:16:29,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +11: [2023-05-11 00:16:29,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +28: [2023-05-11 00:16:29,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +10: [2023-05-11 00:16:29,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +10: [2023-05-11 00:16:29,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +10: [2023-05-11 00:16:29,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +10: [2023-05-11 00:16:29,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +28: [2023-05-11 00:16:29,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +22: [2023-05-11 00:16:29,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +28: [2023-05-11 00:16:29,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +22: [2023-05-11 00:16:29,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +22: [2023-05-11 00:16:29,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +10: [2023-05-11 00:16:29,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +28: [2023-05-11 00:16:29,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +28: [2023-05-11 00:16:29,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +10: [2023-05-11 00:16:29,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +22: [2023-05-11 00:16:29,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +22: [2023-05-11 00:16:29,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +10: [2023-05-11 00:16:29,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +19: [2023-05-11 00:16:29,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +19: [2023-05-11 00:16:29,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +23: [2023-05-11 00:16:29,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +28: [2023-05-11 00:16:29,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 9: [2023-05-11 00:16:29,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +28: [2023-05-11 00:16:29,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +10: [2023-05-11 00:16:29,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +19: [2023-05-11 00:16:29,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +23: [2023-05-11 00:16:29,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +19: [2023-05-11 00:16:29,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +22: [2023-05-11 00:16:29,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +22: [2023-05-11 00:16:29,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 9: [2023-05-11 00:16:29,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +28: [2023-05-11 00:16:29,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +10: [2023-05-11 00:16:29,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +10: [2023-05-11 00:16:29,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +30: [2023-05-11 00:16:29,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +26: [2023-05-11 00:16:29,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +22: [2023-05-11 00:16:29,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +26: [2023-05-11 00:16:29,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +26: [2023-05-11 00:16:29,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:29,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +30: [2023-05-11 00:16:29,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +22: [2023-05-11 00:16:29,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +23: [2023-05-11 00:16:29,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +23: [2023-05-11 00:16:29,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +22: [2023-05-11 00:16:29,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +19: [2023-05-11 00:16:29,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +19: [2023-05-11 00:16:29,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +19: [2023-05-11 00:16:29,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +30: [2023-05-11 00:16:29,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +19: [2023-05-11 00:16:29,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +10: [2023-05-11 00:16:29,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +30: [2023-05-11 00:16:29,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +26: [2023-05-11 00:16:29,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +23: [2023-05-11 00:16:29,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +30: [2023-05-11 00:16:29,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +23: [2023-05-11 00:16:29,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +30: [2023-05-11 00:16:29,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 6: [2023-05-11 00:16:29,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 6: [2023-05-11 00:16:29,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +10: [2023-05-11 00:16:29,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +10: [2023-05-11 00:16:29,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +26: [2023-05-11 00:16:29,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +23: [2023-05-11 00:16:29,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +30: [2023-05-11 00:16:29,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +23: [2023-05-11 00:16:29,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 7: [2023-05-11 00:16:29,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 7: [2023-05-11 00:16:29,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +30: [2023-05-11 00:16:29,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 7: [2023-05-11 00:16:29,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +10: [2023-05-11 00:16:29,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +16: [2023-05-11 00:16:29,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +16: [2023-05-11 00:16:29,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +25: [2023-05-11 00:16:29,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +25: [2023-05-11 00:16:29,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +25: [2023-05-11 00:16:29,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +25: [2023-05-11 00:16:29,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +25: [2023-05-11 00:16:29,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +20: [2023-05-11 00:16:29,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +20: [2023-05-11 00:16:29,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +20: [2023-05-11 00:16:29,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +25: [2023-05-11 00:16:29,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +10: [2023-05-11 00:16:29,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +10: [2023-05-11 00:16:29,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +16: [2023-05-11 00:16:29,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +25: [2023-05-11 00:16:29,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +25: [2023-05-11 00:16:29,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +20: [2023-05-11 00:16:29,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +25: [2023-05-11 00:16:29,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +10: [2023-05-11 00:16:29,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +25: [2023-05-11 00:16:29,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 7: [2023-05-11 00:16:29,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +10: [2023-05-11 00:16:29,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 7: [2023-05-11 00:16:29,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +25: [2023-05-11 00:16:29,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +25: [2023-05-11 00:16:29,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +18: [2023-05-11 00:16:29,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +18: [2023-05-11 00:16:29,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +10: [2023-05-11 00:16:29,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +10: [2023-05-11 00:16:29,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 3: [2023-05-11 00:16:29,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +16: [2023-05-11 00:16:29,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +16: [2023-05-11 00:16:29,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +16: [2023-05-11 00:16:29,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +18: [2023-05-11 00:16:29,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +10: [2023-05-11 00:16:29,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +18: [2023-05-11 00:16:29,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +18: [2023-05-11 00:16:29,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +18: [2023-05-11 00:16:29,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +18: [2023-05-11 00:16:29,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +29: [2023-05-11 00:16:29,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +29: [2023-05-11 00:16:29,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +29: [2023-05-11 00:16:29,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +29: [2023-05-11 00:16:29,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +29: [2023-05-11 00:16:29,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +29: [2023-05-11 00:16:29,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +29: [2023-05-11 00:16:29,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +29: [2023-05-11 00:16:29,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +29: [2023-05-11 00:16:29,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +29: [2023-05-11 00:16:29,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +16: [2023-05-11 00:16:29,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +29: [2023-05-11 00:16:29,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +18: [2023-05-11 00:16:29,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +29: [2023-05-11 00:16:29,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +29: [2023-05-11 00:16:29,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +16: [2023-05-11 00:16:29,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +10: [2023-05-11 00:16:29,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 7: [2023-05-11 00:16:29,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +29: [2023-05-11 00:16:29,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +20: [2023-05-11 00:16:29,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +29: [2023-05-11 00:16:29,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +29: [2023-05-11 00:16:29,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +10: [2023-05-11 00:16:29,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +10: [2023-05-11 00:16:29,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +25: [2023-05-11 00:16:29,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +20: [2023-05-11 00:16:29,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +20: [2023-05-11 00:16:29,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +15: [2023-05-11 00:16:29,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +15: [2023-05-11 00:16:29,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +15: [2023-05-11 00:16:29,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +15: [2023-05-11 00:16:29,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 6: [2023-05-11 00:16:29,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +25: [2023-05-11 00:16:29,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +25: [2023-05-11 00:16:29,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +20: [2023-05-11 00:16:29,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +16: [2023-05-11 00:16:29,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +15: [2023-05-11 00:16:29,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +15: [2023-05-11 00:16:29,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +15: [2023-05-11 00:16:29,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +15: [2023-05-11 00:16:29,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +15: [2023-05-11 00:16:29,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +15: [2023-05-11 00:16:29,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +15: [2023-05-11 00:16:29,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +15: [2023-05-11 00:16:29,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +15: [2023-05-11 00:16:29,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +15: [2023-05-11 00:16:29,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +15: [2023-05-11 00:16:29,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +15: [2023-05-11 00:16:29,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +25: [2023-05-11 00:16:29,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +18: [2023-05-11 00:16:29,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +18: [2023-05-11 00:16:29,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +18: [2023-05-11 00:16:29,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +18: [2023-05-11 00:16:29,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +25: [2023-05-11 00:16:29,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +29: [2023-05-11 00:16:29,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +18: [2023-05-11 00:16:29,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +10: [2023-05-11 00:16:29,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +18: [2023-05-11 00:16:29,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +16: [2023-05-11 00:16:29,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +16: [2023-05-11 00:16:29,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +16: [2023-05-11 00:16:29,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +10: [2023-05-11 00:16:29,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +16: [2023-05-11 00:16:29,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +16: [2023-05-11 00:16:29,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +25: [2023-05-11 00:16:29,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +25: [2023-05-11 00:16:29,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +29: [2023-05-11 00:16:29,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 7: [2023-05-11 00:16:29,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +25: [2023-05-11 00:16:29,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +19: [2023-05-11 00:16:29,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +19: [2023-05-11 00:16:29,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +19: [2023-05-11 00:16:29,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +19: [2023-05-11 00:16:29,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +15: [2023-05-11 00:16:29,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +19: [2023-05-11 00:16:29,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +19: [2023-05-11 00:16:29,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +19: [2023-05-11 00:16:29,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +19: [2023-05-11 00:16:29,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +19: [2023-05-11 00:16:29,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +19: [2023-05-11 00:16:29,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +19: [2023-05-11 00:16:29,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +19: [2023-05-11 00:16:29,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 3: [2023-05-11 00:16:29,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +15: [2023-05-11 00:16:29,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +25: [2023-05-11 00:16:29,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 6: [2023-05-11 00:16:29,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +15: [2023-05-11 00:16:29,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +18: [2023-05-11 00:16:29,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +19: [2023-05-11 00:16:29,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +19: [2023-05-11 00:16:29,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 7: [2023-05-11 00:16:29,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +19: [2023-05-11 00:16:29,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +19: [2023-05-11 00:16:29,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +16: [2023-05-11 00:16:29,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +31: [2023-05-11 00:16:29,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +31: [2023-05-11 00:16:29,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +31: [2023-05-11 00:16:29,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +31: [2023-05-11 00:16:29,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +29: [2023-05-11 00:16:29,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +31: [2023-05-11 00:16:29,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +18: [2023-05-11 00:16:29,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +18: [2023-05-11 00:16:29,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +18: [2023-05-11 00:16:29,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +18: [2023-05-11 00:16:29,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +31: [2023-05-11 00:16:29,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +16: [2023-05-11 00:16:29,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +25: [2023-05-11 00:16:29,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +31: [2023-05-11 00:16:29,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +18: [2023-05-11 00:16:29,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +16: [2023-05-11 00:16:29,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +16: [2023-05-11 00:16:29,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +31: [2023-05-11 00:16:29,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +25: [2023-05-11 00:16:29,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 6: [2023-05-11 00:16:29,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +15: [2023-05-11 00:16:29,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 7: [2023-05-11 00:16:29,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +29: [2023-05-11 00:16:29,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +16: [2023-05-11 00:16:29,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +15: [2023-05-11 00:16:29,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +25: [2023-05-11 00:16:29,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +16: [2023-05-11 00:16:29,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +25: [2023-05-11 00:16:29,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +19: [2023-05-11 00:16:29,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +19: [2023-05-11 00:16:29,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +18: [2023-05-11 00:16:29,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +25: [2023-05-11 00:16:29,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +15: [2023-05-11 00:16:29,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +16: [2023-05-11 00:16:29,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +25: [2023-05-11 00:16:29,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +25: [2023-05-11 00:16:29,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +25: [2023-05-11 00:16:29,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 3: [2023-05-11 00:16:29,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +31: [2023-05-11 00:16:29,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +18: [2023-05-11 00:16:29,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +31: [2023-05-11 00:16:29,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +25: [2023-05-11 00:16:29,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 7: [2023-05-11 00:16:29,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +29: [2023-05-11 00:16:29,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +29: [2023-05-11 00:16:29,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 6: [2023-05-11 00:16:29,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +15: [2023-05-11 00:16:29,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +16: [2023-05-11 00:16:29,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +16: [2023-05-11 00:16:29,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +16: [2023-05-11 00:16:29,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +16: [2023-05-11 00:16:29,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +16: [2023-05-11 00:16:29,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +16: [2023-05-11 00:16:29,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 3: [2023-05-11 00:16:29,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +19: [2023-05-11 00:16:29,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +19: [2023-05-11 00:16:29,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +16: [2023-05-11 00:16:29,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +18: [2023-05-11 00:16:29,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +13: [2023-05-11 00:16:29,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +18: [2023-05-11 00:16:29,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +18: [2023-05-11 00:16:29,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +13: [2023-05-11 00:16:29,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +13: [2023-05-11 00:16:29,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +13: [2023-05-11 00:16:29,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +13: [2023-05-11 00:16:29,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +13: [2023-05-11 00:16:29,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +18: [2023-05-11 00:16:29,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +13: [2023-05-11 00:16:29,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 7: [2023-05-11 00:16:29,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +31: [2023-05-11 00:16:29,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +31: [2023-05-11 00:16:29,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +13: [2023-05-11 00:16:29,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +15: [2023-05-11 00:16:29,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +25: [2023-05-11 00:16:29,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +29: [2023-05-11 00:16:29,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +29: [2023-05-11 00:16:29,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 6: [2023-05-11 00:16:29,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +18: [2023-05-11 00:16:29,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +24: [2023-05-11 00:16:29,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +19: [2023-05-11 00:16:29,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +24: [2023-05-11 00:16:29,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +24: [2023-05-11 00:16:29,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 6: [2023-05-11 00:16:29,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +25: [2023-05-11 00:16:29,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +24: [2023-05-11 00:16:29,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 1: [2023-05-11 00:16:29,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +18: [2023-05-11 00:16:29,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 6: [2023-05-11 00:16:29,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 6: [2023-05-11 00:16:29,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +14: [2023-05-11 00:16:29,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +14: [2023-05-11 00:16:29,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +14: [2023-05-11 00:16:29,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +14: [2023-05-11 00:16:29,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +14: [2023-05-11 00:16:29,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +14: [2023-05-11 00:16:29,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +14: [2023-05-11 00:16:29,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +18: [2023-05-11 00:16:29,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +19: [2023-05-11 00:16:29,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +13: [2023-05-11 00:16:29,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +31: [2023-05-11 00:16:29,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +18: [2023-05-11 00:16:29,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +14: [2023-05-11 00:16:29,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +16: [2023-05-11 00:16:29,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +18: [2023-05-11 00:16:29,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +20: [2023-05-11 00:16:29,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +20: [2023-05-11 00:16:29,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +20: [2023-05-11 00:16:29,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +20: [2023-05-11 00:16:29,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +20: [2023-05-11 00:16:29,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +16: [2023-05-11 00:16:29,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +20: [2023-05-11 00:16:29,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +20: [2023-05-11 00:16:29,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +20: [2023-05-11 00:16:29,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +20: [2023-05-11 00:16:29,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +20: [2023-05-11 00:16:29,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +20: [2023-05-11 00:16:29,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +20: [2023-05-11 00:16:29,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +31: [2023-05-11 00:16:29,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +20: [2023-05-11 00:16:29,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +16: [2023-05-11 00:16:29,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +18: [2023-05-11 00:16:29,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +16: [2023-05-11 00:16:29,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +20: [2023-05-11 00:16:29,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +20: [2023-05-11 00:16:29,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +20: [2023-05-11 00:16:29,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +13: [2023-05-11 00:16:29,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +13: [2023-05-11 00:16:29,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +12: [2023-05-11 00:16:29,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +12: [2023-05-11 00:16:29,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +12: [2023-05-11 00:16:29,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +24: [2023-05-11 00:16:29,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +12: [2023-05-11 00:16:29,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +14: [2023-05-11 00:16:29,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +30: [2023-05-11 00:16:29,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +30: [2023-05-11 00:16:29,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 1: [2023-05-11 00:16:29,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +19: [2023-05-11 00:16:29,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +24: [2023-05-11 00:16:29,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +24: [2023-05-11 00:16:29,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 3: [2023-05-11 00:16:29,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 3: [2023-05-11 00:16:29,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 3: [2023-05-11 00:16:29,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 6: [2023-05-11 00:16:29,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +24: [2023-05-11 00:16:29,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +13: [2023-05-11 00:16:29,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +31: [2023-05-11 00:16:29,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +19: [2023-05-11 00:16:29,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +14: [2023-05-11 00:16:29,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +14: [2023-05-11 00:16:29,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +14: [2023-05-11 00:16:29,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +30: [2023-05-11 00:16:29,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +14: [2023-05-11 00:16:29,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +30: [2023-05-11 00:16:29,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 1: [2023-05-11 00:16:29,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +31: [2023-05-11 00:16:29,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +13: [2023-05-11 00:16:29,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +20: [2023-05-11 00:16:29,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +13: [2023-05-11 00:16:29,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +27: [2023-05-11 00:16:29,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +27: [2023-05-11 00:16:29,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +27: [2023-05-11 00:16:29,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +27: [2023-05-11 00:16:29,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +12: [2023-05-11 00:16:29,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +12: [2023-05-11 00:16:29,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +13: [2023-05-11 00:16:29,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +30: [2023-05-11 00:16:29,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +30: [2023-05-11 00:16:29,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +14: [2023-05-11 00:16:29,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +14: [2023-05-11 00:16:29,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +27: [2023-05-11 00:16:29,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +27: [2023-05-11 00:16:29,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +12: [2023-05-11 00:16:29,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +12: [2023-05-11 00:16:29,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +27: [2023-05-11 00:16:29,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +27: [2023-05-11 00:16:29,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +27: [2023-05-11 00:16:29,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 1: [2023-05-11 00:16:29,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +27: [2023-05-11 00:16:29,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +27: [2023-05-11 00:16:29,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 8: [2023-05-11 00:16:29,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +30: [2023-05-11 00:16:29,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +30: [2023-05-11 00:16:29,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +11: [2023-05-11 00:16:29,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +11: [2023-05-11 00:16:29,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +11: [2023-05-11 00:16:29,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +11: [2023-05-11 00:16:29,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +11: [2023-05-11 00:16:29,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +11: [2023-05-11 00:16:29,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +14: [2023-05-11 00:16:29,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +11: [2023-05-11 00:16:29,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +11: [2023-05-11 00:16:29,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +11: [2023-05-11 00:16:29,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +11: [2023-05-11 00:16:29,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +11: [2023-05-11 00:16:29,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +11: [2023-05-11 00:16:29,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +20: [2023-05-11 00:16:29,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +27: [2023-05-11 00:16:29,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +13: [2023-05-11 00:16:29,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +11: [2023-05-11 00:16:29,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +14: [2023-05-11 00:16:29,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +11: [2023-05-11 00:16:29,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +11: [2023-05-11 00:16:29,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +14: [2023-05-11 00:16:29,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +11: [2023-05-11 00:16:29,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +26: [2023-05-11 00:16:29,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +26: [2023-05-11 00:16:29,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +14: [2023-05-11 00:16:29,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +14: [2023-05-11 00:16:29,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +27: [2023-05-11 00:16:29,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +27: [2023-05-11 00:16:29,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +20: [2023-05-11 00:16:29,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +20: [2023-05-11 00:16:29,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +17: [2023-05-11 00:16:29,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +26: [2023-05-11 00:16:29,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 3: [2023-05-11 00:16:29,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +17: [2023-05-11 00:16:29,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +17: [2023-05-11 00:16:29,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +17: [2023-05-11 00:16:29,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +17: [2023-05-11 00:16:29,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +17: [2023-05-11 00:16:29,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +17: [2023-05-11 00:16:29,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +17: [2023-05-11 00:16:29,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +17: [2023-05-11 00:16:29,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +17: [2023-05-11 00:16:29,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +17: [2023-05-11 00:16:29,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt... +17: [2023-05-11 00:16:29,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +17: [2023-05-11 00:16:29,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +14: [2023-05-11 00:16:29,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +14: [2023-05-11 00:16:29,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +26: [2023-05-11 00:16:29,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +17: [2023-05-11 00:16:29,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +12: [2023-05-11 00:16:29,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +17: [2023-05-11 00:16:29,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... +17: [2023-05-11 00:16:29,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +12: [2023-05-11 00:16:29,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 4: [2023-05-11 00:16:29,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +12: [2023-05-11 00:16:29,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +12: [2023-05-11 00:16:29,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +26: [2023-05-11 00:16:29,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +23: [2023-05-11 00:16:29,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +23: [2023-05-11 00:16:29,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +20: [2023-05-11 00:16:29,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +11: [2023-05-11 00:16:29,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +27: [2023-05-11 00:16:29,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +27: [2023-05-11 00:16:29,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +27: [2023-05-11 00:16:29,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +26: [2023-05-11 00:16:29,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +27: [2023-05-11 00:16:29,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +23: [2023-05-11 00:16:29,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +14: [2023-05-11 00:16:29,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +14: [2023-05-11 00:16:29,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +20: [2023-05-11 00:16:29,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +26: [2023-05-11 00:16:29,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +20: [2023-05-11 00:16:29,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +23: [2023-05-11 00:16:29,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 4: [2023-05-11 00:16:29,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +30: [2023-05-11 00:16:29,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 4: [2023-05-11 00:16:29,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +30: [2023-05-11 00:16:29,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 0: [2023-05-11 00:16:29,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +27: [2023-05-11 00:16:29,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +24: [2023-05-11 00:16:29,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +24: [2023-05-11 00:16:29,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 0: [2023-05-11 00:16:29,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +24: [2023-05-11 00:16:29,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +24: [2023-05-11 00:16:29,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 8: [2023-05-11 00:16:29,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +22: [2023-05-11 00:16:29,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +22: [2023-05-11 00:16:29,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +22: [2023-05-11 00:16:29,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +22: [2023-05-11 00:16:29,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +26: [2023-05-11 00:16:29,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +11: [2023-05-11 00:16:29,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +23: [2023-05-11 00:16:29,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +23: [2023-05-11 00:16:29,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +23: [2023-05-11 00:16:29,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +23: [2023-05-11 00:16:29,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +24: [2023-05-11 00:16:29,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +24: [2023-05-11 00:16:29,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +24: [2023-05-11 00:16:29,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +17: [2023-05-11 00:16:29,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +24: [2023-05-11 00:16:29,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +24: [2023-05-11 00:16:29,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +20: [2023-05-11 00:16:29,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +23: [2023-05-11 00:16:29,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +24: [2023-05-11 00:16:29,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +30: [2023-05-11 00:16:29,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +30: [2023-05-11 00:16:29,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 3: [2023-05-11 00:16:29,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +12: [2023-05-11 00:16:29,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +12: [2023-05-11 00:16:29,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +23: [2023-05-11 00:16:29,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +27: [2023-05-11 00:16:29,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +27: [2023-05-11 00:16:29,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +14: [2023-05-11 00:16:29,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +14: [2023-05-11 00:16:29,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +24: [2023-05-11 00:16:29,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +24: [2023-05-11 00:16:29,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +24: [2023-05-11 00:16:29,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 8: [2023-05-11 00:16:29,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +11: [2023-05-11 00:16:29,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +12: [2023-05-11 00:16:29,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +12: [2023-05-11 00:16:29,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +11: [2023-05-11 00:16:29,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +26: [2023-05-11 00:16:29,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 3: [2023-05-11 00:16:29,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +24: [2023-05-11 00:16:29,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +26: [2023-05-11 00:16:29,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +26: [2023-05-11 00:16:29,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +26: [2023-05-11 00:16:29,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +26: [2023-05-11 00:16:29,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +24: [2023-05-11 00:16:29,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +26: [2023-05-11 00:16:29,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 3: [2023-05-11 00:16:29,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +23: [2023-05-11 00:16:29,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +24: [2023-05-11 00:16:29,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +14: [2023-05-11 00:16:29,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +23: [2023-05-11 00:16:29,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +23: [2023-05-11 00:16:29,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +27: [2023-05-11 00:16:29,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +23: [2023-05-11 00:16:29,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +23: [2023-05-11 00:16:29,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +26: [2023-05-11 00:16:29,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +26: [2023-05-11 00:16:29,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +22: [2023-05-11 00:16:29,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +26: [2023-05-11 00:16:29,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 1: [2023-05-11 00:16:29,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 1: [2023-05-11 00:16:29,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 1: [2023-05-11 00:16:29,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +30: [2023-05-11 00:16:29,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +22: [2023-05-11 00:16:29,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +23: [2023-05-11 00:16:29,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +26: [2023-05-11 00:16:29,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +26: [2023-05-11 00:16:29,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 3: [2023-05-11 00:16:29,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 3: [2023-05-11 00:16:29,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 0: [2023-05-11 00:16:29,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +30: [2023-05-11 00:16:29,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 0: [2023-05-11 00:16:29,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +26: [2023-05-11 00:16:29,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +13: [2023-05-11 00:16:29,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +13: [2023-05-11 00:16:29,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +13: [2023-05-11 00:16:29,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +13: [2023-05-11 00:16:29,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +17: [2023-05-11 00:16:29,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +30: [2023-05-11 00:16:29,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +22: [2023-05-11 00:16:29,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +22: [2023-05-11 00:16:29,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +14: [2023-05-11 00:16:29,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +27: [2023-05-11 00:16:29,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +30: [2023-05-11 00:16:29,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +26: [2023-05-11 00:16:29,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +26: [2023-05-11 00:16:29,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 2: [2023-05-11 00:16:29,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 4: [2023-05-11 00:16:29,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +11: [2023-05-11 00:16:29,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +27: [2023-05-11 00:16:29,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +27: [2023-05-11 00:16:29,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +24: [2023-05-11 00:16:29,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +11: [2023-05-11 00:16:29,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +27: [2023-05-11 00:16:29,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +17: [2023-05-11 00:16:29,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +17: [2023-05-11 00:16:29,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +27: [2023-05-11 00:16:29,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +27: [2023-05-11 00:16:29,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +14: [2023-05-11 00:16:29,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 2: [2023-05-11 00:16:29,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +23: [2023-05-11 00:16:29,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +13: [2023-05-11 00:16:29,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +13: [2023-05-11 00:16:29,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +11: [2023-05-11 00:16:29,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +14: [2023-05-11 00:16:29,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +23: [2023-05-11 00:16:29,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 1: [2023-05-11 00:16:29,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +17: [2023-05-11 00:16:29,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt. +26: [2023-05-11 00:16:29,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +26: [2023-05-11 00:16:29,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +13: [2023-05-11 00:16:29,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +23: [2023-05-11 00:16:29,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +23: [2023-05-11 00:16:29,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +14: [2023-05-11 00:16:29,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +22: [2023-05-11 00:16:29,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +22: [2023-05-11 00:16:29,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +23: [2023-05-11 00:16:29,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +22: [2023-05-11 00:16:29,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +23: [2023-05-11 00:16:29,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +22: [2023-05-11 00:16:29,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +22: [2023-05-11 00:16:29,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +22: [2023-05-11 00:16:29,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +23: [2023-05-11 00:16:29,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +23: [2023-05-11 00:16:29,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +14: [2023-05-11 00:16:29,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +22: [2023-05-11 00:16:29,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +22: [2023-05-11 00:16:29,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +22: [2023-05-11 00:16:29,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +22: [2023-05-11 00:16:29,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +13: [2023-05-11 00:16:29,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +17: [2023-05-11 00:16:29,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +17: [2023-05-11 00:16:29,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +11: [2023-05-11 00:16:29,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +23: [2023-05-11 00:16:29,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +23: [2023-05-11 00:16:29,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +24: [2023-05-11 00:16:29,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +22: [2023-05-11 00:16:29,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:29,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 4: [2023-05-11 00:16:29,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +24: [2023-05-11 00:16:29,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +22: [2023-05-11 00:16:29,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +17: [2023-05-11 00:16:29,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +22: [2023-05-11 00:16:29,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 2: [2023-05-11 00:16:29,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +27: [2023-05-11 00:16:29,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +27: [2023-05-11 00:16:29,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +22: [2023-05-11 00:16:29,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +22: [2023-05-11 00:16:29,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +27: [2023-05-11 00:16:29,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 0: [2023-05-11 00:16:29,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +27: [2023-05-11 00:16:29,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +13: [2023-05-11 00:16:29,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +13: [2023-05-11 00:16:29,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +13: [2023-05-11 00:16:29,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +13: [2023-05-11 00:16:29,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 0: [2023-05-11 00:16:29,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +26: [2023-05-11 00:16:29,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +26: [2023-05-11 00:16:29,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +13: [2023-05-11 00:16:29,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +13: [2023-05-11 00:16:29,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +26: [2023-05-11 00:16:29,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +23: [2023-05-11 00:16:29,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +23: [2023-05-11 00:16:29,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +24: [2023-05-11 00:16:29,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +22: [2023-05-11 00:16:29,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +23: [2023-05-11 00:16:29,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +24: [2023-05-11 00:16:29,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:29,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 0: [2023-05-11 00:16:29,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +23: [2023-05-11 00:16:29,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 5: [2023-05-11 00:16:29,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 5: [2023-05-11 00:16:29,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 0: [2023-05-11 00:16:29,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +26: [2023-05-11 00:16:29,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +22: [2023-05-11 00:16:29,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +24: [2023-05-11 00:16:29,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +26: [2023-05-11 00:16:29,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +22: [2023-05-11 00:16:29,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +26: [2023-05-11 00:16:29,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 4: [2023-05-11 00:16:29,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:29,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +24: [2023-05-11 00:16:29,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +23: [2023-05-11 00:16:29,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 4: [2023-05-11 00:16:29,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +23: [2023-05-11 00:16:29,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 1: [2023-05-11 00:16:29,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 1: [2023-05-11 00:16:29,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:29,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 0: [2023-05-11 00:16:29,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 8: [2023-05-11 00:16:29,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 0: [2023-05-11 00:16:29,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 8: [2023-05-11 00:16:29,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 8: [2023-05-11 00:16:29,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 9: [2023-05-11 00:16:29,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 9: [2023-05-11 00:16:29,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 1: [2023-05-11 00:16:29,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 1: [2023-05-11 00:16:29,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 4: [2023-05-11 00:16:29,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 4: [2023-05-11 00:16:29,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 4: [2023-05-11 00:16:29,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 4: [2023-05-11 00:16:29,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +22: [2023-05-11 00:16:29,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +24: [2023-05-11 00:16:29,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:29,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +13: [2023-05-11 00:16:29,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 9: [2023-05-11 00:16:29,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 9: [2023-05-11 00:16:29,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +13: [2023-05-11 00:16:29,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +26: [2023-05-11 00:16:29,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 8: [2023-05-11 00:16:29,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +26: [2023-05-11 00:16:29,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 9: [2023-05-11 00:16:29,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 2: [2023-05-11 00:16:29,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 2: [2023-05-11 00:16:29,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. + 2: [2023-05-11 00:16:29,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +21: [2023-05-11 00:16:29,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +21: [2023-05-11 00:16:29,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +21: [2023-05-11 00:16:29,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +21: [2023-05-11 00:16:29,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +21: [2023-05-11 00:16:29,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +21: [2023-05-11 00:16:29,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +21: [2023-05-11 00:16:29,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +21: [2023-05-11 00:16:29,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +17: [2023-05-11 00:16:29,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. +17: [2023-05-11 00:16:29,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 8: [2023-05-11 00:16:29,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 8: [2023-05-11 00:16:29,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +17: [2023-05-11 00:16:29,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 2: [2023-05-11 00:16:29,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +17: [2023-05-11 00:16:29,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt. + 2: [2023-05-11 00:16:29,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +28: [2023-05-11 00:16:29,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +28: [2023-05-11 00:16:29,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 8: [2023-05-11 00:16:29,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +21: [2023-05-11 00:16:29,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +28: [2023-05-11 00:16:29,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +28: [2023-05-11 00:16:29,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 8: [2023-05-11 00:16:29,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 8: [2023-05-11 00:16:29,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +17: [2023-05-11 00:16:29,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +17: [2023-05-11 00:16:29,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +21: [2023-05-11 00:16:29,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +21: [2023-05-11 00:16:29,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +17: [2023-05-11 00:16:29,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +28: [2023-05-11 00:16:29,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +28: [2023-05-11 00:16:29,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +17: [2023-05-11 00:16:29,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +21: [2023-05-11 00:16:29,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +28: [2023-05-11 00:16:29,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +28: [2023-05-11 00:16:29,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +11: [2023-05-11 00:16:29,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +11: [2023-05-11 00:16:29,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +11: [2023-05-11 00:16:29,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +11: [2023-05-11 00:16:29,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +29: [2023-05-11 00:16:29,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +29: [2023-05-11 00:16:29,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +29: [2023-05-11 00:16:29,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 5: [2023-05-11 00:16:29,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 5: [2023-05-11 00:16:29,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +21: [2023-05-11 00:16:29,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 5: [2023-05-11 00:16:29,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 5: [2023-05-11 00:16:29,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 2: [2023-05-11 00:16:29,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +29: [2023-05-11 00:16:29,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 5: [2023-05-11 00:16:29,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +21: [2023-05-11 00:16:29,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:29,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +11: [2023-05-11 00:16:29,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +11: [2023-05-11 00:16:29,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +11: [2023-05-11 00:16:29,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 5: [2023-05-11 00:16:29,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +21: [2023-05-11 00:16:29,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +21: [2023-05-11 00:16:29,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +21: [2023-05-11 00:16:29,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +21: [2023-05-11 00:16:29,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +17: [2023-05-11 00:16:29,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +17: [2023-05-11 00:16:29,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +17: [2023-05-11 00:16:29,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +17: [2023-05-11 00:16:29,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +17: [2023-05-11 00:16:29,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +17: [2023-05-11 00:16:29,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +17: [2023-05-11 00:16:29,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +17: [2023-05-11 00:16:29,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +17: [2023-05-11 00:16:29,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +17: [2023-05-11 00:16:29,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +17: [2023-05-11 00:16:29,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +17: [2023-05-11 00:16:29,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt... +17: [2023-05-11 00:16:29,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +17: [2023-05-11 00:16:29,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... +21: [2023-05-11 00:16:29,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 2: [2023-05-11 00:16:29,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +29: [2023-05-11 00:16:29,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +21: [2023-05-11 00:16:29,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 2: [2023-05-11 00:16:29,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +29: [2023-05-11 00:16:29,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +11: [2023-05-11 00:16:29,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +11: [2023-05-11 00:16:29,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +21: [2023-05-11 00:16:29,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +21: [2023-05-11 00:16:29,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +21: [2023-05-11 00:16:29,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +11: [2023-05-11 00:16:29,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +29: [2023-05-11 00:16:29,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +12: [2023-05-11 00:16:29,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +12: [2023-05-11 00:16:29,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +17: [2023-05-11 00:16:29,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +17: [2023-05-11 00:16:29,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt... + 5: [2023-05-11 00:16:29,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 5: [2023-05-11 00:16:29,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +10: [2023-05-11 00:16:29,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +10: [2023-05-11 00:16:29,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +12: [2023-05-11 00:16:29,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +11: [2023-05-11 00:16:29,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +12: [2023-05-11 00:16:29,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +12: [2023-05-11 00:16:29,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +12: [2023-05-11 00:16:29,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +12: [2023-05-11 00:16:29,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +12: [2023-05-11 00:16:29,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +29: [2023-05-11 00:16:29,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +10: [2023-05-11 00:16:29,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +10: [2023-05-11 00:16:29,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +21: [2023-05-11 00:16:29,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +17: [2023-05-11 00:16:30,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +17: [2023-05-11 00:16:30,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +11: [2023-05-11 00:16:30,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +17: [2023-05-11 00:16:30,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +10: [2023-05-11 00:16:30,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 5: [2023-05-11 00:16:30,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +31: [2023-05-11 00:16:30,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +31: [2023-05-11 00:16:30,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +31: [2023-05-11 00:16:30,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +31: [2023-05-11 00:16:30,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 5: [2023-05-11 00:16:30,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 5: [2023-05-11 00:16:30,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:30,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +21: [2023-05-11 00:16:30,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +11: [2023-05-11 00:16:30,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +11: [2023-05-11 00:16:30,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +21: [2023-05-11 00:16:30,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +11: [2023-05-11 00:16:30,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +11: [2023-05-11 00:16:30,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +11: [2023-05-11 00:16:30,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +10: [2023-05-11 00:16:30,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +21: [2023-05-11 00:16:30,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +21: [2023-05-11 00:16:30,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 5: [2023-05-11 00:16:30,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +21: [2023-05-11 00:16:30,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +21: [2023-05-11 00:16:30,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +10: [2023-05-11 00:16:30,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +12: [2023-05-11 00:16:30,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +11: [2023-05-11 00:16:30,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +28: [2023-05-11 00:16:30,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +28: [2023-05-11 00:16:30,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +28: [2023-05-11 00:16:30,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +28: [2023-05-11 00:16:30,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +28: [2023-05-11 00:16:30,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +28: [2023-05-11 00:16:30,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +28: [2023-05-11 00:16:30,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +28: [2023-05-11 00:16:30,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +28: [2023-05-11 00:16:30,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +17: [2023-05-11 00:16:30,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +28: [2023-05-11 00:16:30,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +28: [2023-05-11 00:16:30,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +28: [2023-05-11 00:16:30,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +28: [2023-05-11 00:16:30,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +31: [2023-05-11 00:16:30,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +28: [2023-05-11 00:16:30,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +17: [2023-05-11 00:16:30,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt. +28: [2023-05-11 00:16:30,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +28: [2023-05-11 00:16:30,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +31: [2023-05-11 00:16:30,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +17: [2023-05-11 00:16:30,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +17: [2023-05-11 00:16:30,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +10: [2023-05-11 00:16:30,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +11: [2023-05-11 00:16:30,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +11: [2023-05-11 00:16:30,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +31: [2023-05-11 00:16:30,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +12: [2023-05-11 00:16:30,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +31: [2023-05-11 00:16:30,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +11: [2023-05-11 00:16:30,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +12: [2023-05-11 00:16:30,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +21: [2023-05-11 00:16:30,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +10: [2023-05-11 00:16:30,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +10: [2023-05-11 00:16:30,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +10: [2023-05-11 00:16:30,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +10: [2023-05-11 00:16:30,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +11: [2023-05-11 00:16:30,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +21: [2023-05-11 00:16:30,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +10: [2023-05-11 00:16:30,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +10: [2023-05-11 00:16:30,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +17: [2023-05-11 00:16:30,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +12: [2023-05-11 00:16:30,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 6: [2023-05-11 00:16:30,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +10: [2023-05-11 00:16:30,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +10: [2023-05-11 00:16:30,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +28: [2023-05-11 00:16:30,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +28: [2023-05-11 00:16:30,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +10: [2023-05-11 00:16:30,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +28: [2023-05-11 00:16:30,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +10: [2023-05-11 00:16:30,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +10: [2023-05-11 00:16:30,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +12: [2023-05-11 00:16:30,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 6: [2023-05-11 00:16:30,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +31: [2023-05-11 00:16:30,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +12: [2023-05-11 00:16:30,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +10: [2023-05-11 00:16:30,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +31: [2023-05-11 00:16:30,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +19: [2023-05-11 00:16:30,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +19: [2023-05-11 00:16:30,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +19: [2023-05-11 00:16:30,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +19: [2023-05-11 00:16:30,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +11: [2023-05-11 00:16:30,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +11: [2023-05-11 00:16:30,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +11: [2023-05-11 00:16:30,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +10: [2023-05-11 00:16:30,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +11: [2023-05-11 00:16:30,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +11: [2023-05-11 00:16:30,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +11: [2023-05-11 00:16:30,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +12: [2023-05-11 00:16:30,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +10: [2023-05-11 00:16:30,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +10: [2023-05-11 00:16:30,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +28: [2023-05-11 00:16:30,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +28: [2023-05-11 00:16:30,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +28: [2023-05-11 00:16:30,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +10: [2023-05-11 00:16:30,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +31: [2023-05-11 00:16:30,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +10: [2023-05-11 00:16:30,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +19: [2023-05-11 00:16:30,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +19: [2023-05-11 00:16:30,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +19: [2023-05-11 00:16:30,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +11: [2023-05-11 00:16:30,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +19: [2023-05-11 00:16:30,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +28: [2023-05-11 00:16:30,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +31: [2023-05-11 00:16:30,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +11: [2023-05-11 00:16:30,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... + 6: [2023-05-11 00:16:30,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +12: [2023-05-11 00:16:30,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +10: [2023-05-11 00:16:30,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +10: [2023-05-11 00:16:30,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +10: [2023-05-11 00:16:30,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +31: [2023-05-11 00:16:30,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +10: [2023-05-11 00:16:30,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +31: [2023-05-11 00:16:30,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +28: [2023-05-11 00:16:30,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +10: [2023-05-11 00:16:30,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +31: [2023-05-11 00:16:30,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +31: [2023-05-11 00:16:30,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +26: [2023-05-11 00:16:30,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +26: [2023-05-11 00:16:30,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +26: [2023-05-11 00:16:30,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +26: [2023-05-11 00:16:30,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +26: [2023-05-11 00:16:30,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +26: [2023-05-11 00:16:30,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +26: [2023-05-11 00:16:30,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +26: [2023-05-11 00:16:30,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +15: [2023-05-11 00:16:30,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +22: [2023-05-11 00:16:30,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +22: [2023-05-11 00:16:30,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +22: [2023-05-11 00:16:30,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +22: [2023-05-11 00:16:30,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +22: [2023-05-11 00:16:30,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +22: [2023-05-11 00:16:30,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +22: [2023-05-11 00:16:30,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +22: [2023-05-11 00:16:30,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +15: [2023-05-11 00:16:30,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +26: [2023-05-11 00:16:30,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +26: [2023-05-11 00:16:30,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +26: [2023-05-11 00:16:30,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +15: [2023-05-11 00:16:30,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +22: [2023-05-11 00:16:30,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +22: [2023-05-11 00:16:30,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +22: [2023-05-11 00:16:30,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +22: [2023-05-11 00:16:30,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +23: [2023-05-11 00:16:30,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +23: [2023-05-11 00:16:30,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +23: [2023-05-11 00:16:30,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +23: [2023-05-11 00:16:30,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +23: [2023-05-11 00:16:30,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +23: [2023-05-11 00:16:30,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +23: [2023-05-11 00:16:30,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +26: [2023-05-11 00:16:30,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +15: [2023-05-11 00:16:30,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +23: [2023-05-11 00:16:30,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +10: [2023-05-11 00:16:30,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 7: [2023-05-11 00:16:30,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 7: [2023-05-11 00:16:30,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 7: [2023-05-11 00:16:30,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 0: [2023-05-11 00:16:30,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 0: [2023-05-11 00:16:30,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 0: [2023-05-11 00:16:30,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +26: [2023-05-11 00:16:30,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +15: [2023-05-11 00:16:30,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +25: [2023-05-11 00:16:30,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +26: [2023-05-11 00:16:30,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +25: [2023-05-11 00:16:30,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +25: [2023-05-11 00:16:30,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +25: [2023-05-11 00:16:30,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +25: [2023-05-11 00:16:30,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +25: [2023-05-11 00:16:30,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +25: [2023-05-11 00:16:30,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +25: [2023-05-11 00:16:30,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +15: [2023-05-11 00:16:30,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +15: [2023-05-11 00:16:30,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +10: [2023-05-11 00:16:30,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +22: [2023-05-11 00:16:30,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +22: [2023-05-11 00:16:30,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +26: [2023-05-11 00:16:30,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +15: [2023-05-11 00:16:30,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:30,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +22: [2023-05-11 00:16:30,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +23: [2023-05-11 00:16:30,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +23: [2023-05-11 00:16:30,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +22: [2023-05-11 00:16:30,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +22: [2023-05-11 00:16:30,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +18: [2023-05-11 00:16:30,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 0: [2023-05-11 00:16:30,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +26: [2023-05-11 00:16:30,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +22: [2023-05-11 00:16:30,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +22: [2023-05-11 00:16:30,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +18: [2023-05-11 00:16:30,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 8: [2023-05-11 00:16:30,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 9: [2023-05-11 00:16:30,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 9: [2023-05-11 00:16:30,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 9: [2023-05-11 00:16:30,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 9: [2023-05-11 00:16:30,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +23: [2023-05-11 00:16:30,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +18: [2023-05-11 00:16:30,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +26: [2023-05-11 00:16:30,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +26: [2023-05-11 00:16:30,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 2: [2023-05-11 00:16:30,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +18: [2023-05-11 00:16:30,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 8: [2023-05-11 00:16:30,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +17: [2023-05-11 00:16:30,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +17: [2023-05-11 00:16:30,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. + 0: [2023-05-11 00:16:30,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +25: [2023-05-11 00:16:30,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +25: [2023-05-11 00:16:30,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +17: [2023-05-11 00:16:30,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +26: [2023-05-11 00:16:30,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +26: [2023-05-11 00:16:30,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +26: [2023-05-11 00:16:30,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +17: [2023-05-11 00:16:30,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt. +26: [2023-05-11 00:16:30,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +26: [2023-05-11 00:16:30,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 2: [2023-05-11 00:16:30,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:30,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +18: [2023-05-11 00:16:30,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +23: [2023-05-11 00:16:30,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +25: [2023-05-11 00:16:30,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +25: [2023-05-11 00:16:30,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +23: [2023-05-11 00:16:30,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:30,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:30,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:30,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +18: [2023-05-11 00:16:30,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +23: [2023-05-11 00:16:30,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +20: [2023-05-11 00:16:30,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +20: [2023-05-11 00:16:30,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +20: [2023-05-11 00:16:30,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +25: [2023-05-11 00:16:30,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +23: [2023-05-11 00:16:30,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +20: [2023-05-11 00:16:30,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +18: [2023-05-11 00:16:30,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +18: [2023-05-11 00:16:30,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +16: [2023-05-11 00:16:30,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +16: [2023-05-11 00:16:30,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +25: [2023-05-11 00:16:30,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +25: [2023-05-11 00:16:30,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +17: [2023-05-11 00:16:30,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +22: [2023-05-11 00:16:30,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +17: [2023-05-11 00:16:30,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +17: [2023-05-11 00:16:30,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +22: [2023-05-11 00:16:30,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 2: [2023-05-11 00:16:30,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +16: [2023-05-11 00:16:30,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +16: [2023-05-11 00:16:30,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +25: [2023-05-11 00:16:30,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +26: [2023-05-11 00:16:30,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +25: [2023-05-11 00:16:30,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +17: [2023-05-11 00:16:30,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +26: [2023-05-11 00:16:30,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +25: [2023-05-11 00:16:30,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 0: [2023-05-11 00:16:30,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +14: [2023-05-11 00:16:30,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 0: [2023-05-11 00:16:30,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +26: [2023-05-11 00:16:30,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:30,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +22: [2023-05-11 00:16:30,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +23: [2023-05-11 00:16:30,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +30: [2023-05-11 00:16:30,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +30: [2023-05-11 00:16:30,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +14: [2023-05-11 00:16:30,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +25: [2023-05-11 00:16:30,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +30: [2023-05-11 00:16:30,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +30: [2023-05-11 00:16:30,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +20: [2023-05-11 00:16:30,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:30,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +25: [2023-05-11 00:16:30,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +14: [2023-05-11 00:16:30,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +16: [2023-05-11 00:16:30,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:30,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +20: [2023-05-11 00:16:30,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +14: [2023-05-11 00:16:30,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +16: [2023-05-11 00:16:30,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +20: [2023-05-11 00:16:30,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 8: [2023-05-11 00:16:30,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +25: [2023-05-11 00:16:30,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +25: [2023-05-11 00:16:30,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +26: [2023-05-11 00:16:30,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 8: [2023-05-11 00:16:30,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 0: [2023-05-11 00:16:30,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +25: [2023-05-11 00:16:30,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +24: [2023-05-11 00:16:30,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 7: [2023-05-11 00:16:30,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +20: [2023-05-11 00:16:30,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +26: [2023-05-11 00:16:30,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +22: [2023-05-11 00:16:30,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +16: [2023-05-11 00:16:30,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +16: [2023-05-11 00:16:30,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +26: [2023-05-11 00:16:30,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 7: [2023-05-11 00:16:30,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 7: [2023-05-11 00:16:30,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 7: [2023-05-11 00:16:30,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:30,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +25: [2023-05-11 00:16:30,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +24: [2023-05-11 00:16:30,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 7: [2023-05-11 00:16:30,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +14: [2023-05-11 00:16:30,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +30: [2023-05-11 00:16:30,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +15: [2023-05-11 00:16:30,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +15: [2023-05-11 00:16:30,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +15: [2023-05-11 00:16:30,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +15: [2023-05-11 00:16:30,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +15: [2023-05-11 00:16:30,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +14: [2023-05-11 00:16:30,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +15: [2023-05-11 00:16:30,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +15: [2023-05-11 00:16:30,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +15: [2023-05-11 00:16:30,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +15: [2023-05-11 00:16:30,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +15: [2023-05-11 00:16:30,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +30: [2023-05-11 00:16:30,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +15: [2023-05-11 00:16:30,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +15: [2023-05-11 00:16:30,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +15: [2023-05-11 00:16:30,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +15: [2023-05-11 00:16:30,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +15: [2023-05-11 00:16:30,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +15: [2023-05-11 00:16:30,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +14: [2023-05-11 00:16:30,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +24: [2023-05-11 00:16:30,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +23: [2023-05-11 00:16:30,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +23: [2023-05-11 00:16:30,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +23: [2023-05-11 00:16:30,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +24: [2023-05-11 00:16:30,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 8: [2023-05-11 00:16:30,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +23: [2023-05-11 00:16:30,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +24: [2023-05-11 00:16:30,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +14: [2023-05-11 00:16:30,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +30: [2023-05-11 00:16:30,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +30: [2023-05-11 00:16:30,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +26: [2023-05-11 00:16:30,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +25: [2023-05-11 00:16:30,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +12: [2023-05-11 00:16:30,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +12: [2023-05-11 00:16:30,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 0: [2023-05-11 00:16:30,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +25: [2023-05-11 00:16:30,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 0: [2023-05-11 00:16:30,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 8: [2023-05-11 00:16:30,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +26: [2023-05-11 00:16:30,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +25: [2023-05-11 00:16:30,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +25: [2023-05-11 00:16:30,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +24: [2023-05-11 00:16:30,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +26: [2023-05-11 00:16:30,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +12: [2023-05-11 00:16:30,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 7: [2023-05-11 00:16:30,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +27: [2023-05-11 00:16:30,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +12: [2023-05-11 00:16:30,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +25: [2023-05-11 00:16:30,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +25: [2023-05-11 00:16:30,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 1: [2023-05-11 00:16:30,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +23: [2023-05-11 00:16:30,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +24: [2023-05-11 00:16:30,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +24: [2023-05-11 00:16:30,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +27: [2023-05-11 00:16:30,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +15: [2023-05-11 00:16:30,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +23: [2023-05-11 00:16:30,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +15: [2023-05-11 00:16:30,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +23: [2023-05-11 00:16:30,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +27: [2023-05-11 00:16:30,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +23: [2023-05-11 00:16:30,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +27: [2023-05-11 00:16:30,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 3: [2023-05-11 00:16:30,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +12: [2023-05-11 00:16:30,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +15: [2023-05-11 00:16:30,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +15: [2023-05-11 00:16:30,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +25: [2023-05-11 00:16:30,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 0: [2023-05-11 00:16:30,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +25: [2023-05-11 00:16:30,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 7: [2023-05-11 00:16:30,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +12: [2023-05-11 00:16:30,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +12: [2023-05-11 00:16:30,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +15: [2023-05-11 00:16:30,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +16: [2023-05-11 00:16:30,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +16: [2023-05-11 00:16:30,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +16: [2023-05-11 00:16:30,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +16: [2023-05-11 00:16:30,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +16: [2023-05-11 00:16:30,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +16: [2023-05-11 00:16:30,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +16: [2023-05-11 00:16:30,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +16: [2023-05-11 00:16:30,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +16: [2023-05-11 00:16:30,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +16: [2023-05-11 00:16:30,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +16: [2023-05-11 00:16:30,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +16: [2023-05-11 00:16:30,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +27: [2023-05-11 00:16:30,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +27: [2023-05-11 00:16:30,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +16: [2023-05-11 00:16:30,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +16: [2023-05-11 00:16:30,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +16: [2023-05-11 00:16:30,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +16: [2023-05-11 00:16:30,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 1: [2023-05-11 00:16:30,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +23: [2023-05-11 00:16:30,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +23: [2023-05-11 00:16:30,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +12: [2023-05-11 00:16:30,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +27: [2023-05-11 00:16:30,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +30: [2023-05-11 00:16:30,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +30: [2023-05-11 00:16:30,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +30: [2023-05-11 00:16:30,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +30: [2023-05-11 00:16:30,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +30: [2023-05-11 00:16:30,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +27: [2023-05-11 00:16:30,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +30: [2023-05-11 00:16:30,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +30: [2023-05-11 00:16:30,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +30: [2023-05-11 00:16:30,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +30: [2023-05-11 00:16:30,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +30: [2023-05-11 00:16:30,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +29: [2023-05-11 00:16:30,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +29: [2023-05-11 00:16:30,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +29: [2023-05-11 00:16:30,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +29: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +15: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +19: [2023-05-11 00:16:30,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +19: [2023-05-11 00:16:30,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +19: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +19: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +19: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +29: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +30: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +30: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 6: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +29: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +29: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +29: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +29: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +29: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 6: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +23: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +29: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +29: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +19: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 6: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +23: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +15: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +18: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +18: [2023-05-11 00:16:30,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +18: [2023-05-11 00:16:30,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +18: [2023-05-11 00:16:30,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +18: [2023-05-11 00:16:30,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +18: [2023-05-11 00:16:30,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +18: [2023-05-11 00:16:30,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +18: [2023-05-11 00:16:30,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +18: [2023-05-11 00:16:30,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +18: [2023-05-11 00:16:30,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +18: [2023-05-11 00:16:30,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +18: [2023-05-11 00:16:30,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +18: [2023-05-11 00:16:30,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +18: [2023-05-11 00:16:30,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +19: [2023-05-11 00:16:30,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +19: [2023-05-11 00:16:30,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +18: [2023-05-11 00:16:30,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +19: [2023-05-11 00:16:30,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +19: [2023-05-11 00:16:30,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +19: [2023-05-11 00:16:30,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +19: [2023-05-11 00:16:30,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +19: [2023-05-11 00:16:30,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +19: [2023-05-11 00:16:30,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +24: [2023-05-11 00:16:30,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +18: [2023-05-11 00:16:30,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +29: [2023-05-11 00:16:30,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +29: [2023-05-11 00:16:30,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +29: [2023-05-11 00:16:30,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +29: [2023-05-11 00:16:30,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +24: [2023-05-11 00:16:30,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +24: [2023-05-11 00:16:30,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +24: [2023-05-11 00:16:30,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +15: [2023-05-11 00:16:30,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +19: [2023-05-11 00:16:30,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +31: [2023-05-11 00:16:30,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +19: [2023-05-11 00:16:30,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 0: [2023-05-11 00:16:30,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +31: [2023-05-11 00:16:30,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 4: [2023-05-11 00:16:30,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +31: [2023-05-11 00:16:30,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +31: [2023-05-11 00:16:30,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +24: [2023-05-11 00:16:30,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +24: [2023-05-11 00:16:30,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +24: [2023-05-11 00:16:30,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +24: [2023-05-11 00:16:30,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +24: [2023-05-11 00:16:30,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +31: [2023-05-11 00:16:30,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +24: [2023-05-11 00:16:30,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +31: [2023-05-11 00:16:30,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +24: [2023-05-11 00:16:30,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +24: [2023-05-11 00:16:30,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +20: [2023-05-11 00:16:30,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +20: [2023-05-11 00:16:30,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +20: [2023-05-11 00:16:30,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +20: [2023-05-11 00:16:30,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +20: [2023-05-11 00:16:30,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +20: [2023-05-11 00:16:30,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +20: [2023-05-11 00:16:30,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +20: [2023-05-11 00:16:30,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +24: [2023-05-11 00:16:30,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +20: [2023-05-11 00:16:30,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +20: [2023-05-11 00:16:30,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +20: [2023-05-11 00:16:30,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +20: [2023-05-11 00:16:30,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +27: [2023-05-11 00:16:30,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +27: [2023-05-11 00:16:30,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +27: [2023-05-11 00:16:30,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +27: [2023-05-11 00:16:30,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +20: [2023-05-11 00:16:30,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +20: [2023-05-11 00:16:30,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +20: [2023-05-11 00:16:30,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +24: [2023-05-11 00:16:30,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +30: [2023-05-11 00:16:30,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +24: [2023-05-11 00:16:30,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +30: [2023-05-11 00:16:30,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +24: [2023-05-11 00:16:30,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +27: [2023-05-11 00:16:30,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +27: [2023-05-11 00:16:30,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +13: [2023-05-11 00:16:30,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +13: [2023-05-11 00:16:30,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +13: [2023-05-11 00:16:30,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +16: [2023-05-11 00:16:30,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +31: [2023-05-11 00:16:30,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +30: [2023-05-11 00:16:30,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 8: [2023-05-11 00:16:30,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +30: [2023-05-11 00:16:30,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 8: [2023-05-11 00:16:30,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +20: [2023-05-11 00:16:30,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +12: [2023-05-11 00:16:30,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +17: [2023-05-11 00:16:30,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +13: [2023-05-11 00:16:30,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +12: [2023-05-11 00:16:30,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +17: [2023-05-11 00:16:30,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +17: [2023-05-11 00:16:30,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +17: [2023-05-11 00:16:30,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +31: [2023-05-11 00:16:30,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +17: [2023-05-11 00:16:30,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +17: [2023-05-11 00:16:30,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +17: [2023-05-11 00:16:30,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +17: [2023-05-11 00:16:30,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +17: [2023-05-11 00:16:30,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +17: [2023-05-11 00:16:30,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +17: [2023-05-11 00:16:30,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... +17: [2023-05-11 00:16:30,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +17: [2023-05-11 00:16:30,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +16: [2023-05-11 00:16:30,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +17: [2023-05-11 00:16:30,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt... + 1: [2023-05-11 00:16:30,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +19: [2023-05-11 00:16:30,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +30: [2023-05-11 00:16:30,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +27: [2023-05-11 00:16:30,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +27: [2023-05-11 00:16:30,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +17: [2023-05-11 00:16:30,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +17: [2023-05-11 00:16:30,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt... +19: [2023-05-11 00:16:30,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +30: [2023-05-11 00:16:30,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +12: [2023-05-11 00:16:30,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +12: [2023-05-11 00:16:30,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +12: [2023-05-11 00:16:30,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +12: [2023-05-11 00:16:30,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +12: [2023-05-11 00:16:30,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +12: [2023-05-11 00:16:30,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 3: [2023-05-11 00:16:30,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +23: [2023-05-11 00:16:30,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +29: [2023-05-11 00:16:30,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +16: [2023-05-11 00:16:30,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +16: [2023-05-11 00:16:30,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +19: [2023-05-11 00:16:30,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +24: [2023-05-11 00:16:30,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +30: [2023-05-11 00:16:30,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +23: [2023-05-11 00:16:30,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +23: [2023-05-11 00:16:30,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +18: [2023-05-11 00:16:30,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +18: [2023-05-11 00:16:30,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +18: [2023-05-11 00:16:30,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +19: [2023-05-11 00:16:30,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +13: [2023-05-11 00:16:30,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +27: [2023-05-11 00:16:30,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +13: [2023-05-11 00:16:30,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +23: [2023-05-11 00:16:30,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +27: [2023-05-11 00:16:30,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +31: [2023-05-11 00:16:30,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +13: [2023-05-11 00:16:30,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +20: [2023-05-11 00:16:30,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +27: [2023-05-11 00:16:30,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +31: [2023-05-11 00:16:30,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +31: [2023-05-11 00:16:30,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +20: [2023-05-11 00:16:30,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +13: [2023-05-11 00:16:30,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +27: [2023-05-11 00:16:30,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 6: [2023-05-11 00:16:30,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +29: [2023-05-11 00:16:30,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +29: [2023-05-11 00:16:30,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 6: [2023-05-11 00:16:30,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +29: [2023-05-11 00:16:30,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +16: [2023-05-11 00:16:30,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +30: [2023-05-11 00:16:30,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +30: [2023-05-11 00:16:30,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +24: [2023-05-11 00:16:30,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +24: [2023-05-11 00:16:30,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +19: [2023-05-11 00:16:30,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 2: [2023-05-11 00:16:30,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 2: [2023-05-11 00:16:30,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +27: [2023-05-11 00:16:30,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +11: [2023-05-11 00:16:30,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +11: [2023-05-11 00:16:30,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +11: [2023-05-11 00:16:30,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +11: [2023-05-11 00:16:30,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +14: [2023-05-11 00:16:30,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +14: [2023-05-11 00:16:30,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +14: [2023-05-11 00:16:30,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +14: [2023-05-11 00:16:30,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +14: [2023-05-11 00:16:30,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +11: [2023-05-11 00:16:30,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +14: [2023-05-11 00:16:30,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +14: [2023-05-11 00:16:30,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +14: [2023-05-11 00:16:30,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:30,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +17: [2023-05-11 00:16:30,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +27: [2023-05-11 00:16:30,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +27: [2023-05-11 00:16:30,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 2: [2023-05-11 00:16:30,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +14: [2023-05-11 00:16:30,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +14: [2023-05-11 00:16:30,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +14: [2023-05-11 00:16:30,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +16: [2023-05-11 00:16:30,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +30: [2023-05-11 00:16:30,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +14: [2023-05-11 00:16:30,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +30: [2023-05-11 00:16:30,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +14: [2023-05-11 00:16:30,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +14: [2023-05-11 00:16:30,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +16: [2023-05-11 00:16:30,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +19: [2023-05-11 00:16:30,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +18: [2023-05-11 00:16:30,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +20: [2023-05-11 00:16:30,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +19: [2023-05-11 00:16:30,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +24: [2023-05-11 00:16:30,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +20: [2023-05-11 00:16:30,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +13: [2023-05-11 00:16:30,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +13: [2023-05-11 00:16:30,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +17: [2023-05-11 00:16:30,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +18: [2023-05-11 00:16:30,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +14: [2023-05-11 00:16:30,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +14: [2023-05-11 00:16:30,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +18: [2023-05-11 00:16:30,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:30,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +27: [2023-05-11 00:16:30,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +13: [2023-05-11 00:16:30,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +16: [2023-05-11 00:16:30,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +13: [2023-05-11 00:16:30,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 3: [2023-05-11 00:16:30,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +20: [2023-05-11 00:16:30,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +20: [2023-05-11 00:16:30,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +31: [2023-05-11 00:16:30,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +18: [2023-05-11 00:16:30,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +19: [2023-05-11 00:16:30,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +31: [2023-05-11 00:16:30,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +29: [2023-05-11 00:16:30,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:30,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +30: [2023-05-11 00:16:30,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +31: [2023-05-11 00:16:30,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +13: [2023-05-11 00:16:30,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +13: [2023-05-11 00:16:30,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +13: [2023-05-11 00:16:30,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +31: [2023-05-11 00:16:30,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +31: [2023-05-11 00:16:30,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 1: [2023-05-11 00:16:30,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +27: [2023-05-11 00:16:30,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +27: [2023-05-11 00:16:30,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +27: [2023-05-11 00:16:30,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +13: [2023-05-11 00:16:30,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +27: [2023-05-11 00:16:30,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +29: [2023-05-11 00:16:30,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +13: [2023-05-11 00:16:30,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +31: [2023-05-11 00:16:30,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +20: [2023-05-11 00:16:30,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +13: [2023-05-11 00:16:30,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +24: [2023-05-11 00:16:30,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +29: [2023-05-11 00:16:30,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +31: [2023-05-11 00:16:30,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 2: [2023-05-11 00:16:30,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +24: [2023-05-11 00:16:30,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +11: [2023-05-11 00:16:30,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +11: [2023-05-11 00:16:30,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +17: [2023-05-11 00:16:30,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +17: [2023-05-11 00:16:30,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +18: [2023-05-11 00:16:30,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +13: [2023-05-11 00:16:30,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +13: [2023-05-11 00:16:30,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 6: [2023-05-11 00:16:30,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +13: [2023-05-11 00:16:30,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +13: [2023-05-11 00:16:30,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +27: [2023-05-11 00:16:30,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +20: [2023-05-11 00:16:30,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +27: [2023-05-11 00:16:30,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +29: [2023-05-11 00:16:30,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +11: [2023-05-11 00:16:30,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +11: [2023-05-11 00:16:30,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +17: [2023-05-11 00:16:30,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +27: [2023-05-11 00:16:30,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +31: [2023-05-11 00:16:30,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +11: [2023-05-11 00:16:30,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +31: [2023-05-11 00:16:30,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +24: [2023-05-11 00:16:30,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +31: [2023-05-11 00:16:30,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +13: [2023-05-11 00:16:30,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +31: [2023-05-11 00:16:30,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +13: [2023-05-11 00:16:30,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 3: [2023-05-11 00:16:30,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +13: [2023-05-11 00:16:30,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +14: [2023-05-11 00:16:30,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +14: [2023-05-11 00:16:30,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +11: [2023-05-11 00:16:30,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +11: [2023-05-11 00:16:30,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +13: [2023-05-11 00:16:30,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +17: [2023-05-11 00:16:30,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt. +11: [2023-05-11 00:16:30,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +31: [2023-05-11 00:16:30,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 1: [2023-05-11 00:16:30,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +17: [2023-05-11 00:16:30,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +14: [2023-05-11 00:16:30,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +27: [2023-05-11 00:16:30,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:30,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +11: [2023-05-11 00:16:30,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +24: [2023-05-11 00:16:30,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +31: [2023-05-11 00:16:30,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +13: [2023-05-11 00:16:30,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +13: [2023-05-11 00:16:30,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +14: [2023-05-11 00:16:30,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +14: [2023-05-11 00:16:30,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +31: [2023-05-11 00:16:30,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +14: [2023-05-11 00:16:30,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +11: [2023-05-11 00:16:30,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +31: [2023-05-11 00:16:30,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +11: [2023-05-11 00:16:30,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +17: [2023-05-11 00:16:30,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +13: [2023-05-11 00:16:30,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:30,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 3: [2023-05-11 00:16:30,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +11: [2023-05-11 00:16:30,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +14: [2023-05-11 00:16:30,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 2: [2023-05-11 00:16:30,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +11: [2023-05-11 00:16:30,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +11: [2023-05-11 00:16:30,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +13: [2023-05-11 00:16:30,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +14: [2023-05-11 00:16:30,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +31: [2023-05-11 00:16:30,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +31: [2023-05-11 00:16:30,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +13: [2023-05-11 00:16:30,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 5: [2023-05-11 00:16:30,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +31: [2023-05-11 00:16:30,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +31: [2023-05-11 00:16:30,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +26: [2023-05-11 00:16:30,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +26: [2023-05-11 00:16:30,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 5: [2023-05-11 00:16:30,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 5: [2023-05-11 00:16:30,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +31: [2023-05-11 00:16:30,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +31: [2023-05-11 00:16:30,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +11: [2023-05-11 00:16:30,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +11: [2023-05-11 00:16:30,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +26: [2023-05-11 00:16:30,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +26: [2023-05-11 00:16:30,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 4: [2023-05-11 00:16:30,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 4: [2023-05-11 00:16:30,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 4: [2023-05-11 00:16:30,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 4: [2023-05-11 00:16:30,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 2: [2023-05-11 00:16:30,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +13: [2023-05-11 00:16:30,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:30,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +11: [2023-05-11 00:16:30,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 2: [2023-05-11 00:16:30,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +11: [2023-05-11 00:16:30,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +26: [2023-05-11 00:16:30,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 5: [2023-05-11 00:16:30,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 0: [2023-05-11 00:16:30,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 5: [2023-05-11 00:16:30,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +22: [2023-05-11 00:16:30,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +22: [2023-05-11 00:16:30,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +22: [2023-05-11 00:16:30,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +22: [2023-05-11 00:16:30,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:30,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +22: [2023-05-11 00:16:30,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +26: [2023-05-11 00:16:30,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:30,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +22: [2023-05-11 00:16:30,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 5: [2023-05-11 00:16:30,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 5: [2023-05-11 00:16:30,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +26: [2023-05-11 00:16:30,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 4: [2023-05-11 00:16:30,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +26: [2023-05-11 00:16:30,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:30,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 8: [2023-05-11 00:16:30,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 8: [2023-05-11 00:16:30,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +22: [2023-05-11 00:16:30,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +22: [2023-05-11 00:16:30,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:30,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 4: [2023-05-11 00:16:30,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +11: [2023-05-11 00:16:30,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 8: [2023-05-11 00:16:30,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +11: [2023-05-11 00:16:30,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +11: [2023-05-11 00:16:30,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +22: [2023-05-11 00:16:30,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:30,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:30,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:30,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:30,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +10: [2023-05-11 00:16:30,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +10: [2023-05-11 00:16:30,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +10: [2023-05-11 00:16:30,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +10: [2023-05-11 00:16:30,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +22: [2023-05-11 00:16:30,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +21: [2023-05-11 00:16:30,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +21: [2023-05-11 00:16:30,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +21: [2023-05-11 00:16:30,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 4: [2023-05-11 00:16:30,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +21: [2023-05-11 00:16:30,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +22: [2023-05-11 00:16:30,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +22: [2023-05-11 00:16:30,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +22: [2023-05-11 00:16:30,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +10: [2023-05-11 00:16:30,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +23: [2023-05-11 00:16:30,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +23: [2023-05-11 00:16:30,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +10: [2023-05-11 00:16:30,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +10: [2023-05-11 00:16:30,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 0: [2023-05-11 00:16:30,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 4: [2023-05-11 00:16:30,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +21: [2023-05-11 00:16:30,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +21: [2023-05-11 00:16:30,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +22: [2023-05-11 00:16:30,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +21: [2023-05-11 00:16:30,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +22: [2023-05-11 00:16:30,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +23: [2023-05-11 00:16:30,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +22: [2023-05-11 00:16:30,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +10: [2023-05-11 00:16:30,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +23: [2023-05-11 00:16:30,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +22: [2023-05-11 00:16:30,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +22: [2023-05-11 00:16:30,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +22: [2023-05-11 00:16:30,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +22: [2023-05-11 00:16:30,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +22: [2023-05-11 00:16:30,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +22: [2023-05-11 00:16:30,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 4: [2023-05-11 00:16:30,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +21: [2023-05-11 00:16:30,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 0: [2023-05-11 00:16:30,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 7: [2023-05-11 00:16:30,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 7: [2023-05-11 00:16:30,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +23: [2023-05-11 00:16:30,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +22: [2023-05-11 00:16:30,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +22: [2023-05-11 00:16:30,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +23: [2023-05-11 00:16:30,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 9: [2023-05-11 00:16:30,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 9: [2023-05-11 00:16:30,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 9: [2023-05-11 00:16:30,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 7: [2023-05-11 00:16:30,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +26: [2023-05-11 00:16:30,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +26: [2023-05-11 00:16:30,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +26: [2023-05-11 00:16:30,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +26: [2023-05-11 00:16:30,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 7: [2023-05-11 00:16:30,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +23: [2023-05-11 00:16:30,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +26: [2023-05-11 00:16:30,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +26: [2023-05-11 00:16:30,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +26: [2023-05-11 00:16:30,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +26: [2023-05-11 00:16:30,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +23: [2023-05-11 00:16:30,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +17: [2023-05-11 00:16:30,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +17: [2023-05-11 00:16:30,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +26: [2023-05-11 00:16:30,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +26: [2023-05-11 00:16:30,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +26: [2023-05-11 00:16:30,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +26: [2023-05-11 00:16:30,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +26: [2023-05-11 00:16:30,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +26: [2023-05-11 00:16:30,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +26: [2023-05-11 00:16:30,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +17: [2023-05-11 00:16:30,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +17: [2023-05-11 00:16:30,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +17: [2023-05-11 00:16:30,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +17: [2023-05-11 00:16:30,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +17: [2023-05-11 00:16:30,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 8: [2023-05-11 00:16:30,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 8: [2023-05-11 00:16:30,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 8: [2023-05-11 00:16:30,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 2: [2023-05-11 00:16:30,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +17: [2023-05-11 00:16:30,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +26: [2023-05-11 00:16:30,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +23: [2023-05-11 00:16:30,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +23: [2023-05-11 00:16:30,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +23: [2023-05-11 00:16:30,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +23: [2023-05-11 00:16:30,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +17: [2023-05-11 00:16:30,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +23: [2023-05-11 00:16:30,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +23: [2023-05-11 00:16:30,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +23: [2023-05-11 00:16:30,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +23: [2023-05-11 00:16:30,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +23: [2023-05-11 00:16:30,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +23: [2023-05-11 00:16:30,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +23: [2023-05-11 00:16:30,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +17: [2023-05-11 00:16:30,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +17: [2023-05-11 00:16:30,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +23: [2023-05-11 00:16:30,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +23: [2023-05-11 00:16:30,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +23: [2023-05-11 00:16:30,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +26: [2023-05-11 00:16:30,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +23: [2023-05-11 00:16:30,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +26: [2023-05-11 00:16:30,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +25: [2023-05-11 00:16:30,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +25: [2023-05-11 00:16:30,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +25: [2023-05-11 00:16:30,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +25: [2023-05-11 00:16:30,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +26: [2023-05-11 00:16:30,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +23: [2023-05-11 00:16:30,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +17: [2023-05-11 00:16:30,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +17: [2023-05-11 00:16:30,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +11: [2023-05-11 00:16:30,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +11: [2023-05-11 00:16:30,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +11: [2023-05-11 00:16:30,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +15: [2023-05-11 00:16:30,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +15: [2023-05-11 00:16:30,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +15: [2023-05-11 00:16:30,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +15: [2023-05-11 00:16:30,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +17: [2023-05-11 00:16:30,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +23: [2023-05-11 00:16:30,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +11: [2023-05-11 00:16:30,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +23: [2023-05-11 00:16:30,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:30,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +11: [2023-05-11 00:16:30,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +11: [2023-05-11 00:16:30,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +11: [2023-05-11 00:16:30,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +17: [2023-05-11 00:16:30,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +11: [2023-05-11 00:16:30,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +11: [2023-05-11 00:16:30,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 2: [2023-05-11 00:16:30,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:30,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 8: [2023-05-11 00:16:30,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +10: [2023-05-11 00:16:30,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +10: [2023-05-11 00:16:30,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +10: [2023-05-11 00:16:30,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +10: [2023-05-11 00:16:30,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +10: [2023-05-11 00:16:30,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +10: [2023-05-11 00:16:30,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +25: [2023-05-11 00:16:30,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +28: [2023-05-11 00:16:30,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +28: [2023-05-11 00:16:30,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +28: [2023-05-11 00:16:30,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +28: [2023-05-11 00:16:30,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +10: [2023-05-11 00:16:30,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +10: [2023-05-11 00:16:30,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +10: [2023-05-11 00:16:30,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +10: [2023-05-11 00:16:30,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +10: [2023-05-11 00:16:30,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +26: [2023-05-11 00:16:30,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +10: [2023-05-11 00:16:30,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +10: [2023-05-11 00:16:30,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +10: [2023-05-11 00:16:30,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +26: [2023-05-11 00:16:30,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +25: [2023-05-11 00:16:30,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +23: [2023-05-11 00:16:30,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +25: [2023-05-11 00:16:30,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +25: [2023-05-11 00:16:30,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +26: [2023-05-11 00:16:30,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +10: [2023-05-11 00:16:30,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +10: [2023-05-11 00:16:30,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +23: [2023-05-11 00:16:30,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:30,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +23: [2023-05-11 00:16:30,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +15: [2023-05-11 00:16:30,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +17: [2023-05-11 00:16:30,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +17: [2023-05-11 00:16:30,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +15: [2023-05-11 00:16:30,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +15: [2023-05-11 00:16:30,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +17: [2023-05-11 00:16:30,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +17: [2023-05-11 00:16:30,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +17: [2023-05-11 00:16:30,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt. +11: [2023-05-11 00:16:30,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +15: [2023-05-11 00:16:30,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +28: [2023-05-11 00:16:30,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +28: [2023-05-11 00:16:30,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:30,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +23: [2023-05-11 00:16:30,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +23: [2023-05-11 00:16:30,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +24: [2023-05-11 00:16:30,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +24: [2023-05-11 00:16:30,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +24: [2023-05-11 00:16:30,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +24: [2023-05-11 00:16:30,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +10: [2023-05-11 00:16:30,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +28: [2023-05-11 00:16:30,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +28: [2023-05-11 00:16:30,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:30,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +11: [2023-05-11 00:16:30,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +10: [2023-05-11 00:16:30,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +10: [2023-05-11 00:16:30,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +26: [2023-05-11 00:16:30,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +11: [2023-05-11 00:16:30,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +17: [2023-05-11 00:16:30,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +17: [2023-05-11 00:16:30,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +17: [2023-05-11 00:16:30,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +17: [2023-05-11 00:16:30,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt... +23: [2023-05-11 00:16:30,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:30,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +24: [2023-05-11 00:16:30,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +24: [2023-05-11 00:16:30,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +24: [2023-05-11 00:16:30,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +24: [2023-05-11 00:16:30,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +10: [2023-05-11 00:16:30,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +21: [2023-05-11 00:16:30,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +21: [2023-05-11 00:16:30,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +21: [2023-05-11 00:16:30,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +21: [2023-05-11 00:16:30,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +21: [2023-05-11 00:16:30,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +21: [2023-05-11 00:16:30,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +21: [2023-05-11 00:16:30,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +21: [2023-05-11 00:16:30,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +11: [2023-05-11 00:16:30,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +21: [2023-05-11 00:16:30,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +21: [2023-05-11 00:16:30,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +21: [2023-05-11 00:16:30,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +10: [2023-05-11 00:16:30,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +21: [2023-05-11 00:16:30,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +21: [2023-05-11 00:16:30,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +11: [2023-05-11 00:16:30,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:30,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +26: [2023-05-11 00:16:30,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +21: [2023-05-11 00:16:30,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +11: [2023-05-11 00:16:30,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +21: [2023-05-11 00:16:30,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +21: [2023-05-11 00:16:30,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:30,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +11: [2023-05-11 00:16:30,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +10: [2023-05-11 00:16:30,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +11: [2023-05-11 00:16:30,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +28: [2023-05-11 00:16:30,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +28: [2023-05-11 00:16:30,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +28: [2023-05-11 00:16:30,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +28: [2023-05-11 00:16:30,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +28: [2023-05-11 00:16:30,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +28: [2023-05-11 00:16:30,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +28: [2023-05-11 00:16:30,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +28: [2023-05-11 00:16:30,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +29: [2023-05-11 00:16:30,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +29: [2023-05-11 00:16:30,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +10: [2023-05-11 00:16:30,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +29: [2023-05-11 00:16:30,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +29: [2023-05-11 00:16:30,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +28: [2023-05-11 00:16:30,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +28: [2023-05-11 00:16:30,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +28: [2023-05-11 00:16:30,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +28: [2023-05-11 00:16:30,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +17: [2023-05-11 00:16:30,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +12: [2023-05-11 00:16:30,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +12: [2023-05-11 00:16:30,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +12: [2023-05-11 00:16:30,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +17: [2023-05-11 00:16:30,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +12: [2023-05-11 00:16:30,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +12: [2023-05-11 00:16:30,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +12: [2023-05-11 00:16:30,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +12: [2023-05-11 00:16:30,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +12: [2023-05-11 00:16:30,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:30,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +21: [2023-05-11 00:16:30,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +11: [2023-05-11 00:16:30,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:30,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +17: [2023-05-11 00:16:30,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +17: [2023-05-11 00:16:30,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +10: [2023-05-11 00:16:30,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +17: [2023-05-11 00:16:30,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +28: [2023-05-11 00:16:30,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +17: [2023-05-11 00:16:30,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +29: [2023-05-11 00:16:30,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +21: [2023-05-11 00:16:30,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +21: [2023-05-11 00:16:30,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +29: [2023-05-11 00:16:30,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +21: [2023-05-11 00:16:30,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +28: [2023-05-11 00:16:30,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +11: [2023-05-11 00:16:30,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +11: [2023-05-11 00:16:30,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +17: [2023-05-11 00:16:30,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt. +17: [2023-05-11 00:16:30,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt... +28: [2023-05-11 00:16:30,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +28: [2023-05-11 00:16:30,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +12: [2023-05-11 00:16:30,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +28: [2023-05-11 00:16:30,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +29: [2023-05-11 00:16:30,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:30,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +28: [2023-05-11 00:16:30,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +29: [2023-05-11 00:16:30,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:30,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +28: [2023-05-11 00:16:30,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +21: [2023-05-11 00:16:30,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +28: [2023-05-11 00:16:30,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +21: [2023-05-11 00:16:30,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 6: [2023-05-11 00:16:30,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 6: [2023-05-11 00:16:30,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 6: [2023-05-11 00:16:30,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +16: [2023-05-11 00:16:30,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +16: [2023-05-11 00:16:30,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +12: [2023-05-11 00:16:30,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +12: [2023-05-11 00:16:30,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +12: [2023-05-11 00:16:30,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +28: [2023-05-11 00:16:30,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +16: [2023-05-11 00:16:30,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +16: [2023-05-11 00:16:30,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +28: [2023-05-11 00:16:30,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +12: [2023-05-11 00:16:30,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +28: [2023-05-11 00:16:30,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +21: [2023-05-11 00:16:30,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +16: [2023-05-11 00:16:30,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +16: [2023-05-11 00:16:30,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +16: [2023-05-11 00:16:30,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +16: [2023-05-11 00:16:30,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +12: [2023-05-11 00:16:30,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +12: [2023-05-11 00:16:30,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +12: [2023-05-11 00:16:30,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +20: [2023-05-11 00:16:30,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +20: [2023-05-11 00:16:30,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +20: [2023-05-11 00:16:30,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +20: [2023-05-11 00:16:30,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +28: [2023-05-11 00:16:30,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +21: [2023-05-11 00:16:30,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 2: [2023-05-11 00:16:30,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 2: [2023-05-11 00:16:30,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. + 2: [2023-05-11 00:16:30,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +20: [2023-05-11 00:16:30,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +20: [2023-05-11 00:16:30,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +20: [2023-05-11 00:16:30,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +20: [2023-05-11 00:16:30,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 5: [2023-05-11 00:16:30,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 5: [2023-05-11 00:16:30,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 5: [2023-05-11 00:16:30,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 5: [2023-05-11 00:16:30,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 5: [2023-05-11 00:16:30,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 5: [2023-05-11 00:16:30,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 5: [2023-05-11 00:16:30,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 5: [2023-05-11 00:16:30,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 5: [2023-05-11 00:16:30,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +18: [2023-05-11 00:16:30,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +18: [2023-05-11 00:16:30,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +18: [2023-05-11 00:16:30,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +18: [2023-05-11 00:16:30,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 5: [2023-05-11 00:16:30,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 5: [2023-05-11 00:16:30,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 5: [2023-05-11 00:16:30,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 9: [2023-05-11 00:16:30,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 5: [2023-05-11 00:16:30,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 9: [2023-05-11 00:16:30,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +18: [2023-05-11 00:16:30,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 9: [2023-05-11 00:16:30,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 2: [2023-05-11 00:16:30,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... + 2: [2023-05-11 00:16:30,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +18: [2023-05-11 00:16:30,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +19: [2023-05-11 00:16:30,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +19: [2023-05-11 00:16:30,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +19: [2023-05-11 00:16:30,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 4: [2023-05-11 00:16:30,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +19: [2023-05-11 00:16:30,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 4: [2023-05-11 00:16:30,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +31: [2023-05-11 00:16:30,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +31: [2023-05-11 00:16:30,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +31: [2023-05-11 00:16:30,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +18: [2023-05-11 00:16:30,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +31: [2023-05-11 00:16:30,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 2: [2023-05-11 00:16:30,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +15: [2023-05-11 00:16:30,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +15: [2023-05-11 00:16:30,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +15: [2023-05-11 00:16:30,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +15: [2023-05-11 00:16:30,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +15: [2023-05-11 00:16:30,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +15: [2023-05-11 00:16:30,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +15: [2023-05-11 00:16:30,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +15: [2023-05-11 00:16:30,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +15: [2023-05-11 00:16:30,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +15: [2023-05-11 00:16:30,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +15: [2023-05-11 00:16:30,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +15: [2023-05-11 00:16:30,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +18: [2023-05-11 00:16:30,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +30: [2023-05-11 00:16:30,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +30: [2023-05-11 00:16:30,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +15: [2023-05-11 00:16:30,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 4: [2023-05-11 00:16:30,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +15: [2023-05-11 00:16:30,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +27: [2023-05-11 00:16:30,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +15: [2023-05-11 00:16:30,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +15: [2023-05-11 00:16:30,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +27: [2023-05-11 00:16:30,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +27: [2023-05-11 00:16:30,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +12: [2023-05-11 00:16:30,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +12: [2023-05-11 00:16:30,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +12: [2023-05-11 00:16:30,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 4: [2023-05-11 00:16:30,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +27: [2023-05-11 00:16:30,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +12: [2023-05-11 00:16:30,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +30: [2023-05-11 00:16:30,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +30: [2023-05-11 00:16:30,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 5: [2023-05-11 00:16:30,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +17: [2023-05-11 00:16:30,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +17: [2023-05-11 00:16:30,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +17: [2023-05-11 00:16:30,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +17: [2023-05-11 00:16:30,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 1: [2023-05-11 00:16:30,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 1: [2023-05-11 00:16:30,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +17: [2023-05-11 00:16:30,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +17: [2023-05-11 00:16:30,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +19: [2023-05-11 00:16:30,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +19: [2023-05-11 00:16:30,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +13: [2023-05-11 00:16:30,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +13: [2023-05-11 00:16:30,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +19: [2023-05-11 00:16:30,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +13: [2023-05-11 00:16:30,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +13: [2023-05-11 00:16:30,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +31: [2023-05-11 00:16:30,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +31: [2023-05-11 00:16:30,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +17: [2023-05-11 00:16:30,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +17: [2023-05-11 00:16:30,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +19: [2023-05-11 00:16:30,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +31: [2023-05-11 00:16:30,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +31: [2023-05-11 00:16:30,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +30: [2023-05-11 00:16:30,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +15: [2023-05-11 00:16:30,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +12: [2023-05-11 00:16:30,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +12: [2023-05-11 00:16:30,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +12: [2023-05-11 00:16:30,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +27: [2023-05-11 00:16:30,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +27: [2023-05-11 00:16:30,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +30: [2023-05-11 00:16:30,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +30: [2023-05-11 00:16:30,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +30: [2023-05-11 00:16:30,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +27: [2023-05-11 00:16:30,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +12: [2023-05-11 00:16:30,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +15: [2023-05-11 00:16:30,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +15: [2023-05-11 00:16:30,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +13: [2023-05-11 00:16:30,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +13: [2023-05-11 00:16:30,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +27: [2023-05-11 00:16:30,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +19: [2023-05-11 00:16:30,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +19: [2023-05-11 00:16:30,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +19: [2023-05-11 00:16:30,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +19: [2023-05-11 00:16:30,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +19: [2023-05-11 00:16:30,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +19: [2023-05-11 00:16:30,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +19: [2023-05-11 00:16:30,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +15: [2023-05-11 00:16:30,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +17: [2023-05-11 00:16:30,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 3: [2023-05-11 00:16:30,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +13: [2023-05-11 00:16:30,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +17: [2023-05-11 00:16:30,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +19: [2023-05-11 00:16:30,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +13: [2023-05-11 00:16:30,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +14: [2023-05-11 00:16:30,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +14: [2023-05-11 00:16:30,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +14: [2023-05-11 00:16:30,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +15: [2023-05-11 00:16:30,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +14: [2023-05-11 00:16:30,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 4: [2023-05-11 00:16:30,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +15: [2023-05-11 00:16:30,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +19: [2023-05-11 00:16:30,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +19: [2023-05-11 00:16:30,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 3: [2023-05-11 00:16:30,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +17: [2023-05-11 00:16:30,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +19: [2023-05-11 00:16:30,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +19: [2023-05-11 00:16:30,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +19: [2023-05-11 00:16:30,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +19: [2023-05-11 00:16:30,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +12: [2023-05-11 00:16:30,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +12: [2023-05-11 00:16:30,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +17: [2023-05-11 00:16:30,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +12: [2023-05-11 00:16:30,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +12: [2023-05-11 00:16:30,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 3: [2023-05-11 00:16:30,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +12: [2023-05-11 00:16:30,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +12: [2023-05-11 00:16:30,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 9: [2023-05-11 00:16:30,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +15: [2023-05-11 00:16:30,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 7: [2023-05-11 00:16:30,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 9: [2023-05-11 00:16:30,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 7: [2023-05-11 00:16:30,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +14: [2023-05-11 00:16:30,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +14: [2023-05-11 00:16:30,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +14: [2023-05-11 00:16:30,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +14: [2023-05-11 00:16:30,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +16: [2023-05-11 00:16:30,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +16: [2023-05-11 00:16:30,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +16: [2023-05-11 00:16:30,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +17: [2023-05-11 00:16:30,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 7: [2023-05-11 00:16:30,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +16: [2023-05-11 00:16:30,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +16: [2023-05-11 00:16:30,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +16: [2023-05-11 00:16:30,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +16: [2023-05-11 00:16:30,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +16: [2023-05-11 00:16:30,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +16: [2023-05-11 00:16:30,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +16: [2023-05-11 00:16:30,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +16: [2023-05-11 00:16:30,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +25: [2023-05-11 00:16:30,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:30,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +19: [2023-05-11 00:16:30,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +25: [2023-05-11 00:16:30,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +16: [2023-05-11 00:16:30,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +29: [2023-05-11 00:16:30,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +29: [2023-05-11 00:16:30,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +29: [2023-05-11 00:16:30,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +29: [2023-05-11 00:16:30,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +29: [2023-05-11 00:16:30,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +17: [2023-05-11 00:16:30,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +29: [2023-05-11 00:16:30,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +29: [2023-05-11 00:16:30,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +29: [2023-05-11 00:16:30,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +29: [2023-05-11 00:16:30,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +29: [2023-05-11 00:16:30,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +29: [2023-05-11 00:16:30,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +16: [2023-05-11 00:16:30,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +29: [2023-05-11 00:16:30,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +16: [2023-05-11 00:16:30,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +16: [2023-05-11 00:16:30,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +16: [2023-05-11 00:16:30,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +25: [2023-05-11 00:16:30,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +25: [2023-05-11 00:16:30,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +25: [2023-05-11 00:16:30,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +25: [2023-05-11 00:16:30,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +25: [2023-05-11 00:16:30,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +25: [2023-05-11 00:16:30,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +25: [2023-05-11 00:16:30,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +19: [2023-05-11 00:16:30,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +25: [2023-05-11 00:16:30,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +25: [2023-05-11 00:16:30,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +25: [2023-05-11 00:16:30,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +19: [2023-05-11 00:16:30,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 9: [2023-05-11 00:16:30,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +15: [2023-05-11 00:16:30,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:30,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +29: [2023-05-11 00:16:30,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +25: [2023-05-11 00:16:30,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +25: [2023-05-11 00:16:30,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +12: [2023-05-11 00:16:30,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +29: [2023-05-11 00:16:30,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +19: [2023-05-11 00:16:30,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +29: [2023-05-11 00:16:30,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +29: [2023-05-11 00:16:30,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +12: [2023-05-11 00:16:30,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +19: [2023-05-11 00:16:30,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +17: [2023-05-11 00:16:30,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 3: [2023-05-11 00:16:30,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 7: [2023-05-11 00:16:30,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +17: [2023-05-11 00:16:30,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +19: [2023-05-11 00:16:30,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +19: [2023-05-11 00:16:30,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:30,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +19: [2023-05-11 00:16:30,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +19: [2023-05-11 00:16:30,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 1: [2023-05-11 00:16:30,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +30: [2023-05-11 00:16:30,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +30: [2023-05-11 00:16:30,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +30: [2023-05-11 00:16:30,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +30: [2023-05-11 00:16:30,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +30: [2023-05-11 00:16:30,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +30: [2023-05-11 00:16:30,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 1: [2023-05-11 00:16:30,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +30: [2023-05-11 00:16:30,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +30: [2023-05-11 00:16:30,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +30: [2023-05-11 00:16:30,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +30: [2023-05-11 00:16:30,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +30: [2023-05-11 00:16:30,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +30: [2023-05-11 00:16:30,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +30: [2023-05-11 00:16:30,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 3: [2023-05-11 00:16:30,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +30: [2023-05-11 00:16:30,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +13: [2023-05-11 00:16:30,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +13: [2023-05-11 00:16:30,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +13: [2023-05-11 00:16:30,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +13: [2023-05-11 00:16:30,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +24: [2023-05-11 00:16:30,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +24: [2023-05-11 00:16:30,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +25: [2023-05-11 00:16:30,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +24: [2023-05-11 00:16:30,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +24: [2023-05-11 00:16:30,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +13: [2023-05-11 00:16:30,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +13: [2023-05-11 00:16:30,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +13: [2023-05-11 00:16:30,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +13: [2023-05-11 00:16:30,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +13: [2023-05-11 00:16:30,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +13: [2023-05-11 00:16:30,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +29: [2023-05-11 00:16:30,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +13: [2023-05-11 00:16:30,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +13: [2023-05-11 00:16:30,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +13: [2023-05-11 00:16:30,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +30: [2023-05-11 00:16:30,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +24: [2023-05-11 00:16:30,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +24: [2023-05-11 00:16:30,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +30: [2023-05-11 00:16:30,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +24: [2023-05-11 00:16:30,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +24: [2023-05-11 00:16:30,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +24: [2023-05-11 00:16:30,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +24: [2023-05-11 00:16:30,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +24: [2023-05-11 00:16:30,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +24: [2023-05-11 00:16:30,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +24: [2023-05-11 00:16:30,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +13: [2023-05-11 00:16:30,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +13: [2023-05-11 00:16:30,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +29: [2023-05-11 00:16:30,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 6: [2023-05-11 00:16:30,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +13: [2023-05-11 00:16:30,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +24: [2023-05-11 00:16:30,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +24: [2023-05-11 00:16:30,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +24: [2023-05-11 00:16:30,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +19: [2023-05-11 00:16:30,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 6: [2023-05-11 00:16:30,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +25: [2023-05-11 00:16:30,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 0: [2023-05-11 00:16:30,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 0: [2023-05-11 00:16:30,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 0: [2023-05-11 00:16:30,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 0: [2023-05-11 00:16:30,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 6: [2023-05-11 00:16:30,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +16: [2023-05-11 00:16:30,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +25: [2023-05-11 00:16:30,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +16: [2023-05-11 00:16:30,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +25: [2023-05-11 00:16:30,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +29: [2023-05-11 00:16:30,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +16: [2023-05-11 00:16:30,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +25: [2023-05-11 00:16:30,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 3: [2023-05-11 00:16:30,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +30: [2023-05-11 00:16:30,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +30: [2023-05-11 00:16:30,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +31: [2023-05-11 00:16:30,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +31: [2023-05-11 00:16:30,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +22: [2023-05-11 00:16:30,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +22: [2023-05-11 00:16:30,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +22: [2023-05-11 00:16:30,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +22: [2023-05-11 00:16:30,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +22: [2023-05-11 00:16:30,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +22: [2023-05-11 00:16:30,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +16: [2023-05-11 00:16:30,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +22: [2023-05-11 00:16:30,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +22: [2023-05-11 00:16:30,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +31: [2023-05-11 00:16:30,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +31: [2023-05-11 00:16:30,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +13: [2023-05-11 00:16:30,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +29: [2023-05-11 00:16:30,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:30,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +31: [2023-05-11 00:16:30,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +31: [2023-05-11 00:16:30,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +31: [2023-05-11 00:16:30,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +31: [2023-05-11 00:16:30,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +31: [2023-05-11 00:16:30,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +31: [2023-05-11 00:16:30,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +13: [2023-05-11 00:16:30,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +13: [2023-05-11 00:16:30,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +16: [2023-05-11 00:16:30,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +16: [2023-05-11 00:16:30,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +31: [2023-05-11 00:16:30,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +31: [2023-05-11 00:16:30,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +29: [2023-05-11 00:16:30,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +31: [2023-05-11 00:16:30,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +24: [2023-05-11 00:16:30,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +16: [2023-05-11 00:16:30,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +31: [2023-05-11 00:16:30,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +31: [2023-05-11 00:16:30,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 3: [2023-05-11 00:16:30,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +31: [2023-05-11 00:16:30,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +25: [2023-05-11 00:16:30,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +29: [2023-05-11 00:16:30,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +16: [2023-05-11 00:16:30,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +29: [2023-05-11 00:16:30,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +22: [2023-05-11 00:16:30,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +30: [2023-05-11 00:16:30,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +30: [2023-05-11 00:16:30,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +13: [2023-05-11 00:16:30,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +13: [2023-05-11 00:16:30,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +31: [2023-05-11 00:16:30,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +22: [2023-05-11 00:16:30,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +13: [2023-05-11 00:16:30,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +18: [2023-05-11 00:16:30,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +18: [2023-05-11 00:16:30,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +18: [2023-05-11 00:16:30,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +18: [2023-05-11 00:16:30,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +18: [2023-05-11 00:16:30,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +18: [2023-05-11 00:16:30,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +18: [2023-05-11 00:16:30,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +18: [2023-05-11 00:16:30,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +22: [2023-05-11 00:16:30,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +22: [2023-05-11 00:16:30,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +18: [2023-05-11 00:16:30,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +24: [2023-05-11 00:16:30,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +18: [2023-05-11 00:16:30,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +18: [2023-05-11 00:16:30,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +24: [2023-05-11 00:16:30,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +18: [2023-05-11 00:16:30,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +18: [2023-05-11 00:16:30,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +27: [2023-05-11 00:16:30,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +27: [2023-05-11 00:16:30,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +27: [2023-05-11 00:16:30,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +27: [2023-05-11 00:16:30,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +18: [2023-05-11 00:16:30,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +27: [2023-05-11 00:16:30,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +30: [2023-05-11 00:16:30,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +27: [2023-05-11 00:16:30,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +27: [2023-05-11 00:16:30,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +27: [2023-05-11 00:16:30,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +27: [2023-05-11 00:16:30,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +18: [2023-05-11 00:16:30,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +29: [2023-05-11 00:16:30,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +30: [2023-05-11 00:16:30,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +18: [2023-05-11 00:16:30,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +13: [2023-05-11 00:16:30,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +27: [2023-05-11 00:16:30,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +27: [2023-05-11 00:16:30,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +27: [2023-05-11 00:16:30,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +27: [2023-05-11 00:16:30,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +27: [2023-05-11 00:16:30,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +27: [2023-05-11 00:16:30,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +22: [2023-05-11 00:16:30,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +17: [2023-05-11 00:16:30,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +17: [2023-05-11 00:16:30,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +27: [2023-05-11 00:16:30,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +24: [2023-05-11 00:16:30,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 8: [2023-05-11 00:16:30,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 8: [2023-05-11 00:16:30,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 8: [2023-05-11 00:16:30,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +31: [2023-05-11 00:16:30,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +17: [2023-05-11 00:16:30,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. +17: [2023-05-11 00:16:30,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt. + 4: [2023-05-11 00:16:30,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 4: [2023-05-11 00:16:30,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 6: [2023-05-11 00:16:30,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +11: [2023-05-11 00:16:30,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +11: [2023-05-11 00:16:30,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +11: [2023-05-11 00:16:30,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +22: [2023-05-11 00:16:30,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +11: [2023-05-11 00:16:30,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +24: [2023-05-11 00:16:30,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +22: [2023-05-11 00:16:30,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +30: [2023-05-11 00:16:30,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +22: [2023-05-11 00:16:30,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 4: [2023-05-11 00:16:30,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +13: [2023-05-11 00:16:30,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +30: [2023-05-11 00:16:30,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +18: [2023-05-11 00:16:30,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +31: [2023-05-11 00:16:30,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +27: [2023-05-11 00:16:30,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +24: [2023-05-11 00:16:30,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +18: [2023-05-11 00:16:30,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +18: [2023-05-11 00:16:30,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +24: [2023-05-11 00:16:30,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +27: [2023-05-11 00:16:30,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +14: [2023-05-11 00:16:30,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +14: [2023-05-11 00:16:30,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +14: [2023-05-11 00:16:30,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +14: [2023-05-11 00:16:30,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +27: [2023-05-11 00:16:30,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +14: [2023-05-11 00:16:30,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +14: [2023-05-11 00:16:30,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +26: [2023-05-11 00:16:30,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +26: [2023-05-11 00:16:30,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 4: [2023-05-11 00:16:30,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +17: [2023-05-11 00:16:30,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +26: [2023-05-11 00:16:30,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +14: [2023-05-11 00:16:30,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +14: [2023-05-11 00:16:30,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +14: [2023-05-11 00:16:30,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +11: [2023-05-11 00:16:30,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +14: [2023-05-11 00:16:30,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +14: [2023-05-11 00:16:30,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +14: [2023-05-11 00:16:30,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +17: [2023-05-11 00:16:30,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +17: [2023-05-11 00:16:30,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +26: [2023-05-11 00:16:30,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +11: [2023-05-11 00:16:30,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:30,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:30,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +17: [2023-05-11 00:16:30,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt... +31: [2023-05-11 00:16:30,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +18: [2023-05-11 00:16:30,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +27: [2023-05-11 00:16:30,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +24: [2023-05-11 00:16:30,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +20: [2023-05-11 00:16:30,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +20: [2023-05-11 00:16:30,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +20: [2023-05-11 00:16:30,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +20: [2023-05-11 00:16:30,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +20: [2023-05-11 00:16:30,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +31: [2023-05-11 00:16:30,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +20: [2023-05-11 00:16:30,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +20: [2023-05-11 00:16:30,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +20: [2023-05-11 00:16:30,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +20: [2023-05-11 00:16:30,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +20: [2023-05-11 00:16:30,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +22: [2023-05-11 00:16:30,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +20: [2023-05-11 00:16:30,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +22: [2023-05-11 00:16:30,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +20: [2023-05-11 00:16:30,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +20: [2023-05-11 00:16:30,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +20: [2023-05-11 00:16:30,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +31: [2023-05-11 00:16:30,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +22: [2023-05-11 00:16:30,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +18: [2023-05-11 00:16:30,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +18: [2023-05-11 00:16:30,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +22: [2023-05-11 00:16:30,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +27: [2023-05-11 00:16:30,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +14: [2023-05-11 00:16:30,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +14: [2023-05-11 00:16:30,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 2: [2023-05-11 00:16:30,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 2: [2023-05-11 00:16:30,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 2: [2023-05-11 00:16:30,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +23: [2023-05-11 00:16:30,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +23: [2023-05-11 00:16:30,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +23: [2023-05-11 00:16:30,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 2: [2023-05-11 00:16:30,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +27: [2023-05-11 00:16:30,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +20: [2023-05-11 00:16:30,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +20: [2023-05-11 00:16:30,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +26: [2023-05-11 00:16:30,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +26: [2023-05-11 00:16:30,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +23: [2023-05-11 00:16:30,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +26: [2023-05-11 00:16:30,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +18: [2023-05-11 00:16:30,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +14: [2023-05-11 00:16:30,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +14: [2023-05-11 00:16:30,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +31: [2023-05-11 00:16:30,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +22: [2023-05-11 00:16:30,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +26: [2023-05-11 00:16:30,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +23: [2023-05-11 00:16:30,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +31: [2023-05-11 00:16:30,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +23: [2023-05-11 00:16:30,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +22: [2023-05-11 00:16:30,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +27: [2023-05-11 00:16:30,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +23: [2023-05-11 00:16:30,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +23: [2023-05-11 00:16:30,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +22: [2023-05-11 00:16:30,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +22: [2023-05-11 00:16:30,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +17: [2023-05-11 00:16:30,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +17: [2023-05-11 00:16:30,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +28: [2023-05-11 00:16:30,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +28: [2023-05-11 00:16:30,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +17: [2023-05-11 00:16:30,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +28: [2023-05-11 00:16:30,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +17: [2023-05-11 00:16:30,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +28: [2023-05-11 00:16:30,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +14: [2023-05-11 00:16:30,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +17: [2023-05-11 00:16:30,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +17: [2023-05-11 00:16:30,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... +17: [2023-05-11 00:16:30,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt. +17: [2023-05-11 00:16:30,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt... + 2: [2023-05-11 00:16:30,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +23: [2023-05-11 00:16:30,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 0: [2023-05-11 00:16:30,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 0: [2023-05-11 00:16:30,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +20: [2023-05-11 00:16:30,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +20: [2023-05-11 00:16:30,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +20: [2023-05-11 00:16:30,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +23: [2023-05-11 00:16:30,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +23: [2023-05-11 00:16:30,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +23: [2023-05-11 00:16:30,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +23: [2023-05-11 00:16:30,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +26: [2023-05-11 00:16:30,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +26: [2023-05-11 00:16:30,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +26: [2023-05-11 00:16:30,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +26: [2023-05-11 00:16:30,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +26: [2023-05-11 00:16:30,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +26: [2023-05-11 00:16:30,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +18: [2023-05-11 00:16:30,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +23: [2023-05-11 00:16:30,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +26: [2023-05-11 00:16:30,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +14: [2023-05-11 00:16:30,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +14: [2023-05-11 00:16:30,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +23: [2023-05-11 00:16:30,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +23: [2023-05-11 00:16:30,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +26: [2023-05-11 00:16:30,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 0: [2023-05-11 00:16:30,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +26: [2023-05-11 00:16:30,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +26: [2023-05-11 00:16:30,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +14: [2023-05-11 00:16:30,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +27: [2023-05-11 00:16:30,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +28: [2023-05-11 00:16:30,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +10: [2023-05-11 00:16:30,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +10: [2023-05-11 00:16:30,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +28: [2023-05-11 00:16:30,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +22: [2023-05-11 00:16:30,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +22: [2023-05-11 00:16:30,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +14: [2023-05-11 00:16:30,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +28: [2023-05-11 00:16:30,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +28: [2023-05-11 00:16:30,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +10: [2023-05-11 00:16:30,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +10: [2023-05-11 00:16:30,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +26: [2023-05-11 00:16:30,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +26: [2023-05-11 00:16:30,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +10: [2023-05-11 00:16:30,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +10: [2023-05-11 00:16:30,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +20: [2023-05-11 00:16:30,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +20: [2023-05-11 00:16:30,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +20: [2023-05-11 00:16:30,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +10: [2023-05-11 00:16:30,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +10: [2023-05-11 00:16:30,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +20: [2023-05-11 00:16:30,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +22: [2023-05-11 00:16:30,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +22: [2023-05-11 00:16:30,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +22: [2023-05-11 00:16:30,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +22: [2023-05-11 00:16:30,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 4: [2023-05-11 00:16:30,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 4: [2023-05-11 00:16:30,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +22: [2023-05-11 00:16:30,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +22: [2023-05-11 00:16:30,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +14: [2023-05-11 00:16:30,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +17: [2023-05-11 00:16:30,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +17: [2023-05-11 00:16:30,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +17: [2023-05-11 00:16:30,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +17: [2023-05-11 00:16:30,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +17: [2023-05-11 00:16:30,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +17: [2023-05-11 00:16:30,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +17: [2023-05-11 00:16:30,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +14: [2023-05-11 00:16:30,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +26: [2023-05-11 00:16:30,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +26: [2023-05-11 00:16:30,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +17: [2023-05-11 00:16:30,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +23: [2023-05-11 00:16:30,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +14: [2023-05-11 00:16:30,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +23: [2023-05-11 00:16:30,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 8: [2023-05-11 00:16:30,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 8: [2023-05-11 00:16:30,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 8: [2023-05-11 00:16:30,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +23: [2023-05-11 00:16:30,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +23: [2023-05-11 00:16:30,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +21: [2023-05-11 00:16:30,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +21: [2023-05-11 00:16:30,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +21: [2023-05-11 00:16:30,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +21: [2023-05-11 00:16:30,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +20: [2023-05-11 00:16:30,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +10: [2023-05-11 00:16:30,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +23: [2023-05-11 00:16:30,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +21: [2023-05-11 00:16:30,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +21: [2023-05-11 00:16:30,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +21: [2023-05-11 00:16:30,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 8: [2023-05-11 00:16:30,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +23: [2023-05-11 00:16:30,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +21: [2023-05-11 00:16:30,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +26: [2023-05-11 00:16:30,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +21: [2023-05-11 00:16:30,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +21: [2023-05-11 00:16:30,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +21: [2023-05-11 00:16:30,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +21: [2023-05-11 00:16:30,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +26: [2023-05-11 00:16:30,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +26: [2023-05-11 00:16:30,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +26: [2023-05-11 00:16:30,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +10: [2023-05-11 00:16:30,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +17: [2023-05-11 00:16:30,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +10: [2023-05-11 00:16:30,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +17: [2023-05-11 00:16:30,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +17: [2023-05-11 00:16:30,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +23: [2023-05-11 00:16:30,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +11: [2023-05-11 00:16:30,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +11: [2023-05-11 00:16:30,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +11: [2023-05-11 00:16:30,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:30,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 0: [2023-05-11 00:16:30,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +11: [2023-05-11 00:16:30,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +11: [2023-05-11 00:16:30,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +11: [2023-05-11 00:16:30,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +11: [2023-05-11 00:16:30,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +11: [2023-05-11 00:16:30,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +23: [2023-05-11 00:16:30,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +11: [2023-05-11 00:16:30,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +11: [2023-05-11 00:16:30,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +11: [2023-05-11 00:16:30,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +11: [2023-05-11 00:16:30,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +11: [2023-05-11 00:16:30,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +23: [2023-05-11 00:16:30,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +23: [2023-05-11 00:16:30,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +11: [2023-05-11 00:16:30,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +23: [2023-05-11 00:16:30,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +23: [2023-05-11 00:16:30,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 0: [2023-05-11 00:16:30,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +11: [2023-05-11 00:16:30,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +21: [2023-05-11 00:16:30,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +10: [2023-05-11 00:16:30,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +10: [2023-05-11 00:16:30,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +23: [2023-05-11 00:16:30,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +26: [2023-05-11 00:16:30,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +26: [2023-05-11 00:16:30,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +17: [2023-05-11 00:16:30,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +23: [2023-05-11 00:16:30,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 2: [2023-05-11 00:16:30,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +17: [2023-05-11 00:16:30,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +21: [2023-05-11 00:16:30,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +17: [2023-05-11 00:16:30,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +21: [2023-05-11 00:16:30,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +21: [2023-05-11 00:16:30,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +21: [2023-05-11 00:16:30,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +21: [2023-05-11 00:16:30,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +26: [2023-05-11 00:16:30,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +23: [2023-05-11 00:16:30,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +11: [2023-05-11 00:16:30,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +23: [2023-05-11 00:16:30,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +10: [2023-05-11 00:16:30,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +10: [2023-05-11 00:16:30,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +10: [2023-05-11 00:16:30,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +11: [2023-05-11 00:16:30,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +10: [2023-05-11 00:16:30,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +21: [2023-05-11 00:16:30,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +10: [2023-05-11 00:16:30,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +10: [2023-05-11 00:16:30,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +17: [2023-05-11 00:16:30,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +11: [2023-05-11 00:16:30,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +26: [2023-05-11 00:16:30,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +21: [2023-05-11 00:16:30,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +26: [2023-05-11 00:16:30,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +21: [2023-05-11 00:16:30,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +21: [2023-05-11 00:16:30,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +10: [2023-05-11 00:16:30,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +11: [2023-05-11 00:16:30,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +11: [2023-05-11 00:16:30,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +10: [2023-05-11 00:16:30,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +21: [2023-05-11 00:16:30,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +17: [2023-05-11 00:16:30,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +10: [2023-05-11 00:16:30,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +21: [2023-05-11 00:16:30,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +10: [2023-05-11 00:16:30,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +26: [2023-05-11 00:16:30,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +10: [2023-05-11 00:16:30,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +11: [2023-05-11 00:16:30,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +21: [2023-05-11 00:16:30,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +21: [2023-05-11 00:16:30,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 5: [2023-05-11 00:16:30,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +21: [2023-05-11 00:16:30,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 5: [2023-05-11 00:16:30,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 5: [2023-05-11 00:16:30,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +21: [2023-05-11 00:16:30,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +21: [2023-05-11 00:16:30,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +10: [2023-05-11 00:16:30,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +10: [2023-05-11 00:16:30,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +11: [2023-05-11 00:16:30,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +21: [2023-05-11 00:16:30,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +21: [2023-05-11 00:16:30,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +21: [2023-05-11 00:16:30,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +10: [2023-05-11 00:16:30,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +10: [2023-05-11 00:16:30,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 5: [2023-05-11 00:16:30,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 5: [2023-05-11 00:16:30,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 5: [2023-05-11 00:16:30,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +12: [2023-05-11 00:16:30,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +12: [2023-05-11 00:16:30,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +12: [2023-05-11 00:16:30,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +12: [2023-05-11 00:16:30,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +12: [2023-05-11 00:16:30,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +12: [2023-05-11 00:16:30,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +12: [2023-05-11 00:16:30,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +11: [2023-05-11 00:16:30,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +12: [2023-05-11 00:16:30,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +10: [2023-05-11 00:16:30,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +10: [2023-05-11 00:16:30,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +10: [2023-05-11 00:16:30,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +28: [2023-05-11 00:16:30,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +28: [2023-05-11 00:16:30,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +28: [2023-05-11 00:16:30,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +28: [2023-05-11 00:16:30,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +28: [2023-05-11 00:16:30,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +10: [2023-05-11 00:16:30,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +28: [2023-05-11 00:16:30,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +28: [2023-05-11 00:16:30,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +28: [2023-05-11 00:16:30,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +28: [2023-05-11 00:16:30,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +28: [2023-05-11 00:16:30,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +28: [2023-05-11 00:16:30,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +28: [2023-05-11 00:16:30,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +28: [2023-05-11 00:16:30,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +28: [2023-05-11 00:16:30,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +28: [2023-05-11 00:16:30,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +28: [2023-05-11 00:16:30,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +12: [2023-05-11 00:16:30,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +12: [2023-05-11 00:16:30,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +12: [2023-05-11 00:16:30,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +28: [2023-05-11 00:16:30,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +28: [2023-05-11 00:16:30,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +12: [2023-05-11 00:16:30,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +12: [2023-05-11 00:16:30,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +12: [2023-05-11 00:16:30,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +12: [2023-05-11 00:16:30,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +28: [2023-05-11 00:16:30,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +28: [2023-05-11 00:16:30,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +12: [2023-05-11 00:16:30,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +28: [2023-05-11 00:16:30,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +28: [2023-05-11 00:16:30,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +28: [2023-05-11 00:16:30,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +28: [2023-05-11 00:16:30,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 9: [2023-05-11 00:16:30,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 9: [2023-05-11 00:16:30,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 9: [2023-05-11 00:16:30,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 9: [2023-05-11 00:16:30,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +15: [2023-05-11 00:16:30,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 9: [2023-05-11 00:16:30,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +15: [2023-05-11 00:16:30,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +15: [2023-05-11 00:16:30,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 9: [2023-05-11 00:16:30,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +15: [2023-05-11 00:16:30,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 9: [2023-05-11 00:16:30,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +15: [2023-05-11 00:16:30,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +15: [2023-05-11 00:16:30,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:30,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +25: [2023-05-11 00:16:30,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +25: [2023-05-11 00:16:30,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:30,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:30,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +15: [2023-05-11 00:16:30,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:30,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:30,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +25: [2023-05-11 00:16:30,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +15: [2023-05-11 00:16:30,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +15: [2023-05-11 00:16:30,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +15: [2023-05-11 00:16:30,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +15: [2023-05-11 00:16:30,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +15: [2023-05-11 00:16:30,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +17: [2023-05-11 00:16:30,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +17: [2023-05-11 00:16:30,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +17: [2023-05-11 00:16:30,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +17: [2023-05-11 00:16:30,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +17: [2023-05-11 00:16:30,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +17: [2023-05-11 00:16:30,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt. +19: [2023-05-11 00:16:30,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +19: [2023-05-11 00:16:30,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +19: [2023-05-11 00:16:30,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +19: [2023-05-11 00:16:30,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +19: [2023-05-11 00:16:30,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +19: [2023-05-11 00:16:30,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +19: [2023-05-11 00:16:30,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +19: [2023-05-11 00:16:30,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +17: [2023-05-11 00:16:30,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +17: [2023-05-11 00:16:30,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +15: [2023-05-11 00:16:30,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +15: [2023-05-11 00:16:30,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +16: [2023-05-11 00:16:30,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +16: [2023-05-11 00:16:30,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 4: [2023-05-11 00:16:30,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +16: [2023-05-11 00:16:30,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 4: [2023-05-11 00:16:30,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +15: [2023-05-11 00:16:30,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +15: [2023-05-11 00:16:30,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 9: [2023-05-11 00:16:30,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +16: [2023-05-11 00:16:30,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +17: [2023-05-11 00:16:30,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +17: [2023-05-11 00:16:30,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +17: [2023-05-11 00:16:30,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +17: [2023-05-11 00:16:30,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 9: [2023-05-11 00:16:30,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +16: [2023-05-11 00:16:30,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:30,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +25: [2023-05-11 00:16:30,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +17: [2023-05-11 00:16:30,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... + 9: [2023-05-11 00:16:30,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +17: [2023-05-11 00:16:30,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +19: [2023-05-11 00:16:30,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +19: [2023-05-11 00:16:30,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +19: [2023-05-11 00:16:30,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +19: [2023-05-11 00:16:30,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +17: [2023-05-11 00:16:30,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +19: [2023-05-11 00:16:30,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +17: [2023-05-11 00:16:30,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt... +19: [2023-05-11 00:16:30,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +16: [2023-05-11 00:16:30,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +16: [2023-05-11 00:16:30,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:30,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +15: [2023-05-11 00:16:30,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +25: [2023-05-11 00:16:30,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +15: [2023-05-11 00:16:30,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 9: [2023-05-11 00:16:30,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +15: [2023-05-11 00:16:30,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +16: [2023-05-11 00:16:30,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +12: [2023-05-11 00:16:30,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +12: [2023-05-11 00:16:30,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +12: [2023-05-11 00:16:30,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 9: [2023-05-11 00:16:30,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 4: [2023-05-11 00:16:30,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 4: [2023-05-11 00:16:30,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +15: [2023-05-11 00:16:30,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +12: [2023-05-11 00:16:30,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +19: [2023-05-11 00:16:30,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +17: [2023-05-11 00:16:30,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 7: [2023-05-11 00:16:30,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +15: [2023-05-11 00:16:30,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +15: [2023-05-11 00:16:30,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 7: [2023-05-11 00:16:30,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +19: [2023-05-11 00:16:30,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +15: [2023-05-11 00:16:30,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +19: [2023-05-11 00:16:30,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +17: [2023-05-11 00:16:30,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +30: [2023-05-11 00:16:30,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +25: [2023-05-11 00:16:30,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +25: [2023-05-11 00:16:30,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +30: [2023-05-11 00:16:30,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +30: [2023-05-11 00:16:30,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +15: [2023-05-11 00:16:30,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +30: [2023-05-11 00:16:30,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +12: [2023-05-11 00:16:30,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +12: [2023-05-11 00:16:30,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +13: [2023-05-11 00:16:30,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +13: [2023-05-11 00:16:30,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 4: [2023-05-11 00:16:30,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +13: [2023-05-11 00:16:30,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +13: [2023-05-11 00:16:30,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +19: [2023-05-11 00:16:30,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +15: [2023-05-11 00:16:30,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +19: [2023-05-11 00:16:30,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +12: [2023-05-11 00:16:30,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +19: [2023-05-11 00:16:30,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +19: [2023-05-11 00:16:30,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 4: [2023-05-11 00:16:30,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +15: [2023-05-11 00:16:30,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +15: [2023-05-11 00:16:30,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +19: [2023-05-11 00:16:30,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:30,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +12: [2023-05-11 00:16:30,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:30,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +17: [2023-05-11 00:16:30,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +17: [2023-05-11 00:16:30,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +15: [2023-05-11 00:16:30,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +17: [2023-05-11 00:16:30,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +19: [2023-05-11 00:16:30,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +17: [2023-05-11 00:16:30,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:30,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +30: [2023-05-11 00:16:30,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +15: [2023-05-11 00:16:30,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +13: [2023-05-11 00:16:30,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +30: [2023-05-11 00:16:30,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:30,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 7: [2023-05-11 00:16:30,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +13: [2023-05-11 00:16:30,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +17: [2023-05-11 00:16:30,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +15: [2023-05-11 00:16:30,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +17: [2023-05-11 00:16:30,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +30: [2023-05-11 00:16:30,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +17: [2023-05-11 00:16:30,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +17: [2023-05-11 00:16:30,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +30: [2023-05-11 00:16:30,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:30,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +15: [2023-05-11 00:16:30,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +13: [2023-05-11 00:16:30,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +13: [2023-05-11 00:16:30,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:30,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +19: [2023-05-11 00:16:30,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +19: [2023-05-11 00:16:30,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +19: [2023-05-11 00:16:30,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +14: [2023-05-11 00:16:30,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +14: [2023-05-11 00:16:30,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +29: [2023-05-11 00:16:30,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +14: [2023-05-11 00:16:30,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +14: [2023-05-11 00:16:30,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +29: [2023-05-11 00:16:30,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +29: [2023-05-11 00:16:30,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +29: [2023-05-11 00:16:30,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +17: [2023-05-11 00:16:30,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt. +17: [2023-05-11 00:16:30,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt... +17: [2023-05-11 00:16:30,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +15: [2023-05-11 00:16:30,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +12: [2023-05-11 00:16:30,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +12: [2023-05-11 00:16:30,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +12: [2023-05-11 00:16:30,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +12: [2023-05-11 00:16:30,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +12: [2023-05-11 00:16:30,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +12: [2023-05-11 00:16:30,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +17: [2023-05-11 00:16:30,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +19: [2023-05-11 00:16:30,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +19: [2023-05-11 00:16:30,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +25: [2023-05-11 00:16:30,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:30,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:30,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +24: [2023-05-11 00:16:30,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +24: [2023-05-11 00:16:30,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +24: [2023-05-11 00:16:30,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +24: [2023-05-11 00:16:30,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +14: [2023-05-11 00:16:30,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +14: [2023-05-11 00:16:30,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:30,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +17: [2023-05-11 00:16:30,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +29: [2023-05-11 00:16:30,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +29: [2023-05-11 00:16:30,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +14: [2023-05-11 00:16:30,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +14: [2023-05-11 00:16:30,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +17: [2023-05-11 00:16:30,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +29: [2023-05-11 00:16:30,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +29: [2023-05-11 00:16:30,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +24: [2023-05-11 00:16:30,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +24: [2023-05-11 00:16:30,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +24: [2023-05-11 00:16:30,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +24: [2023-05-11 00:16:30,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +12: [2023-05-11 00:16:30,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +12: [2023-05-11 00:16:30,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +19: [2023-05-11 00:16:30,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +19: [2023-05-11 00:16:30,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +19: [2023-05-11 00:16:30,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +25: [2023-05-11 00:16:30,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +25: [2023-05-11 00:16:30,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +25: [2023-05-11 00:16:30,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +19: [2023-05-11 00:16:30,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 5: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +16: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +16: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +16: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +16: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +16: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +16: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +16: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +16: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 5: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +16: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 5: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +16: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 5: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +16: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +16: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +16: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +16: [2023-05-11 00:16:30,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:30,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +25: [2023-05-11 00:16:30,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 5: [2023-05-11 00:16:30,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 1: [2023-05-11 00:16:30,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +25: [2023-05-11 00:16:30,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 1: [2023-05-11 00:16:30,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 5: [2023-05-11 00:16:30,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +25: [2023-05-11 00:16:30,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +25: [2023-05-11 00:16:30,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 5: [2023-05-11 00:16:30,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +16: [2023-05-11 00:16:30,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +16: [2023-05-11 00:16:30,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 1: [2023-05-11 00:16:30,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 7: [2023-05-11 00:16:30,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 7: [2023-05-11 00:16:30,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 7: [2023-05-11 00:16:30,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 7: [2023-05-11 00:16:30,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +16: [2023-05-11 00:16:30,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 5: [2023-05-11 00:16:30,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +16: [2023-05-11 00:16:30,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +16: [2023-05-11 00:16:30,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +27: [2023-05-11 00:16:30,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +16: [2023-05-11 00:16:30,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +27: [2023-05-11 00:16:30,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +27: [2023-05-11 00:16:30,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +27: [2023-05-11 00:16:30,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 7: [2023-05-11 00:16:30,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +29: [2023-05-11 00:16:30,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +29: [2023-05-11 00:16:30,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +29: [2023-05-11 00:16:30,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +29: [2023-05-11 00:16:30,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +29: [2023-05-11 00:16:30,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +29: [2023-05-11 00:16:30,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +29: [2023-05-11 00:16:30,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +29: [2023-05-11 00:16:30,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +29: [2023-05-11 00:16:30,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +29: [2023-05-11 00:16:30,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +29: [2023-05-11 00:16:30,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +29: [2023-05-11 00:16:30,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +29: [2023-05-11 00:16:30,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +29: [2023-05-11 00:16:30,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +29: [2023-05-11 00:16:30,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 5: [2023-05-11 00:16:30,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +29: [2023-05-11 00:16:30,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +23: [2023-05-11 00:16:30,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +23: [2023-05-11 00:16:30,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +30: [2023-05-11 00:16:30,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +30: [2023-05-11 00:16:30,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +30: [2023-05-11 00:16:30,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +30: [2023-05-11 00:16:30,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +30: [2023-05-11 00:16:30,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +30: [2023-05-11 00:16:30,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +30: [2023-05-11 00:16:30,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +16: [2023-05-11 00:16:30,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +30: [2023-05-11 00:16:30,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +30: [2023-05-11 00:16:30,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +30: [2023-05-11 00:16:30,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +30: [2023-05-11 00:16:30,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +22: [2023-05-11 00:16:30,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +30: [2023-05-11 00:16:30,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +30: [2023-05-11 00:16:30,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +22: [2023-05-11 00:16:30,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +23: [2023-05-11 00:16:30,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +16: [2023-05-11 00:16:30,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +18: [2023-05-11 00:16:30,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +18: [2023-05-11 00:16:30,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +18: [2023-05-11 00:16:30,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +18: [2023-05-11 00:16:30,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +18: [2023-05-11 00:16:30,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 5: [2023-05-11 00:16:30,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 5: [2023-05-11 00:16:30,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +18: [2023-05-11 00:16:30,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +22: [2023-05-11 00:16:30,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +18: [2023-05-11 00:16:30,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +18: [2023-05-11 00:16:30,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +30: [2023-05-11 00:16:30,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +23: [2023-05-11 00:16:30,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +30: [2023-05-11 00:16:30,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +30: [2023-05-11 00:16:30,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +22: [2023-05-11 00:16:30,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 5: [2023-05-11 00:16:30,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +22: [2023-05-11 00:16:30,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +22: [2023-05-11 00:16:30,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +27: [2023-05-11 00:16:30,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +27: [2023-05-11 00:16:30,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +27: [2023-05-11 00:16:30,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +16: [2023-05-11 00:16:30,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +27: [2023-05-11 00:16:30,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +22: [2023-05-11 00:16:30,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +22: [2023-05-11 00:16:30,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +16: [2023-05-11 00:16:30,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +23: [2023-05-11 00:16:30,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 7: [2023-05-11 00:16:30,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +23: [2023-05-11 00:16:30,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 1: [2023-05-11 00:16:30,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 5: [2023-05-11 00:16:30,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +22: [2023-05-11 00:16:30,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +29: [2023-05-11 00:16:30,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +23: [2023-05-11 00:16:30,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +23: [2023-05-11 00:16:30,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +30: [2023-05-11 00:16:30,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +30: [2023-05-11 00:16:30,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +30: [2023-05-11 00:16:30,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +24: [2023-05-11 00:16:30,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +24: [2023-05-11 00:16:30,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +24: [2023-05-11 00:16:30,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +24: [2023-05-11 00:16:30,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +24: [2023-05-11 00:16:30,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 1: [2023-05-11 00:16:30,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +24: [2023-05-11 00:16:30,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +24: [2023-05-11 00:16:30,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +24: [2023-05-11 00:16:30,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 3: [2023-05-11 00:16:30,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +18: [2023-05-11 00:16:30,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +18: [2023-05-11 00:16:30,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +18: [2023-05-11 00:16:30,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +24: [2023-05-11 00:16:30,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +24: [2023-05-11 00:16:30,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +24: [2023-05-11 00:16:30,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +24: [2023-05-11 00:16:30,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +24: [2023-05-11 00:16:30,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +24: [2023-05-11 00:16:30,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +18: [2023-05-11 00:16:30,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +22: [2023-05-11 00:16:30,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +29: [2023-05-11 00:16:30,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +29: [2023-05-11 00:16:30,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +20: [2023-05-11 00:16:30,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +20: [2023-05-11 00:16:30,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +20: [2023-05-11 00:16:30,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 3: [2023-05-11 00:16:30,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +20: [2023-05-11 00:16:30,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 3: [2023-05-11 00:16:30,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +29: [2023-05-11 00:16:30,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 7: [2023-05-11 00:16:30,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +20: [2023-05-11 00:16:30,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +20: [2023-05-11 00:16:30,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +20: [2023-05-11 00:16:30,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +20: [2023-05-11 00:16:30,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +20: [2023-05-11 00:16:30,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +20: [2023-05-11 00:16:30,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +20: [2023-05-11 00:16:30,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +20: [2023-05-11 00:16:30,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +30: [2023-05-11 00:16:30,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +30: [2023-05-11 00:16:30,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +29: [2023-05-11 00:16:30,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +30: [2023-05-11 00:16:30,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +24: [2023-05-11 00:16:30,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +24: [2023-05-11 00:16:30,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +18: [2023-05-11 00:16:30,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +18: [2023-05-11 00:16:30,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +21: [2023-05-11 00:16:30,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +21: [2023-05-11 00:16:30,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +21: [2023-05-11 00:16:30,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +21: [2023-05-11 00:16:30,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +21: [2023-05-11 00:16:30,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +21: [2023-05-11 00:16:30,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +22: [2023-05-11 00:16:30,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +22: [2023-05-11 00:16:30,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +21: [2023-05-11 00:16:30,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +24: [2023-05-11 00:16:30,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 6: [2023-05-11 00:16:30,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +18: [2023-05-11 00:16:30,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +18: [2023-05-11 00:16:30,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +30: [2023-05-11 00:16:30,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +20: [2023-05-11 00:16:30,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +22: [2023-05-11 00:16:30,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +21: [2023-05-11 00:16:30,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +10: [2023-05-11 00:16:30,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +10: [2023-05-11 00:16:30,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +10: [2023-05-11 00:16:30,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +20: [2023-05-11 00:16:30,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +20: [2023-05-11 00:16:30,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +29: [2023-05-11 00:16:30,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +29: [2023-05-11 00:16:30,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +20: [2023-05-11 00:16:30,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +10: [2023-05-11 00:16:30,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +29: [2023-05-11 00:16:30,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +24: [2023-05-11 00:16:30,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +24: [2023-05-11 00:16:30,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +22: [2023-05-11 00:16:30,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +22: [2023-05-11 00:16:30,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +30: [2023-05-11 00:16:30,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +13: [2023-05-11 00:16:30,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +22: [2023-05-11 00:16:30,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +13: [2023-05-11 00:16:30,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +13: [2023-05-11 00:16:30,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +13: [2023-05-11 00:16:30,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +21: [2023-05-11 00:16:30,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +13: [2023-05-11 00:16:30,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +13: [2023-05-11 00:16:30,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +13: [2023-05-11 00:16:30,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +24: [2023-05-11 00:16:30,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +13: [2023-05-11 00:16:30,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +13: [2023-05-11 00:16:30,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +13: [2023-05-11 00:16:30,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +13: [2023-05-11 00:16:30,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +13: [2023-05-11 00:16:30,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +13: [2023-05-11 00:16:30,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +13: [2023-05-11 00:16:30,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +13: [2023-05-11 00:16:30,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +13: [2023-05-11 00:16:30,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 2: [2023-05-11 00:16:30,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +11: [2023-05-11 00:16:30,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +11: [2023-05-11 00:16:30,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +11: [2023-05-11 00:16:30,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +11: [2023-05-11 00:16:30,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +11: [2023-05-11 00:16:30,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +11: [2023-05-11 00:16:30,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +11: [2023-05-11 00:16:30,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 2: [2023-05-11 00:16:30,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 8: [2023-05-11 00:16:30,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +10: [2023-05-11 00:16:30,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +10: [2023-05-11 00:16:30,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +11: [2023-05-11 00:16:30,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 2: [2023-05-11 00:16:30,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +11: [2023-05-11 00:16:30,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +11: [2023-05-11 00:16:30,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +11: [2023-05-11 00:16:30,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +11: [2023-05-11 00:16:30,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +20: [2023-05-11 00:16:30,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +20: [2023-05-11 00:16:30,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +24: [2023-05-11 00:16:30,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +21: [2023-05-11 00:16:30,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +21: [2023-05-11 00:16:30,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 2: [2023-05-11 00:16:30,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +10: [2023-05-11 00:16:30,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 2: [2023-05-11 00:16:30,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +27: [2023-05-11 00:16:30,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +27: [2023-05-11 00:16:30,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +27: [2023-05-11 00:16:30,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +27: [2023-05-11 00:16:30,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +27: [2023-05-11 00:16:30,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +10: [2023-05-11 00:16:30,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +31: [2023-05-11 00:16:30,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +31: [2023-05-11 00:16:30,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +31: [2023-05-11 00:16:30,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +31: [2023-05-11 00:16:30,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +31: [2023-05-11 00:16:30,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +31: [2023-05-11 00:16:30,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +27: [2023-05-11 00:16:30,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 6: [2023-05-11 00:16:30,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +18: [2023-05-11 00:16:30,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +18: [2023-05-11 00:16:30,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 3: [2023-05-11 00:16:30,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +27: [2023-05-11 00:16:30,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +27: [2023-05-11 00:16:30,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +27: [2023-05-11 00:16:30,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +27: [2023-05-11 00:16:30,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +27: [2023-05-11 00:16:30,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +27: [2023-05-11 00:16:30,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 3: [2023-05-11 00:16:30,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +21: [2023-05-11 00:16:30,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +24: [2023-05-11 00:16:30,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +24: [2023-05-11 00:16:30,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +18: [2023-05-11 00:16:30,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 1: [2023-05-11 00:16:30,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 1: [2023-05-11 00:16:30,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +27: [2023-05-11 00:16:30,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 8: [2023-05-11 00:16:30,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +27: [2023-05-11 00:16:30,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +18: [2023-05-11 00:16:30,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 1: [2023-05-11 00:16:30,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 6: [2023-05-11 00:16:30,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 1: [2023-05-11 00:16:30,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 8: [2023-05-11 00:16:30,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 0: [2023-05-11 00:16:30,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 0: [2023-05-11 00:16:30,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +21: [2023-05-11 00:16:30,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +21: [2023-05-11 00:16:30,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +31: [2023-05-11 00:16:30,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +31: [2023-05-11 00:16:30,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 3: [2023-05-11 00:16:30,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +23: [2023-05-11 00:16:30,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +23: [2023-05-11 00:16:30,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +23: [2023-05-11 00:16:30,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +23: [2023-05-11 00:16:30,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +23: [2023-05-11 00:16:30,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +23: [2023-05-11 00:16:30,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +23: [2023-05-11 00:16:30,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +23: [2023-05-11 00:16:30,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +23: [2023-05-11 00:16:30,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +23: [2023-05-11 00:16:30,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +23: [2023-05-11 00:16:30,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +31: [2023-05-11 00:16:30,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +23: [2023-05-11 00:16:30,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +23: [2023-05-11 00:16:30,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +23: [2023-05-11 00:16:30,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +31: [2023-05-11 00:16:30,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 3: [2023-05-11 00:16:30,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +31: [2023-05-11 00:16:30,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +23: [2023-05-11 00:16:30,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +27: [2023-05-11 00:16:30,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +23: [2023-05-11 00:16:30,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +27: [2023-05-11 00:16:30,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +21: [2023-05-11 00:16:30,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +20: [2023-05-11 00:16:30,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +20: [2023-05-11 00:16:30,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +20: [2023-05-11 00:16:30,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +20: [2023-05-11 00:16:30,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +24: [2023-05-11 00:16:30,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +22: [2023-05-11 00:16:30,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +22: [2023-05-11 00:16:30,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +22: [2023-05-11 00:16:30,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +20: [2023-05-11 00:16:30,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +20: [2023-05-11 00:16:30,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +22: [2023-05-11 00:16:30,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +26: [2023-05-11 00:16:30,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +26: [2023-05-11 00:16:30,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +26: [2023-05-11 00:16:30,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +26: [2023-05-11 00:16:30,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +26: [2023-05-11 00:16:30,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +26: [2023-05-11 00:16:30,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +26: [2023-05-11 00:16:30,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +26: [2023-05-11 00:16:30,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +31: [2023-05-11 00:16:30,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +11: [2023-05-11 00:16:30,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 2: [2023-05-11 00:16:30,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +11: [2023-05-11 00:16:30,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +13: [2023-05-11 00:16:30,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +13: [2023-05-11 00:16:30,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +13: [2023-05-11 00:16:30,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 3: [2023-05-11 00:16:30,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +18: [2023-05-11 00:16:30,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +11: [2023-05-11 00:16:30,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 8: [2023-05-11 00:16:30,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +11: [2023-05-11 00:16:30,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +11: [2023-05-11 00:16:30,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +21: [2023-05-11 00:16:30,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 6: [2023-05-11 00:16:30,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +21: [2023-05-11 00:16:30,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +21: [2023-05-11 00:16:30,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 0: [2023-05-11 00:16:30,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +13: [2023-05-11 00:16:30,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +14: [2023-05-11 00:16:30,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +14: [2023-05-11 00:16:30,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 3: [2023-05-11 00:16:30,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +14: [2023-05-11 00:16:30,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +14: [2023-05-11 00:16:30,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +14: [2023-05-11 00:16:30,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +14: [2023-05-11 00:16:31,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +14: [2023-05-11 00:16:31,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +14: [2023-05-11 00:16:31,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +14: [2023-05-11 00:16:31,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +21: [2023-05-11 00:16:31,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +14: [2023-05-11 00:16:31,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +14: [2023-05-11 00:16:31,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +14: [2023-05-11 00:16:31,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 1: [2023-05-11 00:16:31,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +14: [2023-05-11 00:16:31,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +14: [2023-05-11 00:16:31,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +14: [2023-05-11 00:16:31,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +14: [2023-05-11 00:16:31,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +27: [2023-05-11 00:16:31,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +10: [2023-05-11 00:16:31,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +10: [2023-05-11 00:16:31,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +10: [2023-05-11 00:16:31,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +10: [2023-05-11 00:16:31,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +18: [2023-05-11 00:16:31,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +21: [2023-05-11 00:16:31,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +31: [2023-05-11 00:16:31,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +18: [2023-05-11 00:16:31,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +18: [2023-05-11 00:16:31,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +23: [2023-05-11 00:16:31,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +21: [2023-05-11 00:16:31,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +31: [2023-05-11 00:16:31,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +31: [2023-05-11 00:16:31,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +27: [2023-05-11 00:16:31,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +31: [2023-05-11 00:16:31,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +22: [2023-05-11 00:16:31,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +22: [2023-05-11 00:16:31,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +27: [2023-05-11 00:16:31,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 9: [2023-05-11 00:16:31,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 3: [2023-05-11 00:16:31,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +10: [2023-05-11 00:16:31,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +10: [2023-05-11 00:16:31,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +22: [2023-05-11 00:16:31,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +20: [2023-05-11 00:16:31,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +20: [2023-05-11 00:16:31,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +20: [2023-05-11 00:16:31,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +20: [2023-05-11 00:16:31,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 8: [2023-05-11 00:16:31,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +10: [2023-05-11 00:16:31,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +10: [2023-05-11 00:16:31,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +10: [2023-05-11 00:16:31,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +10: [2023-05-11 00:16:31,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +11: [2023-05-11 00:16:31,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +11: [2023-05-11 00:16:31,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +13: [2023-05-11 00:16:31,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +31: [2023-05-11 00:16:31,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +20: [2023-05-11 00:16:31,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +20: [2023-05-11 00:16:31,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +11: [2023-05-11 00:16:31,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +21: [2023-05-11 00:16:31,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +22: [2023-05-11 00:16:31,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +27: [2023-05-11 00:16:31,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +23: [2023-05-11 00:16:31,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +23: [2023-05-11 00:16:31,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +10: [2023-05-11 00:16:31,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +10: [2023-05-11 00:16:31,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +31: [2023-05-11 00:16:31,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +23: [2023-05-11 00:16:31,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +13: [2023-05-11 00:16:31,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +13: [2023-05-11 00:16:31,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +11: [2023-05-11 00:16:31,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +13: [2023-05-11 00:16:31,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +14: [2023-05-11 00:16:31,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +11: [2023-05-11 00:16:31,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +18: [2023-05-11 00:16:31,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +27: [2023-05-11 00:16:31,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +21: [2023-05-11 00:16:31,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +18: [2023-05-11 00:16:31,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 8: [2023-05-11 00:16:31,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +11: [2023-05-11 00:16:31,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +11: [2023-05-11 00:16:31,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 6: [2023-05-11 00:16:31,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +10: [2023-05-11 00:16:31,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +10: [2023-05-11 00:16:31,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +10: [2023-05-11 00:16:31,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +31: [2023-05-11 00:16:31,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +11: [2023-05-11 00:16:31,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +27: [2023-05-11 00:16:31,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +27: [2023-05-11 00:16:31,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +31: [2023-05-11 00:16:31,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 6: [2023-05-11 00:16:31,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 9: [2023-05-11 00:16:31,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +21: [2023-05-11 00:16:31,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +31: [2023-05-11 00:16:31,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +11: [2023-05-11 00:16:31,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +21: [2023-05-11 00:16:31,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +22: [2023-05-11 00:16:31,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +22: [2023-05-11 00:16:31,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +28: [2023-05-11 00:16:31,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +28: [2023-05-11 00:16:31,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +28: [2023-05-11 00:16:31,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +14: [2023-05-11 00:16:31,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +28: [2023-05-11 00:16:31,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 0: [2023-05-11 00:16:31,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +23: [2023-05-11 00:16:31,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +10: [2023-05-11 00:16:31,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +23: [2023-05-11 00:16:31,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +23: [2023-05-11 00:16:31,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +26: [2023-05-11 00:16:31,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +26: [2023-05-11 00:16:31,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +26: [2023-05-11 00:16:31,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +10: [2023-05-11 00:16:31,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +18: [2023-05-11 00:16:31,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +18: [2023-05-11 00:16:31,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +18: [2023-05-11 00:16:31,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +18: [2023-05-11 00:16:31,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +27: [2023-05-11 00:16:31,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +20: [2023-05-11 00:16:31,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 3: [2023-05-11 00:16:31,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +10: [2023-05-11 00:16:31,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +11: [2023-05-11 00:16:31,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +14: [2023-05-11 00:16:31,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +22: [2023-05-11 00:16:31,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +22: [2023-05-11 00:16:31,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 2: [2023-05-11 00:16:31,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 2: [2023-05-11 00:16:31,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +20: [2023-05-11 00:16:31,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +22: [2023-05-11 00:16:31,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +21: [2023-05-11 00:16:31,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 9: [2023-05-11 00:16:31,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 8: [2023-05-11 00:16:31,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +10: [2023-05-11 00:16:31,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 0: [2023-05-11 00:16:31,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 0: [2023-05-11 00:16:31,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 8: [2023-05-11 00:16:31,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +11: [2023-05-11 00:16:31,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +22: [2023-05-11 00:16:31,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +18: [2023-05-11 00:16:31,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +18: [2023-05-11 00:16:31,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 2: [2023-05-11 00:16:31,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +10: [2023-05-11 00:16:31,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +10: [2023-05-11 00:16:31,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +26: [2023-05-11 00:16:31,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +26: [2023-05-11 00:16:31,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 3: [2023-05-11 00:16:31,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +21: [2023-05-11 00:16:31,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +21: [2023-05-11 00:16:31,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 3: [2023-05-11 00:16:31,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +31: [2023-05-11 00:16:31,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +11: [2023-05-11 00:16:31,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +21: [2023-05-11 00:16:31,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +11: [2023-05-11 00:16:31,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +21: [2023-05-11 00:16:31,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 8: [2023-05-11 00:16:31,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +11: [2023-05-11 00:16:31,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +11: [2023-05-11 00:16:31,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +26: [2023-05-11 00:16:31,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +23: [2023-05-11 00:16:31,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +26: [2023-05-11 00:16:31,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 0: [2023-05-11 00:16:31,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +22: [2023-05-11 00:16:31,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +28: [2023-05-11 00:16:31,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +22: [2023-05-11 00:16:31,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +28: [2023-05-11 00:16:31,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +28: [2023-05-11 00:16:31,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +28: [2023-05-11 00:16:31,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +28: [2023-05-11 00:16:31,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +21: [2023-05-11 00:16:31,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +28: [2023-05-11 00:16:31,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +26: [2023-05-11 00:16:31,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +26: [2023-05-11 00:16:31,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +28: [2023-05-11 00:16:31,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +28: [2023-05-11 00:16:31,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +28: [2023-05-11 00:16:31,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +28: [2023-05-11 00:16:31,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +28: [2023-05-11 00:16:31,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +31: [2023-05-11 00:16:31,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +31: [2023-05-11 00:16:31,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +31: [2023-05-11 00:16:31,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +31: [2023-05-11 00:16:31,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +31: [2023-05-11 00:16:31,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 6: [2023-05-11 00:16:31,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +10: [2023-05-11 00:16:31,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +21: [2023-05-11 00:16:31,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +14: [2023-05-11 00:16:31,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +28: [2023-05-11 00:16:31,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +26: [2023-05-11 00:16:31,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 0: [2023-05-11 00:16:31,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +26: [2023-05-11 00:16:31,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +10: [2023-05-11 00:16:31,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +31: [2023-05-11 00:16:31,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +26: [2023-05-11 00:16:31,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:31,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +25: [2023-05-11 00:16:31,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +25: [2023-05-11 00:16:31,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +25: [2023-05-11 00:16:31,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +31: [2023-05-11 00:16:31,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +14: [2023-05-11 00:16:31,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +10: [2023-05-11 00:16:31,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +31: [2023-05-11 00:16:31,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +31: [2023-05-11 00:16:31,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +25: [2023-05-11 00:16:31,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +25: [2023-05-11 00:16:31,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +25: [2023-05-11 00:16:31,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:31,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +19: [2023-05-11 00:16:31,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +19: [2023-05-11 00:16:31,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +19: [2023-05-11 00:16:31,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +26: [2023-05-11 00:16:31,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +26: [2023-05-11 00:16:31,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +26: [2023-05-11 00:16:31,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +26: [2023-05-11 00:16:31,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +28: [2023-05-11 00:16:31,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +14: [2023-05-11 00:16:31,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +28: [2023-05-11 00:16:31,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +14: [2023-05-11 00:16:31,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +28: [2023-05-11 00:16:31,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +15: [2023-05-11 00:16:31,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +28: [2023-05-11 00:16:31,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +28: [2023-05-11 00:16:31,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +15: [2023-05-11 00:16:31,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +15: [2023-05-11 00:16:31,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +28: [2023-05-11 00:16:31,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +15: [2023-05-11 00:16:31,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +17: [2023-05-11 00:16:31,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +17: [2023-05-11 00:16:31,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +17: [2023-05-11 00:16:31,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +17: [2023-05-11 00:16:31,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +31: [2023-05-11 00:16:31,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +17: [2023-05-11 00:16:31,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +17: [2023-05-11 00:16:31,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +17: [2023-05-11 00:16:31,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +17: [2023-05-11 00:16:31,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +19: [2023-05-11 00:16:31,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +17: [2023-05-11 00:16:31,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. + 9: [2023-05-11 00:16:31,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 9: [2023-05-11 00:16:31,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +19: [2023-05-11 00:16:31,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +14: [2023-05-11 00:16:31,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +26: [2023-05-11 00:16:31,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +12: [2023-05-11 00:16:31,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +12: [2023-05-11 00:16:31,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +17: [2023-05-11 00:16:31,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +25: [2023-05-11 00:16:31,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +25: [2023-05-11 00:16:31,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +17: [2023-05-11 00:16:31,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +25: [2023-05-11 00:16:31,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +25: [2023-05-11 00:16:31,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +16: [2023-05-11 00:16:31,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +16: [2023-05-11 00:16:31,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +26: [2023-05-11 00:16:31,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +17: [2023-05-11 00:16:31,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt. +16: [2023-05-11 00:16:31,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 0: [2023-05-11 00:16:31,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +19: [2023-05-11 00:16:31,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +12: [2023-05-11 00:16:31,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +12: [2023-05-11 00:16:31,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +12: [2023-05-11 00:16:31,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +12: [2023-05-11 00:16:31,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +12: [2023-05-11 00:16:31,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +12: [2023-05-11 00:16:31,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 0: [2023-05-11 00:16:31,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +28: [2023-05-11 00:16:31,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +28: [2023-05-11 00:16:31,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +16: [2023-05-11 00:16:31,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +28: [2023-05-11 00:16:31,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +26: [2023-05-11 00:16:31,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +28: [2023-05-11 00:16:31,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 9: [2023-05-11 00:16:31,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 9: [2023-05-11 00:16:31,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 0: [2023-05-11 00:16:31,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +28: [2023-05-11 00:16:31,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +26: [2023-05-11 00:16:31,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 9: [2023-05-11 00:16:31,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +15: [2023-05-11 00:16:31,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +15: [2023-05-11 00:16:31,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +15: [2023-05-11 00:16:31,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +15: [2023-05-11 00:16:31,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +26: [2023-05-11 00:16:31,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +26: [2023-05-11 00:16:31,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 5: [2023-05-11 00:16:31,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 5: [2023-05-11 00:16:31,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 5: [2023-05-11 00:16:31,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +28: [2023-05-11 00:16:31,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +16: [2023-05-11 00:16:31,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +16: [2023-05-11 00:16:31,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +17: [2023-05-11 00:16:31,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +17: [2023-05-11 00:16:31,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +25: [2023-05-11 00:16:31,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +17: [2023-05-11 00:16:31,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +25: [2023-05-11 00:16:31,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +25: [2023-05-11 00:16:31,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +17: [2023-05-11 00:16:31,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt... +25: [2023-05-11 00:16:31,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +28: [2023-05-11 00:16:31,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +12: [2023-05-11 00:16:31,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +16: [2023-05-11 00:16:31,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +16: [2023-05-11 00:16:31,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +28: [2023-05-11 00:16:31,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +26: [2023-05-11 00:16:31,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +25: [2023-05-11 00:16:31,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +26: [2023-05-11 00:16:31,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +25: [2023-05-11 00:16:31,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +25: [2023-05-11 00:16:31,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +25: [2023-05-11 00:16:31,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +28: [2023-05-11 00:16:31,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 4: [2023-05-11 00:16:31,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 4: [2023-05-11 00:16:31,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +17: [2023-05-11 00:16:31,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +25: [2023-05-11 00:16:31,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +28: [2023-05-11 00:16:31,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 5: [2023-05-11 00:16:31,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +12: [2023-05-11 00:16:31,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +12: [2023-05-11 00:16:31,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +12: [2023-05-11 00:16:31,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +12: [2023-05-11 00:16:31,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 9: [2023-05-11 00:16:31,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +12: [2023-05-11 00:16:31,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +12: [2023-05-11 00:16:31,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +12: [2023-05-11 00:16:31,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 9: [2023-05-11 00:16:31,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +12: [2023-05-11 00:16:31,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +17: [2023-05-11 00:16:31,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +25: [2023-05-11 00:16:31,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +25: [2023-05-11 00:16:31,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +17: [2023-05-11 00:16:31,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +17: [2023-05-11 00:16:31,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +15: [2023-05-11 00:16:31,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +15: [2023-05-11 00:16:31,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +15: [2023-05-11 00:16:31,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +15: [2023-05-11 00:16:31,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +15: [2023-05-11 00:16:31,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +15: [2023-05-11 00:16:31,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +15: [2023-05-11 00:16:31,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +15: [2023-05-11 00:16:31,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +15: [2023-05-11 00:16:31,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +15: [2023-05-11 00:16:31,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +15: [2023-05-11 00:16:31,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +15: [2023-05-11 00:16:31,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +15: [2023-05-11 00:16:31,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +15: [2023-05-11 00:16:31,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +17: [2023-05-11 00:16:31,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +25: [2023-05-11 00:16:31,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +15: [2023-05-11 00:16:31,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +17: [2023-05-11 00:16:31,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +17: [2023-05-11 00:16:31,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +17: [2023-05-11 00:16:31,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... + 4: [2023-05-11 00:16:31,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +15: [2023-05-11 00:16:31,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +17: [2023-05-11 00:16:31,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +12: [2023-05-11 00:16:31,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +12: [2023-05-11 00:16:31,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +12: [2023-05-11 00:16:31,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +25: [2023-05-11 00:16:31,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +25: [2023-05-11 00:16:31,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 4: [2023-05-11 00:16:31,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +17: [2023-05-11 00:16:31,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +17: [2023-05-11 00:16:31,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +25: [2023-05-11 00:16:31,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +25: [2023-05-11 00:16:31,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +12: [2023-05-11 00:16:31,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +25: [2023-05-11 00:16:31,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +17: [2023-05-11 00:16:31,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +17: [2023-05-11 00:16:31,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +12: [2023-05-11 00:16:31,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +12: [2023-05-11 00:16:31,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +17: [2023-05-11 00:16:31,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt. +17: [2023-05-11 00:16:31,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt... +12: [2023-05-11 00:16:31,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +25: [2023-05-11 00:16:31,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +25: [2023-05-11 00:16:31,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +15: [2023-05-11 00:16:31,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +25: [2023-05-11 00:16:31,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 4: [2023-05-11 00:16:31,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +15: [2023-05-11 00:16:31,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +15: [2023-05-11 00:16:31,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +15: [2023-05-11 00:16:31,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +17: [2023-05-11 00:16:31,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +30: [2023-05-11 00:16:31,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +30: [2023-05-11 00:16:31,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +19: [2023-05-11 00:16:31,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +29: [2023-05-11 00:16:31,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +29: [2023-05-11 00:16:31,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +29: [2023-05-11 00:16:31,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +29: [2023-05-11 00:16:31,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +19: [2023-05-11 00:16:31,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +15: [2023-05-11 00:16:31,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +30: [2023-05-11 00:16:31,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +30: [2023-05-11 00:16:31,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +30: [2023-05-11 00:16:31,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +30: [2023-05-11 00:16:31,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +30: [2023-05-11 00:16:31,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +12: [2023-05-11 00:16:31,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +12: [2023-05-11 00:16:31,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +15: [2023-05-11 00:16:31,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +12: [2023-05-11 00:16:31,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +12: [2023-05-11 00:16:31,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +15: [2023-05-11 00:16:31,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +30: [2023-05-11 00:16:31,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +15: [2023-05-11 00:16:31,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +12: [2023-05-11 00:16:31,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +12: [2023-05-11 00:16:31,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 4: [2023-05-11 00:16:31,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 4: [2023-05-11 00:16:31,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 4: [2023-05-11 00:16:31,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 1: [2023-05-11 00:16:31,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +29: [2023-05-11 00:16:31,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +29: [2023-05-11 00:16:31,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +29: [2023-05-11 00:16:31,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +12: [2023-05-11 00:16:31,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +12: [2023-05-11 00:16:31,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +29: [2023-05-11 00:16:31,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +30: [2023-05-11 00:16:31,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +30: [2023-05-11 00:16:31,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +30: [2023-05-11 00:16:31,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 4: [2023-05-11 00:16:31,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +30: [2023-05-11 00:16:31,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +30: [2023-05-11 00:16:31,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +30: [2023-05-11 00:16:31,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +13: [2023-05-11 00:16:31,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +30: [2023-05-11 00:16:31,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +13: [2023-05-11 00:16:31,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +13: [2023-05-11 00:16:31,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +13: [2023-05-11 00:16:31,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 4: [2023-05-11 00:16:31,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +19: [2023-05-11 00:16:31,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +19: [2023-05-11 00:16:31,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +19: [2023-05-11 00:16:31,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +19: [2023-05-11 00:16:31,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +19: [2023-05-11 00:16:31,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +30: [2023-05-11 00:16:31,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +13: [2023-05-11 00:16:31,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +13: [2023-05-11 00:16:31,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +13: [2023-05-11 00:16:31,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +13: [2023-05-11 00:16:31,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 7: [2023-05-11 00:16:31,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 7: [2023-05-11 00:16:31,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 7: [2023-05-11 00:16:31,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +30: [2023-05-11 00:16:31,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +24: [2023-05-11 00:16:31,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +24: [2023-05-11 00:16:31,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +24: [2023-05-11 00:16:31,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +24: [2023-05-11 00:16:31,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +30: [2023-05-11 00:16:31,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 7: [2023-05-11 00:16:31,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +30: [2023-05-11 00:16:31,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 7: [2023-05-11 00:16:31,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +30: [2023-05-11 00:16:31,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +30: [2023-05-11 00:16:31,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +19: [2023-05-11 00:16:31,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +19: [2023-05-11 00:16:31,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +19: [2023-05-11 00:16:31,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +24: [2023-05-11 00:16:31,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +24: [2023-05-11 00:16:31,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +24: [2023-05-11 00:16:31,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +24: [2023-05-11 00:16:31,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +30: [2023-05-11 00:16:31,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +27: [2023-05-11 00:16:31,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +27: [2023-05-11 00:16:31,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +27: [2023-05-11 00:16:31,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +30: [2023-05-11 00:16:31,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +27: [2023-05-11 00:16:31,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +30: [2023-05-11 00:16:31,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +27: [2023-05-11 00:16:31,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +27: [2023-05-11 00:16:31,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +27: [2023-05-11 00:16:31,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +27: [2023-05-11 00:16:31,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +30: [2023-05-11 00:16:31,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +30: [2023-05-11 00:16:31,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 6: [2023-05-11 00:16:31,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +23: [2023-05-11 00:16:31,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +23: [2023-05-11 00:16:31,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +23: [2023-05-11 00:16:31,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +23: [2023-05-11 00:16:31,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +30: [2023-05-11 00:16:31,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +30: [2023-05-11 00:16:31,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +30: [2023-05-11 00:16:31,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +30: [2023-05-11 00:16:31,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 2: [2023-05-11 00:16:31,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +20: [2023-05-11 00:16:31,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +20: [2023-05-11 00:16:31,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +20: [2023-05-11 00:16:31,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +20: [2023-05-11 00:16:31,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 2: [2023-05-11 00:16:31,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +22: [2023-05-11 00:16:31,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +22: [2023-05-11 00:16:31,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +22: [2023-05-11 00:16:31,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +22: [2023-05-11 00:16:31,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +22: [2023-05-11 00:16:31,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +22: [2023-05-11 00:16:31,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +22: [2023-05-11 00:16:31,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +22: [2023-05-11 00:16:31,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +30: [2023-05-11 00:16:31,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +30: [2023-05-11 00:16:31,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 6: [2023-05-11 00:16:31,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +27: [2023-05-11 00:16:31,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +27: [2023-05-11 00:16:31,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +23: [2023-05-11 00:16:31,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +27: [2023-05-11 00:16:31,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +27: [2023-05-11 00:16:31,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +27: [2023-05-11 00:16:31,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +21: [2023-05-11 00:16:31,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +21: [2023-05-11 00:16:31,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +21: [2023-05-11 00:16:31,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +21: [2023-05-11 00:16:31,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +21: [2023-05-11 00:16:31,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +27: [2023-05-11 00:16:31,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +21: [2023-05-11 00:16:31,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +27: [2023-05-11 00:16:31,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +23: [2023-05-11 00:16:31,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +23: [2023-05-11 00:16:31,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +27: [2023-05-11 00:16:31,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +27: [2023-05-11 00:16:31,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +27: [2023-05-11 00:16:31,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +21: [2023-05-11 00:16:31,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +21: [2023-05-11 00:16:31,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +18: [2023-05-11 00:16:31,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +18: [2023-05-11 00:16:31,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +18: [2023-05-11 00:16:31,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +18: [2023-05-11 00:16:31,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +23: [2023-05-11 00:16:31,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +20: [2023-05-11 00:16:31,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +20: [2023-05-11 00:16:31,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +20: [2023-05-11 00:16:31,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +14: [2023-05-11 00:16:31,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +14: [2023-05-11 00:16:31,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +20: [2023-05-11 00:16:31,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +14: [2023-05-11 00:16:31,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +14: [2023-05-11 00:16:31,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +22: [2023-05-11 00:16:31,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +22: [2023-05-11 00:16:31,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +27: [2023-05-11 00:16:31,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +27: [2023-05-11 00:16:31,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 6: [2023-05-11 00:16:31,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +22: [2023-05-11 00:16:31,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +21: [2023-05-11 00:16:31,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +27: [2023-05-11 00:16:31,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +18: [2023-05-11 00:16:31,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +27: [2023-05-11 00:16:31,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +21: [2023-05-11 00:16:31,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +21: [2023-05-11 00:16:31,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +22: [2023-05-11 00:16:31,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +18: [2023-05-11 00:16:31,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +18: [2023-05-11 00:16:31,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +18: [2023-05-11 00:16:31,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +21: [2023-05-11 00:16:31,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +14: [2023-05-11 00:16:31,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +21: [2023-05-11 00:16:31,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +27: [2023-05-11 00:16:31,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +22: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +24: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +24: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +24: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +24: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +14: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +24: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +24: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 5: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 5: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 5: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 5: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +24: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +24: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +24: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +24: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +24: [2023-05-11 00:16:31,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +24: [2023-05-11 00:16:31,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +21: [2023-05-11 00:16:31,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +24: [2023-05-11 00:16:31,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +24: [2023-05-11 00:16:31,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +17: [2023-05-11 00:16:31,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +17: [2023-05-11 00:16:31,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +17: [2023-05-11 00:16:31,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. + 6: [2023-05-11 00:16:31,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +24: [2023-05-11 00:16:31,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +14: [2023-05-11 00:16:31,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +20: [2023-05-11 00:16:31,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +20: [2023-05-11 00:16:31,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +24: [2023-05-11 00:16:31,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +14: [2023-05-11 00:16:31,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +20: [2023-05-11 00:16:31,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +20: [2023-05-11 00:16:31,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +17: [2023-05-11 00:16:31,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt. +29: [2023-05-11 00:16:31,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +29: [2023-05-11 00:16:31,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +29: [2023-05-11 00:16:31,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +27: [2023-05-11 00:16:31,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +29: [2023-05-11 00:16:31,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 6: [2023-05-11 00:16:31,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +22: [2023-05-11 00:16:31,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +21: [2023-05-11 00:16:31,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +22: [2023-05-11 00:16:31,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +25: [2023-05-11 00:16:31,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +25: [2023-05-11 00:16:31,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +25: [2023-05-11 00:16:31,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +25: [2023-05-11 00:16:31,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +27: [2023-05-11 00:16:31,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +29: [2023-05-11 00:16:31,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +29: [2023-05-11 00:16:31,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +29: [2023-05-11 00:16:31,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +29: [2023-05-11 00:16:31,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +29: [2023-05-11 00:16:31,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +29: [2023-05-11 00:16:31,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +29: [2023-05-11 00:16:31,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 8: [2023-05-11 00:16:31,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +27: [2023-05-11 00:16:31,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +29: [2023-05-11 00:16:31,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +20: [2023-05-11 00:16:31,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +20: [2023-05-11 00:16:31,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +20: [2023-05-11 00:16:31,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +25: [2023-05-11 00:16:31,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +25: [2023-05-11 00:16:31,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +25: [2023-05-11 00:16:31,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +25: [2023-05-11 00:16:31,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +29: [2023-05-11 00:16:31,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +29: [2023-05-11 00:16:31,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +29: [2023-05-11 00:16:31,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 1: [2023-05-11 00:16:31,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 1: [2023-05-11 00:16:31,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +20: [2023-05-11 00:16:31,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +20: [2023-05-11 00:16:31,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +20: [2023-05-11 00:16:31,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 9: [2023-05-11 00:16:31,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +20: [2023-05-11 00:16:31,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +20: [2023-05-11 00:16:31,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +22: [2023-05-11 00:16:31,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +29: [2023-05-11 00:16:31,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +27: [2023-05-11 00:16:31,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +12: [2023-05-11 00:16:31,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +12: [2023-05-11 00:16:31,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +12: [2023-05-11 00:16:31,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +12: [2023-05-11 00:16:31,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +27: [2023-05-11 00:16:31,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +21: [2023-05-11 00:16:31,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +21: [2023-05-11 00:16:31,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +12: [2023-05-11 00:16:31,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +12: [2023-05-11 00:16:31,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +17: [2023-05-11 00:16:31,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +21: [2023-05-11 00:16:31,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +21: [2023-05-11 00:16:31,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +27: [2023-05-11 00:16:31,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +12: [2023-05-11 00:16:31,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +17: [2023-05-11 00:16:31,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +17: [2023-05-11 00:16:31,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +17: [2023-05-11 00:16:31,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt... +12: [2023-05-11 00:16:31,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +20: [2023-05-11 00:16:31,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +21: [2023-05-11 00:16:31,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +27: [2023-05-11 00:16:31,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +27: [2023-05-11 00:16:31,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +20: [2023-05-11 00:16:31,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 5: [2023-05-11 00:16:31,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +20: [2023-05-11 00:16:31,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +20: [2023-05-11 00:16:31,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 6: [2023-05-11 00:16:31,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +24: [2023-05-11 00:16:31,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +24: [2023-05-11 00:16:31,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +27: [2023-05-11 00:16:31,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +21: [2023-05-11 00:16:31,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +21: [2023-05-11 00:16:31,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +22: [2023-05-11 00:16:31,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +22: [2023-05-11 00:16:31,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +22: [2023-05-11 00:16:31,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +20: [2023-05-11 00:16:31,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +20: [2023-05-11 00:16:31,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 8: [2023-05-11 00:16:31,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 8: [2023-05-11 00:16:31,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 1: [2023-05-11 00:16:31,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +15: [2023-05-11 00:16:31,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +15: [2023-05-11 00:16:31,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +29: [2023-05-11 00:16:31,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +15: [2023-05-11 00:16:31,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +25: [2023-05-11 00:16:31,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +15: [2023-05-11 00:16:31,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +22: [2023-05-11 00:16:31,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +25: [2023-05-11 00:16:31,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 2: [2023-05-11 00:16:31,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 2: [2023-05-11 00:16:31,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +17: [2023-05-11 00:16:31,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +25: [2023-05-11 00:16:31,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +15: [2023-05-11 00:16:31,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +29: [2023-05-11 00:16:31,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +15: [2023-05-11 00:16:31,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +21: [2023-05-11 00:16:31,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +17: [2023-05-11 00:16:31,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... + 4: [2023-05-11 00:16:31,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +20: [2023-05-11 00:16:31,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +21: [2023-05-11 00:16:31,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +17: [2023-05-11 00:16:31,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +17: [2023-05-11 00:16:31,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +17: [2023-05-11 00:16:31,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +17: [2023-05-11 00:16:31,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +10: [2023-05-11 00:16:31,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +15: [2023-05-11 00:16:31,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +10: [2023-05-11 00:16:31,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +10: [2023-05-11 00:16:31,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +25: [2023-05-11 00:16:31,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +24: [2023-05-11 00:16:31,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +24: [2023-05-11 00:16:31,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +22: [2023-05-11 00:16:31,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +15: [2023-05-11 00:16:31,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +10: [2023-05-11 00:16:31,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +10: [2023-05-11 00:16:31,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +10: [2023-05-11 00:16:31,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +10: [2023-05-11 00:16:31,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +10: [2023-05-11 00:16:31,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +20: [2023-05-11 00:16:31,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +21: [2023-05-11 00:16:31,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +21: [2023-05-11 00:16:31,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +20: [2023-05-11 00:16:31,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +22: [2023-05-11 00:16:31,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +24: [2023-05-11 00:16:31,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +12: [2023-05-11 00:16:31,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +15: [2023-05-11 00:16:31,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +22: [2023-05-11 00:16:31,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +22: [2023-05-11 00:16:31,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +12: [2023-05-11 00:16:31,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +25: [2023-05-11 00:16:31,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +25: [2023-05-11 00:16:31,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 1: [2023-05-11 00:16:31,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 3: [2023-05-11 00:16:31,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 3: [2023-05-11 00:16:31,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 2: [2023-05-11 00:16:31,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 9: [2023-05-11 00:16:31,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +25: [2023-05-11 00:16:31,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +29: [2023-05-11 00:16:31,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +29: [2023-05-11 00:16:31,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +25: [2023-05-11 00:16:31,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +29: [2023-05-11 00:16:31,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +12: [2023-05-11 00:16:31,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +21: [2023-05-11 00:16:31,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +21: [2023-05-11 00:16:31,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +20: [2023-05-11 00:16:31,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +11: [2023-05-11 00:16:31,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +11: [2023-05-11 00:16:31,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +11: [2023-05-11 00:16:31,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +11: [2023-05-11 00:16:31,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 6: [2023-05-11 00:16:31,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 0: [2023-05-11 00:16:31,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 0: [2023-05-11 00:16:31,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 0: [2023-05-11 00:16:31,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +21: [2023-05-11 00:16:31,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 4: [2023-05-11 00:16:31,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +21: [2023-05-11 00:16:31,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 2: [2023-05-11 00:16:31,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +28: [2023-05-11 00:16:31,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +28: [2023-05-11 00:16:31,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +28: [2023-05-11 00:16:31,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +22: [2023-05-11 00:16:31,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +17: [2023-05-11 00:16:31,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt. +19: [2023-05-11 00:16:31,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +19: [2023-05-11 00:16:31,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +19: [2023-05-11 00:16:31,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 6: [2023-05-11 00:16:31,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +22: [2023-05-11 00:16:31,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 9: [2023-05-11 00:16:31,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +28: [2023-05-11 00:16:31,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +26: [2023-05-11 00:16:31,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +26: [2023-05-11 00:16:31,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +17: [2023-05-11 00:16:31,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt... +12: [2023-05-11 00:16:31,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +20: [2023-05-11 00:16:31,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +15: [2023-05-11 00:16:31,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +21: [2023-05-11 00:16:31,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +24: [2023-05-11 00:16:31,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +24: [2023-05-11 00:16:31,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +21: [2023-05-11 00:16:31,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +12: [2023-05-11 00:16:31,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +12: [2023-05-11 00:16:31,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 3: [2023-05-11 00:16:31,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 9: [2023-05-11 00:16:31,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 9: [2023-05-11 00:16:31,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 3: [2023-05-11 00:16:31,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +10: [2023-05-11 00:16:31,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +26: [2023-05-11 00:16:31,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +22: [2023-05-11 00:16:31,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +22: [2023-05-11 00:16:31,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 3: [2023-05-11 00:16:31,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +26: [2023-05-11 00:16:31,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 1: [2023-05-11 00:16:31,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +12: [2023-05-11 00:16:31,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +11: [2023-05-11 00:16:31,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +11: [2023-05-11 00:16:31,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +29: [2023-05-11 00:16:31,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +11: [2023-05-11 00:16:31,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +15: [2023-05-11 00:16:31,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 4: [2023-05-11 00:16:31,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 0: [2023-05-11 00:16:31,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +29: [2023-05-11 00:16:31,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +15: [2023-05-11 00:16:31,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 4: [2023-05-11 00:16:31,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +15: [2023-05-11 00:16:31,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 6: [2023-05-11 00:16:31,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +24: [2023-05-11 00:16:31,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +28: [2023-05-11 00:16:31,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +26: [2023-05-11 00:16:31,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +20: [2023-05-11 00:16:31,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +28: [2023-05-11 00:16:31,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +28: [2023-05-11 00:16:31,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +19: [2023-05-11 00:16:31,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +15: [2023-05-11 00:16:31,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +28: [2023-05-11 00:16:31,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +11: [2023-05-11 00:16:31,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +22: [2023-05-11 00:16:31,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +22: [2023-05-11 00:16:31,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +22: [2023-05-11 00:16:31,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +22: [2023-05-11 00:16:31,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +19: [2023-05-11 00:16:31,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +19: [2023-05-11 00:16:31,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +31: [2023-05-11 00:16:31,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +31: [2023-05-11 00:16:31,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 1: [2023-05-11 00:16:31,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +12: [2023-05-11 00:16:31,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +31: [2023-05-11 00:16:31,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 6: [2023-05-11 00:16:31,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 4: [2023-05-11 00:16:31,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 4: [2023-05-11 00:16:31,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +26: [2023-05-11 00:16:31,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +10: [2023-05-11 00:16:31,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +10: [2023-05-11 00:16:31,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 8: [2023-05-11 00:16:31,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 8: [2023-05-11 00:16:31,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 1: [2023-05-11 00:16:31,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 9: [2023-05-11 00:16:31,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +10: [2023-05-11 00:16:31,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +15: [2023-05-11 00:16:31,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 9: [2023-05-11 00:16:31,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +19: [2023-05-11 00:16:31,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +31: [2023-05-11 00:16:31,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 1: [2023-05-11 00:16:31,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +26: [2023-05-11 00:16:31,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +19: [2023-05-11 00:16:31,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +28: [2023-05-11 00:16:31,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +28: [2023-05-11 00:16:31,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +19: [2023-05-11 00:16:31,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +19: [2023-05-11 00:16:31,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +15: [2023-05-11 00:16:31,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +26: [2023-05-11 00:16:31,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +28: [2023-05-11 00:16:31,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +16: [2023-05-11 00:16:31,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +16: [2023-05-11 00:16:31,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +16: [2023-05-11 00:16:31,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +16: [2023-05-11 00:16:31,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +16: [2023-05-11 00:16:31,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +14: [2023-05-11 00:16:31,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +16: [2023-05-11 00:16:31,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 9: [2023-05-11 00:16:31,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +16: [2023-05-11 00:16:31,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +16: [2023-05-11 00:16:31,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +14: [2023-05-11 00:16:31,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +15: [2023-05-11 00:16:31,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +15: [2023-05-11 00:16:31,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +28: [2023-05-11 00:16:31,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +28: [2023-05-11 00:16:31,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +14: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +16: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +29: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +14: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +13: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +13: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +13: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +13: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +13: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +13: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +16: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +16: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +19: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +19: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +16: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +18: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +18: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +18: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +18: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +18: [2023-05-11 00:16:31,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +18: [2023-05-11 00:16:31,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +18: [2023-05-11 00:16:31,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +28: [2023-05-11 00:16:31,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +15: [2023-05-11 00:16:31,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +13: [2023-05-11 00:16:31,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +18: [2023-05-11 00:16:31,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +13: [2023-05-11 00:16:31,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +13: [2023-05-11 00:16:31,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +13: [2023-05-11 00:16:31,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +13: [2023-05-11 00:16:31,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +13: [2023-05-11 00:16:31,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +13: [2023-05-11 00:16:31,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +11: [2023-05-11 00:16:31,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 7: [2023-05-11 00:16:31,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +11: [2023-05-11 00:16:31,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +11: [2023-05-11 00:16:31,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +11: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +28: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +11: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +18: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +18: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +13: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 0: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +25: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +25: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +28: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +17: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +17: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +17: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +17: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +17: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +17: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +18: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +23: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +23: [2023-05-11 00:16:31,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +23: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +23: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +23: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +10: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +23: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +23: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +23: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +23: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +23: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +23: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +15: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +23: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +11: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +18: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +14: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +13: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +14: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +16: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +16: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +23: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +16: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 0: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +23: [2023-05-11 00:16:31,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 0: [2023-05-11 00:16:31,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 7: [2023-05-11 00:16:31,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +11: [2023-05-11 00:16:31,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +10: [2023-05-11 00:16:31,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +11: [2023-05-11 00:16:31,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +16: [2023-05-11 00:16:31,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +13: [2023-05-11 00:16:31,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 7: [2023-05-11 00:16:31,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +14: [2023-05-11 00:16:31,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +14: [2023-05-11 00:16:31,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +14: [2023-05-11 00:16:31,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +17: [2023-05-11 00:16:31,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +17: [2023-05-11 00:16:31,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +18: [2023-05-11 00:16:31,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +11: [2023-05-11 00:16:31,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +25: [2023-05-11 00:16:31,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +25: [2023-05-11 00:16:31,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 0: [2023-05-11 00:16:31,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +14: [2023-05-11 00:16:31,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +14: [2023-05-11 00:16:31,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +14: [2023-05-11 00:16:31,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +23: [2023-05-11 00:16:31,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 2: [2023-05-11 00:16:31,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +23: [2023-05-11 00:16:31,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +18: [2023-05-11 00:16:31,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +18: [2023-05-11 00:16:31,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +26: [2023-05-11 00:16:31,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +26: [2023-05-11 00:16:31,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +14: [2023-05-11 00:16:31,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +15: [2023-05-11 00:16:31,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +26: [2023-05-11 00:16:31,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +26: [2023-05-11 00:16:31,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +26: [2023-05-11 00:16:31,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +26: [2023-05-11 00:16:31,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +15: [2023-05-11 00:16:31,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +26: [2023-05-11 00:16:31,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +12: [2023-05-11 00:16:31,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +12: [2023-05-11 00:16:31,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +12: [2023-05-11 00:16:31,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +26: [2023-05-11 00:16:31,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +18: [2023-05-11 00:16:31,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +10: [2023-05-11 00:16:31,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +12: [2023-05-11 00:16:31,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 2: [2023-05-11 00:16:31,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +11: [2023-05-11 00:16:31,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +14: [2023-05-11 00:16:31,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 4: [2023-05-11 00:16:31,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +14: [2023-05-11 00:16:31,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +31: [2023-05-11 00:16:31,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +14: [2023-05-11 00:16:31,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +28: [2023-05-11 00:16:31,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +10: [2023-05-11 00:16:31,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +10: [2023-05-11 00:16:31,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +10: [2023-05-11 00:16:31,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +28: [2023-05-11 00:16:31,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 0: [2023-05-11 00:16:31,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +31: [2023-05-11 00:16:31,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +10: [2023-05-11 00:16:31,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +15: [2023-05-11 00:16:31,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +11: [2023-05-11 00:16:31,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +11: [2023-05-11 00:16:31,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +31: [2023-05-11 00:16:31,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +26: [2023-05-11 00:16:31,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +26: [2023-05-11 00:16:31,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 3: [2023-05-11 00:16:31,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +28: [2023-05-11 00:16:31,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 3: [2023-05-11 00:16:31,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +28: [2023-05-11 00:16:31,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +26: [2023-05-11 00:16:31,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 0: [2023-05-11 00:16:31,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +26: [2023-05-11 00:16:31,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +19: [2023-05-11 00:16:31,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +13: [2023-05-11 00:16:31,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +31: [2023-05-11 00:16:31,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +19: [2023-05-11 00:16:31,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +16: [2023-05-11 00:16:31,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +19: [2023-05-11 00:16:31,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +26: [2023-05-11 00:16:31,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +26: [2023-05-11 00:16:31,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +25: [2023-05-11 00:16:31,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +25: [2023-05-11 00:16:31,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +10: [2023-05-11 00:16:31,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +25: [2023-05-11 00:16:31,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +25: [2023-05-11 00:16:31,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +28: [2023-05-11 00:16:31,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +28: [2023-05-11 00:16:31,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +28: [2023-05-11 00:16:31,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +28: [2023-05-11 00:16:31,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +28: [2023-05-11 00:16:31,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +19: [2023-05-11 00:16:31,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +15: [2023-05-11 00:16:31,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +31: [2023-05-11 00:16:31,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +31: [2023-05-11 00:16:31,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +31: [2023-05-11 00:16:31,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +15: [2023-05-11 00:16:31,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +31: [2023-05-11 00:16:31,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +11: [2023-05-11 00:16:31,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +19: [2023-05-11 00:16:31,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +11: [2023-05-11 00:16:31,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +23: [2023-05-11 00:16:31,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +18: [2023-05-11 00:16:31,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +19: [2023-05-11 00:16:31,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +12: [2023-05-11 00:16:31,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +15: [2023-05-11 00:16:31,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +13: [2023-05-11 00:16:31,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +13: [2023-05-11 00:16:31,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +12: [2023-05-11 00:16:31,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +12: [2023-05-11 00:16:31,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +12: [2023-05-11 00:16:31,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +11: [2023-05-11 00:16:31,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +23: [2023-05-11 00:16:31,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +23: [2023-05-11 00:16:31,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +11: [2023-05-11 00:16:31,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +10: [2023-05-11 00:16:31,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +10: [2023-05-11 00:16:31,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +18: [2023-05-11 00:16:31,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +10: [2023-05-11 00:16:31,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +26: [2023-05-11 00:16:31,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +26: [2023-05-11 00:16:31,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +23: [2023-05-11 00:16:31,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +18: [2023-05-11 00:16:31,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +31: [2023-05-11 00:16:31,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +10: [2023-05-11 00:16:31,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +16: [2023-05-11 00:16:31,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +11: [2023-05-11 00:16:31,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +11: [2023-05-11 00:16:31,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +17: [2023-05-11 00:16:31,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +11: [2023-05-11 00:16:31,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +31: [2023-05-11 00:16:31,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +31: [2023-05-11 00:16:31,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +28: [2023-05-11 00:16:31,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +31: [2023-05-11 00:16:31,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 1: [2023-05-11 00:16:31,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +17: [2023-05-11 00:16:31,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +17: [2023-05-11 00:16:31,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +15: [2023-05-11 00:16:31,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +15: [2023-05-11 00:16:31,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +17: [2023-05-11 00:16:31,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +14: [2023-05-11 00:16:31,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +11: [2023-05-11 00:16:31,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +31: [2023-05-11 00:16:31,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +13: [2023-05-11 00:16:31,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +13: [2023-05-11 00:16:31,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +13: [2023-05-11 00:16:31,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +15: [2023-05-11 00:16:31,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +18: [2023-05-11 00:16:31,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +25: [2023-05-11 00:16:31,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +15: [2023-05-11 00:16:31,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +25: [2023-05-11 00:16:31,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +23: [2023-05-11 00:16:31,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +19: [2023-05-11 00:16:31,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +14: [2023-05-11 00:16:31,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +31: [2023-05-11 00:16:31,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +31: [2023-05-11 00:16:31,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +18: [2023-05-11 00:16:31,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +23: [2023-05-11 00:16:31,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +25: [2023-05-11 00:16:31,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +23: [2023-05-11 00:16:31,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +25: [2023-05-11 00:16:31,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 9: [2023-05-11 00:16:31,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 7: [2023-05-11 00:16:31,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +23: [2023-05-11 00:16:31,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +14: [2023-05-11 00:16:31,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +31: [2023-05-11 00:16:31,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +31: [2023-05-11 00:16:31,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 4: [2023-05-11 00:16:31,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +26: [2023-05-11 00:16:31,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +11: [2023-05-11 00:16:31,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +11: [2023-05-11 00:16:31,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +18: [2023-05-11 00:16:31,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +26: [2023-05-11 00:16:31,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +17: [2023-05-11 00:16:31,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +17: [2023-05-11 00:16:31,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +31: [2023-05-11 00:16:31,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +25: [2023-05-11 00:16:31,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +25: [2023-05-11 00:16:31,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 0: [2023-05-11 00:16:31,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +13: [2023-05-11 00:16:31,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +16: [2023-05-11 00:16:31,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +25: [2023-05-11 00:16:31,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +25: [2023-05-11 00:16:31,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +28: [2023-05-11 00:16:31,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +28: [2023-05-11 00:16:31,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +19: [2023-05-11 00:16:31,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +28: [2023-05-11 00:16:31,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +16: [2023-05-11 00:16:31,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +17: [2023-05-11 00:16:31,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +12: [2023-05-11 00:16:31,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +12: [2023-05-11 00:16:31,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +12: [2023-05-11 00:16:31,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +16: [2023-05-11 00:16:31,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +19: [2023-05-11 00:16:31,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +15: [2023-05-11 00:16:31,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +12: [2023-05-11 00:16:31,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +19: [2023-05-11 00:16:31,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +14: [2023-05-11 00:16:31,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +17: [2023-05-11 00:16:31,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +14: [2023-05-11 00:16:31,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +10: [2023-05-11 00:16:31,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +18: [2023-05-11 00:16:31,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +10: [2023-05-11 00:16:31,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 7: [2023-05-11 00:16:31,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +10: [2023-05-11 00:16:31,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +10: [2023-05-11 00:16:31,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +15: [2023-05-11 00:16:31,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 3: [2023-05-11 00:16:31,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +19: [2023-05-11 00:16:31,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +14: [2023-05-11 00:16:31,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +26: [2023-05-11 00:16:31,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +13: [2023-05-11 00:16:31,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +12: [2023-05-11 00:16:31,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +10: [2023-05-11 00:16:31,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +12: [2023-05-11 00:16:31,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +10: [2023-05-11 00:16:31,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +31: [2023-05-11 00:16:31,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +16: [2023-05-11 00:16:31,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +14: [2023-05-11 00:16:31,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +11: [2023-05-11 00:16:31,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +26: [2023-05-11 00:16:31,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +16: [2023-05-11 00:16:31,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +28: [2023-05-11 00:16:31,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +28: [2023-05-11 00:16:31,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +28: [2023-05-11 00:16:31,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +16: [2023-05-11 00:16:31,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +18: [2023-05-11 00:16:31,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +12: [2023-05-11 00:16:31,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +12: [2023-05-11 00:16:31,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +31: [2023-05-11 00:16:31,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +30: [2023-05-11 00:16:31,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +30: [2023-05-11 00:16:31,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +30: [2023-05-11 00:16:31,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +30: [2023-05-11 00:16:31,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +11: [2023-05-11 00:16:31,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +14: [2023-05-11 00:16:31,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +10: [2023-05-11 00:16:31,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +31: [2023-05-11 00:16:31,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +31: [2023-05-11 00:16:31,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +26: [2023-05-11 00:16:31,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +10: [2023-05-11 00:16:31,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +26: [2023-05-11 00:16:31,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +31: [2023-05-11 00:16:31,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +31: [2023-05-11 00:16:31,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +30: [2023-05-11 00:16:31,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +30: [2023-05-11 00:16:31,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +30: [2023-05-11 00:16:31,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +30: [2023-05-11 00:16:31,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +31: [2023-05-11 00:16:31,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +26: [2023-05-11 00:16:31,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +31: [2023-05-11 00:16:31,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +26: [2023-05-11 00:16:31,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +31: [2023-05-11 00:16:31,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +31: [2023-05-11 00:16:31,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +27: [2023-05-11 00:16:31,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +27: [2023-05-11 00:16:31,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +27: [2023-05-11 00:16:31,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +27: [2023-05-11 00:16:31,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 5: [2023-05-11 00:16:31,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 5: [2023-05-11 00:16:31,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 5: [2023-05-11 00:16:31,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 5: [2023-05-11 00:16:31,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +27: [2023-05-11 00:16:31,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +27: [2023-05-11 00:16:31,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +27: [2023-05-11 00:16:31,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +17: [2023-05-11 00:16:31,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +17: [2023-05-11 00:16:31,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +17: [2023-05-11 00:16:31,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. +17: [2023-05-11 00:16:31,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt. + 5: [2023-05-11 00:16:31,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +27: [2023-05-11 00:16:31,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +24: [2023-05-11 00:16:31,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +24: [2023-05-11 00:16:31,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +24: [2023-05-11 00:16:31,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +24: [2023-05-11 00:16:31,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +17: [2023-05-11 00:16:31,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +17: [2023-05-11 00:16:31,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +17: [2023-05-11 00:16:31,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +17: [2023-05-11 00:16:31,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt... +24: [2023-05-11 00:16:31,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +24: [2023-05-11 00:16:31,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +24: [2023-05-11 00:16:31,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 6: [2023-05-11 00:16:31,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 6: [2023-05-11 00:16:31,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +24: [2023-05-11 00:16:31,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +17: [2023-05-11 00:16:31,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +17: [2023-05-11 00:16:31,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +17: [2023-05-11 00:16:31,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +17: [2023-05-11 00:16:31,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +17: [2023-05-11 00:16:31,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +30: [2023-05-11 00:16:31,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +30: [2023-05-11 00:16:31,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +30: [2023-05-11 00:16:31,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +17: [2023-05-11 00:16:31,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... +30: [2023-05-11 00:16:31,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +30: [2023-05-11 00:16:31,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +30: [2023-05-11 00:16:31,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +30: [2023-05-11 00:16:31,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +30: [2023-05-11 00:16:31,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +30: [2023-05-11 00:16:31,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +30: [2023-05-11 00:16:31,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +30: [2023-05-11 00:16:31,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +30: [2023-05-11 00:16:31,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +30: [2023-05-11 00:16:31,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +30: [2023-05-11 00:16:31,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +30: [2023-05-11 00:16:31,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +30: [2023-05-11 00:16:31,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 6: [2023-05-11 00:16:31,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +17: [2023-05-11 00:16:31,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt. +17: [2023-05-11 00:16:31,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt... + 6: [2023-05-11 00:16:31,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +30: [2023-05-11 00:16:31,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +30: [2023-05-11 00:16:31,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +30: [2023-05-11 00:16:31,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +29: [2023-05-11 00:16:31,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +29: [2023-05-11 00:16:31,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +29: [2023-05-11 00:16:31,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +29: [2023-05-11 00:16:31,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +20: [2023-05-11 00:16:31,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +20: [2023-05-11 00:16:31,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 1: [2023-05-11 00:16:31,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +20: [2023-05-11 00:16:31,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +20: [2023-05-11 00:16:31,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 1: [2023-05-11 00:16:31,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +22: [2023-05-11 00:16:31,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +22: [2023-05-11 00:16:31,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +30: [2023-05-11 00:16:31,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +30: [2023-05-11 00:16:31,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +22: [2023-05-11 00:16:31,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +30: [2023-05-11 00:16:31,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +22: [2023-05-11 00:16:31,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +22: [2023-05-11 00:16:31,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +22: [2023-05-11 00:16:31,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +22: [2023-05-11 00:16:31,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +22: [2023-05-11 00:16:31,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +22: [2023-05-11 00:16:31,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +22: [2023-05-11 00:16:31,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +22: [2023-05-11 00:16:31,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +22: [2023-05-11 00:16:31,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +30: [2023-05-11 00:16:31,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +29: [2023-05-11 00:16:31,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +20: [2023-05-11 00:16:31,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +27: [2023-05-11 00:16:31,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +27: [2023-05-11 00:16:31,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +27: [2023-05-11 00:16:31,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +27: [2023-05-11 00:16:31,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +27: [2023-05-11 00:16:31,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +27: [2023-05-11 00:16:31,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +29: [2023-05-11 00:16:31,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +20: [2023-05-11 00:16:31,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +27: [2023-05-11 00:16:31,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +27: [2023-05-11 00:16:31,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +27: [2023-05-11 00:16:31,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +27: [2023-05-11 00:16:31,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +27: [2023-05-11 00:16:31,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +27: [2023-05-11 00:16:31,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +20: [2023-05-11 00:16:31,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +29: [2023-05-11 00:16:31,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +20: [2023-05-11 00:16:31,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +29: [2023-05-11 00:16:31,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +27: [2023-05-11 00:16:31,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +27: [2023-05-11 00:16:31,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +27: [2023-05-11 00:16:31,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 1: [2023-05-11 00:16:31,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +27: [2023-05-11 00:16:31,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +21: [2023-05-11 00:16:31,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +21: [2023-05-11 00:16:31,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +21: [2023-05-11 00:16:31,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +21: [2023-05-11 00:16:31,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +21: [2023-05-11 00:16:31,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +21: [2023-05-11 00:16:31,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +21: [2023-05-11 00:16:31,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +22: [2023-05-11 00:16:31,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +22: [2023-05-11 00:16:31,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +30: [2023-05-11 00:16:31,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +21: [2023-05-11 00:16:31,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +22: [2023-05-11 00:16:31,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +22: [2023-05-11 00:16:31,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +22: [2023-05-11 00:16:31,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +27: [2023-05-11 00:16:31,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +27: [2023-05-11 00:16:31,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +27: [2023-05-11 00:16:31,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +22: [2023-05-11 00:16:31,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +22: [2023-05-11 00:16:31,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 6: [2023-05-11 00:16:31,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +24: [2023-05-11 00:16:31,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +24: [2023-05-11 00:16:31,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +24: [2023-05-11 00:16:31,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +21: [2023-05-11 00:16:31,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +24: [2023-05-11 00:16:31,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +24: [2023-05-11 00:16:31,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +24: [2023-05-11 00:16:31,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +24: [2023-05-11 00:16:31,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 8: [2023-05-11 00:16:31,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +24: [2023-05-11 00:16:31,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +24: [2023-05-11 00:16:31,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +24: [2023-05-11 00:16:31,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +24: [2023-05-11 00:16:31,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 8: [2023-05-11 00:16:31,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +24: [2023-05-11 00:16:31,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +24: [2023-05-11 00:16:31,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 6: [2023-05-11 00:16:31,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +29: [2023-05-11 00:16:31,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +29: [2023-05-11 00:16:31,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +29: [2023-05-11 00:16:31,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +29: [2023-05-11 00:16:31,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +29: [2023-05-11 00:16:31,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +29: [2023-05-11 00:16:31,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +29: [2023-05-11 00:16:31,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +24: [2023-05-11 00:16:31,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +24: [2023-05-11 00:16:31,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +29: [2023-05-11 00:16:31,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 1: [2023-05-11 00:16:31,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +13: [2023-05-11 00:16:31,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 1: [2023-05-11 00:16:31,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 8: [2023-05-11 00:16:31,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +29: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +29: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 6: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 8: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +20: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +20: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +29: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +29: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +29: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +20: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 5: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +29: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +29: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +29: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +13: [2023-05-11 00:16:31,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +20: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +20: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 5: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 5: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +20: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +20: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +14: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +14: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +20: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +20: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +24: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +20: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +21: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +21: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +20: [2023-05-11 00:16:31,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +20: [2023-05-11 00:16:31,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +27: [2023-05-11 00:16:31,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 5: [2023-05-11 00:16:31,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +20: [2023-05-11 00:16:31,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +27: [2023-05-11 00:16:31,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +20: [2023-05-11 00:16:31,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +27: [2023-05-11 00:16:31,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +20: [2023-05-11 00:16:31,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +13: [2023-05-11 00:16:31,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +20: [2023-05-11 00:16:31,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +13: [2023-05-11 00:16:31,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +21: [2023-05-11 00:16:31,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +22: [2023-05-11 00:16:31,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +22: [2023-05-11 00:16:31,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +22: [2023-05-11 00:16:31,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 3: [2023-05-11 00:16:31,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +22: [2023-05-11 00:16:31,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +22: [2023-05-11 00:16:31,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 2: [2023-05-11 00:16:31,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 2: [2023-05-11 00:16:31,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +14: [2023-05-11 00:16:31,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +14: [2023-05-11 00:16:31,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +22: [2023-05-11 00:16:31,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +24: [2023-05-11 00:16:31,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +22: [2023-05-11 00:16:31,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +27: [2023-05-11 00:16:31,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +21: [2023-05-11 00:16:31,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +13: [2023-05-11 00:16:31,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +13: [2023-05-11 00:16:31,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +14: [2023-05-11 00:16:31,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +19: [2023-05-11 00:16:31,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +19: [2023-05-11 00:16:31,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +19: [2023-05-11 00:16:31,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +19: [2023-05-11 00:16:31,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +10: [2023-05-11 00:16:31,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +10: [2023-05-11 00:16:31,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +10: [2023-05-11 00:16:31,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +10: [2023-05-11 00:16:31,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +10: [2023-05-11 00:16:31,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +10: [2023-05-11 00:16:31,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +10: [2023-05-11 00:16:31,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +10: [2023-05-11 00:16:31,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +22: [2023-05-11 00:16:31,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +15: [2023-05-11 00:16:31,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +15: [2023-05-11 00:16:31,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 7: [2023-05-11 00:16:31,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +14: [2023-05-11 00:16:31,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +15: [2023-05-11 00:16:31,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +29: [2023-05-11 00:16:31,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +21: [2023-05-11 00:16:31,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +22: [2023-05-11 00:16:31,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +13: [2023-05-11 00:16:31,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +15: [2023-05-11 00:16:31,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +13: [2023-05-11 00:16:31,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +22: [2023-05-11 00:16:31,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +24: [2023-05-11 00:16:31,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +20: [2023-05-11 00:16:31,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +20: [2023-05-11 00:16:31,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +14: [2023-05-11 00:16:31,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +14: [2023-05-11 00:16:31,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +22: [2023-05-11 00:16:31,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 9: [2023-05-11 00:16:31,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +22: [2023-05-11 00:16:31,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +22: [2023-05-11 00:16:31,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +27: [2023-05-11 00:16:31,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +19: [2023-05-11 00:16:31,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +21: [2023-05-11 00:16:31,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 2: [2023-05-11 00:16:31,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +19: [2023-05-11 00:16:31,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +29: [2023-05-11 00:16:31,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +29: [2023-05-11 00:16:31,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +24: [2023-05-11 00:16:31,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +15: [2023-05-11 00:16:31,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +29: [2023-05-11 00:16:31,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +29: [2023-05-11 00:16:31,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +15: [2023-05-11 00:16:31,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +19: [2023-05-11 00:16:31,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +15: [2023-05-11 00:16:31,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +12: [2023-05-11 00:16:31,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +12: [2023-05-11 00:16:31,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +12: [2023-05-11 00:16:31,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +12: [2023-05-11 00:16:31,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +12: [2023-05-11 00:16:31,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +12: [2023-05-11 00:16:31,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +19: [2023-05-11 00:16:31,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +15: [2023-05-11 00:16:31,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +24: [2023-05-11 00:16:31,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +12: [2023-05-11 00:16:31,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +20: [2023-05-11 00:16:31,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +12: [2023-05-11 00:16:31,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +24: [2023-05-11 00:16:31,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +10: [2023-05-11 00:16:31,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +20: [2023-05-11 00:16:31,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +21: [2023-05-11 00:16:31,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +10: [2023-05-11 00:16:31,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +21: [2023-05-11 00:16:31,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +25: [2023-05-11 00:16:31,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +25: [2023-05-11 00:16:31,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +25: [2023-05-11 00:16:31,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +25: [2023-05-11 00:16:31,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +10: [2023-05-11 00:16:31,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +11: [2023-05-11 00:16:31,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +11: [2023-05-11 00:16:31,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +21: [2023-05-11 00:16:31,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +11: [2023-05-11 00:16:31,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +11: [2023-05-11 00:16:31,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +11: [2023-05-11 00:16:31,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +20: [2023-05-11 00:16:31,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +11: [2023-05-11 00:16:31,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +11: [2023-05-11 00:16:31,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +11: [2023-05-11 00:16:31,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +10: [2023-05-11 00:16:31,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +25: [2023-05-11 00:16:31,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +25: [2023-05-11 00:16:31,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +25: [2023-05-11 00:16:31,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +21: [2023-05-11 00:16:31,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 3: [2023-05-11 00:16:31,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +25: [2023-05-11 00:16:31,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +18: [2023-05-11 00:16:31,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +18: [2023-05-11 00:16:31,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +18: [2023-05-11 00:16:31,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +18: [2023-05-11 00:16:31,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +18: [2023-05-11 00:16:31,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +18: [2023-05-11 00:16:31,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +29: [2023-05-11 00:16:31,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +18: [2023-05-11 00:16:31,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +24: [2023-05-11 00:16:31,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 7: [2023-05-11 00:16:31,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 8: [2023-05-11 00:16:31,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +21: [2023-05-11 00:16:31,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +20: [2023-05-11 00:16:31,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +20: [2023-05-11 00:16:31,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +16: [2023-05-11 00:16:31,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +16: [2023-05-11 00:16:31,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +29: [2023-05-11 00:16:31,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +29: [2023-05-11 00:16:31,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +31: [2023-05-11 00:16:31,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +31: [2023-05-11 00:16:31,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +31: [2023-05-11 00:16:31,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +21: [2023-05-11 00:16:31,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +24: [2023-05-11 00:16:31,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +15: [2023-05-11 00:16:31,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +15: [2023-05-11 00:16:31,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +15: [2023-05-11 00:16:31,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +15: [2023-05-11 00:16:31,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +18: [2023-05-11 00:16:31,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +15: [2023-05-11 00:16:31,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +15: [2023-05-11 00:16:31,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +31: [2023-05-11 00:16:31,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +31: [2023-05-11 00:16:31,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +15: [2023-05-11 00:16:31,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +15: [2023-05-11 00:16:31,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +10: [2023-05-11 00:16:31,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +31: [2023-05-11 00:16:31,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +21: [2023-05-11 00:16:31,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +16: [2023-05-11 00:16:31,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +16: [2023-05-11 00:16:31,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +21: [2023-05-11 00:16:31,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +12: [2023-05-11 00:16:31,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +10: [2023-05-11 00:16:31,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +31: [2023-05-11 00:16:31,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +31: [2023-05-11 00:16:31,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +25: [2023-05-11 00:16:31,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +15: [2023-05-11 00:16:31,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 0: [2023-05-11 00:16:31,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +15: [2023-05-11 00:16:31,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 0: [2023-05-11 00:16:31,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +25: [2023-05-11 00:16:31,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +10: [2023-05-11 00:16:31,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +16: [2023-05-11 00:16:31,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +16: [2023-05-11 00:16:31,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +15: [2023-05-11 00:16:31,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +16: [2023-05-11 00:16:31,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +10: [2023-05-11 00:16:31,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +15: [2023-05-11 00:16:31,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 1: [2023-05-11 00:16:31,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +26: [2023-05-11 00:16:31,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +26: [2023-05-11 00:16:31,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +26: [2023-05-11 00:16:31,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +21: [2023-05-11 00:16:31,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +26: [2023-05-11 00:16:31,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +16: [2023-05-11 00:16:31,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +25: [2023-05-11 00:16:31,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +24: [2023-05-11 00:16:31,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +11: [2023-05-11 00:16:31,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +11: [2023-05-11 00:16:31,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +12: [2023-05-11 00:16:31,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +12: [2023-05-11 00:16:31,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +12: [2023-05-11 00:16:31,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +20: [2023-05-11 00:16:31,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +10: [2023-05-11 00:16:31,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +10: [2023-05-11 00:16:31,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +10: [2023-05-11 00:16:31,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 8: [2023-05-11 00:16:31,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 7: [2023-05-11 00:16:31,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +11: [2023-05-11 00:16:31,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +10: [2023-05-11 00:16:31,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +11: [2023-05-11 00:16:31,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +31: [2023-05-11 00:16:31,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +25: [2023-05-11 00:16:31,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +15: [2023-05-11 00:16:31,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +26: [2023-05-11 00:16:31,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +18: [2023-05-11 00:16:31,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +18: [2023-05-11 00:16:31,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +18: [2023-05-11 00:16:31,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +15: [2023-05-11 00:16:31,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +15: [2023-05-11 00:16:31,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +15: [2023-05-11 00:16:31,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +15: [2023-05-11 00:16:31,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +15: [2023-05-11 00:16:31,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +25: [2023-05-11 00:16:31,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +21: [2023-05-11 00:16:31,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +21: [2023-05-11 00:16:31,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +31: [2023-05-11 00:16:31,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +16: [2023-05-11 00:16:31,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +12: [2023-05-11 00:16:31,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +11: [2023-05-11 00:16:31,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +11: [2023-05-11 00:16:31,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +16: [2023-05-11 00:16:31,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +25: [2023-05-11 00:16:31,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +16: [2023-05-11 00:16:31,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +16: [2023-05-11 00:16:31,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 4: [2023-05-11 00:16:31,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +16: [2023-05-11 00:16:31,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 4: [2023-05-11 00:16:31,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 4: [2023-05-11 00:16:31,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +23: [2023-05-11 00:16:31,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +15: [2023-05-11 00:16:31,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +12: [2023-05-11 00:16:31,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +23: [2023-05-11 00:16:31,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +21: [2023-05-11 00:16:31,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +12: [2023-05-11 00:16:31,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +21: [2023-05-11 00:16:31,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +26: [2023-05-11 00:16:31,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +26: [2023-05-11 00:16:31,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +26: [2023-05-11 00:16:31,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +16: [2023-05-11 00:16:31,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +23: [2023-05-11 00:16:31,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +12: [2023-05-11 00:16:31,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +11: [2023-05-11 00:16:31,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +23: [2023-05-11 00:16:31,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +23: [2023-05-11 00:16:31,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +23: [2023-05-11 00:16:31,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +10: [2023-05-11 00:16:31,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +23: [2023-05-11 00:16:31,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +10: [2023-05-11 00:16:31,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +10: [2023-05-11 00:16:31,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +11: [2023-05-11 00:16:31,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +10: [2023-05-11 00:16:31,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +23: [2023-05-11 00:16:31,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +15: [2023-05-11 00:16:31,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +21: [2023-05-11 00:16:31,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +21: [2023-05-11 00:16:31,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +31: [2023-05-11 00:16:31,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +15: [2023-05-11 00:16:31,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +15: [2023-05-11 00:16:31,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +16: [2023-05-11 00:16:31,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +28: [2023-05-11 00:16:31,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +28: [2023-05-11 00:16:31,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +28: [2023-05-11 00:16:31,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +28: [2023-05-11 00:16:31,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +28: [2023-05-11 00:16:31,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +28: [2023-05-11 00:16:31,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +18: [2023-05-11 00:16:31,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +28: [2023-05-11 00:16:31,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +28: [2023-05-11 00:16:31,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +14: [2023-05-11 00:16:31,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +14: [2023-05-11 00:16:31,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +14: [2023-05-11 00:16:31,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +16: [2023-05-11 00:16:31,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +14: [2023-05-11 00:16:31,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +18: [2023-05-11 00:16:31,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +18: [2023-05-11 00:16:31,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +14: [2023-05-11 00:16:31,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +14: [2023-05-11 00:16:31,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +14: [2023-05-11 00:16:31,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +16: [2023-05-11 00:16:31,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +14: [2023-05-11 00:16:31,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +14: [2023-05-11 00:16:31,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +14: [2023-05-11 00:16:31,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +14: [2023-05-11 00:16:31,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +14: [2023-05-11 00:16:31,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +14: [2023-05-11 00:16:31,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +14: [2023-05-11 00:16:31,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +31: [2023-05-11 00:16:31,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +31: [2023-05-11 00:16:31,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +21: [2023-05-11 00:16:31,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +14: [2023-05-11 00:16:31,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +13: [2023-05-11 00:16:31,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +16: [2023-05-11 00:16:31,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +16: [2023-05-11 00:16:31,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +13: [2023-05-11 00:16:31,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +13: [2023-05-11 00:16:31,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +16: [2023-05-11 00:16:31,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +14: [2023-05-11 00:16:31,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +13: [2023-05-11 00:16:31,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +13: [2023-05-11 00:16:31,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +13: [2023-05-11 00:16:31,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +13: [2023-05-11 00:16:31,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +13: [2023-05-11 00:16:31,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +13: [2023-05-11 00:16:31,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +13: [2023-05-11 00:16:31,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +13: [2023-05-11 00:16:31,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +15: [2023-05-11 00:16:31,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +13: [2023-05-11 00:16:31,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +13: [2023-05-11 00:16:31,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +13: [2023-05-11 00:16:31,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +25: [2023-05-11 00:16:31,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +17: [2023-05-11 00:16:31,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +17: [2023-05-11 00:16:31,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +17: [2023-05-11 00:16:31,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +17: [2023-05-11 00:16:31,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +17: [2023-05-11 00:16:31,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +17: [2023-05-11 00:16:31,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +23: [2023-05-11 00:16:31,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +21: [2023-05-11 00:16:31,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +13: [2023-05-11 00:16:31,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +18: [2023-05-11 00:16:31,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +13: [2023-05-11 00:16:31,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +31: [2023-05-11 00:16:31,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +17: [2023-05-11 00:16:31,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +23: [2023-05-11 00:16:31,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +12: [2023-05-11 00:16:31,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +12: [2023-05-11 00:16:31,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +12: [2023-05-11 00:16:31,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +10: [2023-05-11 00:16:31,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +12: [2023-05-11 00:16:31,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +17: [2023-05-11 00:16:31,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +10: [2023-05-11 00:16:31,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +16: [2023-05-11 00:16:31,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 4: [2023-05-11 00:16:31,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +15: [2023-05-11 00:16:31,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +16: [2023-05-11 00:16:31,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +16: [2023-05-11 00:16:31,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +14: [2023-05-11 00:16:31,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +28: [2023-05-11 00:16:31,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +28: [2023-05-11 00:16:31,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +25: [2023-05-11 00:16:31,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +18: [2023-05-11 00:16:31,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +10: [2023-05-11 00:16:31,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +10: [2023-05-11 00:16:31,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +10: [2023-05-11 00:16:31,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +10: [2023-05-11 00:16:31,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +10: [2023-05-11 00:16:31,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +10: [2023-05-11 00:16:31,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +31: [2023-05-11 00:16:31,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +23: [2023-05-11 00:16:31,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +28: [2023-05-11 00:16:31,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +28: [2023-05-11 00:16:31,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +23: [2023-05-11 00:16:31,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +16: [2023-05-11 00:16:31,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +12: [2023-05-11 00:16:31,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +12: [2023-05-11 00:16:31,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +16: [2023-05-11 00:16:31,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +12: [2023-05-11 00:16:31,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +12: [2023-05-11 00:16:31,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +17: [2023-05-11 00:16:31,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +17: [2023-05-11 00:16:31,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +16: [2023-05-11 00:16:31,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +13: [2023-05-11 00:16:31,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +31: [2023-05-11 00:16:31,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +23: [2023-05-11 00:16:31,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +14: [2023-05-11 00:16:31,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +14: [2023-05-11 00:16:31,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +13: [2023-05-11 00:16:31,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +13: [2023-05-11 00:16:31,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +14: [2023-05-11 00:16:31,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +23: [2023-05-11 00:16:31,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +16: [2023-05-11 00:16:31,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +16: [2023-05-11 00:16:31,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +28: [2023-05-11 00:16:31,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +28: [2023-05-11 00:16:31,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +17: [2023-05-11 00:16:31,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +17: [2023-05-11 00:16:31,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +28: [2023-05-11 00:16:31,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +28: [2023-05-11 00:16:31,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +18: [2023-05-11 00:16:31,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 4: [2023-05-11 00:16:31,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +12: [2023-05-11 00:16:31,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +12: [2023-05-11 00:16:31,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +13: [2023-05-11 00:16:31,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +18: [2023-05-11 00:16:31,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +23: [2023-05-11 00:16:31,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +14: [2023-05-11 00:16:31,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +14: [2023-05-11 00:16:31,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +13: [2023-05-11 00:16:31,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +17: [2023-05-11 00:16:31,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +13: [2023-05-11 00:16:31,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +18: [2023-05-11 00:16:31,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +18: [2023-05-11 00:16:31,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +16: [2023-05-11 00:16:31,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +17: [2023-05-11 00:16:31,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +17: [2023-05-11 00:16:31,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +26: [2023-05-11 00:16:31,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +23: [2023-05-11 00:16:31,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +17: [2023-05-11 00:16:31,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 0: [2023-05-11 00:16:31,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 0: [2023-05-11 00:16:31,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +26: [2023-05-11 00:16:31,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +26: [2023-05-11 00:16:31,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +17: [2023-05-11 00:16:31,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +26: [2023-05-11 00:16:31,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +26: [2023-05-11 00:16:31,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +11: [2023-05-11 00:16:31,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +11: [2023-05-11 00:16:31,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +26: [2023-05-11 00:16:31,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +26: [2023-05-11 00:16:31,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +26: [2023-05-11 00:16:31,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +26: [2023-05-11 00:16:31,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +16: [2023-05-11 00:16:31,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +26: [2023-05-11 00:16:31,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +26: [2023-05-11 00:16:31,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +26: [2023-05-11 00:16:31,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +26: [2023-05-11 00:16:31,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +16: [2023-05-11 00:16:31,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +13: [2023-05-11 00:16:31,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 3: [2023-05-11 00:16:31,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 3: [2023-05-11 00:16:31,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 3: [2023-05-11 00:16:31,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 3: [2023-05-11 00:16:31,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +11: [2023-05-11 00:16:31,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +11: [2023-05-11 00:16:31,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +18: [2023-05-11 00:16:31,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +26: [2023-05-11 00:16:31,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +26: [2023-05-11 00:16:31,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +12: [2023-05-11 00:16:31,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +12: [2023-05-11 00:16:31,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +12: [2023-05-11 00:16:31,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +26: [2023-05-11 00:16:31,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +12: [2023-05-11 00:16:31,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +12: [2023-05-11 00:16:31,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +12: [2023-05-11 00:16:31,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +16: [2023-05-11 00:16:31,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +17: [2023-05-11 00:16:31,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +18: [2023-05-11 00:16:31,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +14: [2023-05-11 00:16:31,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 9: [2023-05-11 00:16:31,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +18: [2023-05-11 00:16:31,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +17: [2023-05-11 00:16:31,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +25: [2023-05-11 00:16:31,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +25: [2023-05-11 00:16:31,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +25: [2023-05-11 00:16:31,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 0: [2023-05-11 00:16:31,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +18: [2023-05-11 00:16:31,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +17: [2023-05-11 00:16:31,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 9: [2023-05-11 00:16:31,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +13: [2023-05-11 00:16:31,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +25: [2023-05-11 00:16:31,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 0: [2023-05-11 00:16:31,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +11: [2023-05-11 00:16:31,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +11: [2023-05-11 00:16:31,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +17: [2023-05-11 00:16:31,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +17: [2023-05-11 00:16:31,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +17: [2023-05-11 00:16:31,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +14: [2023-05-11 00:16:31,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +11: [2023-05-11 00:16:31,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +11: [2023-05-11 00:16:31,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +17: [2023-05-11 00:16:31,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +31: [2023-05-11 00:16:31,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +31: [2023-05-11 00:16:31,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +31: [2023-05-11 00:16:31,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +31: [2023-05-11 00:16:31,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 9: [2023-05-11 00:16:31,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +25: [2023-05-11 00:16:31,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +25: [2023-05-11 00:16:31,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +18: [2023-05-11 00:16:31,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +25: [2023-05-11 00:16:31,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +18: [2023-05-11 00:16:31,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +25: [2023-05-11 00:16:31,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +18: [2023-05-11 00:16:31,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +18: [2023-05-11 00:16:31,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +17: [2023-05-11 00:16:31,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +17: [2023-05-11 00:16:31,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +26: [2023-05-11 00:16:31,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +26: [2023-05-11 00:16:31,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +26: [2023-05-11 00:16:31,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +26: [2023-05-11 00:16:31,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +17: [2023-05-11 00:16:31,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +17: [2023-05-11 00:16:31,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +11: [2023-05-11 00:16:31,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +18: [2023-05-11 00:16:31,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +18: [2023-05-11 00:16:31,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +11: [2023-05-11 00:16:31,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +11: [2023-05-11 00:16:31,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +11: [2023-05-11 00:16:31,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 0: [2023-05-11 00:16:31,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +31: [2023-05-11 00:16:31,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +18: [2023-05-11 00:16:31,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +18: [2023-05-11 00:16:31,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 0: [2023-05-11 00:16:31,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 0: [2023-05-11 00:16:31,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +31: [2023-05-11 00:16:31,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +31: [2023-05-11 00:16:31,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +19: [2023-05-11 00:16:31,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +19: [2023-05-11 00:16:31,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +25: [2023-05-11 00:16:31,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +19: [2023-05-11 00:16:31,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +19: [2023-05-11 00:16:31,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +25: [2023-05-11 00:16:31,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +19: [2023-05-11 00:16:31,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +25: [2023-05-11 00:16:31,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +25: [2023-05-11 00:16:31,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +31: [2023-05-11 00:16:31,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +17: [2023-05-11 00:16:31,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +17: [2023-05-11 00:16:31,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +19: [2023-05-11 00:16:31,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +19: [2023-05-11 00:16:31,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +19: [2023-05-11 00:16:31,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +19: [2023-05-11 00:16:31,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +11: [2023-05-11 00:16:31,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +19: [2023-05-11 00:16:31,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +11: [2023-05-11 00:16:31,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +19: [2023-05-11 00:16:31,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +19: [2023-05-11 00:16:31,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +26: [2023-05-11 00:16:31,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +19: [2023-05-11 00:16:31,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +26: [2023-05-11 00:16:31,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +19: [2023-05-11 00:16:31,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +19: [2023-05-11 00:16:31,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +19: [2023-05-11 00:16:31,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +26: [2023-05-11 00:16:31,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +17: [2023-05-11 00:16:31,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +25: [2023-05-11 00:16:31,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +26: [2023-05-11 00:16:31,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +25: [2023-05-11 00:16:31,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 3: [2023-05-11 00:16:31,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +17: [2023-05-11 00:16:31,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 3: [2023-05-11 00:16:31,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 3: [2023-05-11 00:16:31,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 0: [2023-05-11 00:16:31,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +11: [2023-05-11 00:16:31,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +28: [2023-05-11 00:16:31,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +28: [2023-05-11 00:16:31,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +28: [2023-05-11 00:16:31,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 9: [2023-05-11 00:16:31,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 0: [2023-05-11 00:16:31,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +25: [2023-05-11 00:16:31,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +28: [2023-05-11 00:16:31,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +25: [2023-05-11 00:16:31,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +11: [2023-05-11 00:16:31,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 0: [2023-05-11 00:16:31,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 9: [2023-05-11 00:16:31,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +23: [2023-05-11 00:16:31,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +23: [2023-05-11 00:16:31,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 0: [2023-05-11 00:16:31,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 2: [2023-05-11 00:16:31,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 2: [2023-05-11 00:16:31,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. + 2: [2023-05-11 00:16:31,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +23: [2023-05-11 00:16:31,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +23: [2023-05-11 00:16:31,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt. +31: [2023-05-11 00:16:31,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +31: [2023-05-11 00:16:31,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +28: [2023-05-11 00:16:31,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +31: [2023-05-11 00:16:31,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +31: [2023-05-11 00:16:31,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +28: [2023-05-11 00:16:31,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +28: [2023-05-11 00:16:31,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +28: [2023-05-11 00:16:31,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +23: [2023-05-11 00:16:31,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +31: [2023-05-11 00:16:31,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +23: [2023-05-11 00:16:31,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +31: [2023-05-11 00:16:31,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +23: [2023-05-11 00:16:31,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +31: [2023-05-11 00:16:31,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +31: [2023-05-11 00:16:31,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +23: [2023-05-11 00:16:31,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +19: [2023-05-11 00:16:31,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +30: [2023-05-11 00:16:31,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +30: [2023-05-11 00:16:31,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 2: [2023-05-11 00:16:31,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +19: [2023-05-11 00:16:31,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt... +30: [2023-05-11 00:16:31,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +30: [2023-05-11 00:16:31,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +19: [2023-05-11 00:16:31,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +28: [2023-05-11 00:16:31,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +28: [2023-05-11 00:16:31,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +23: [2023-05-11 00:16:31,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +23: [2023-05-11 00:16:31,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +19: [2023-05-11 00:16:31,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +28: [2023-05-11 00:16:31,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +28: [2023-05-11 00:16:31,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +23: [2023-05-11 00:16:31,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +23: [2023-05-11 00:16:31,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +19: [2023-05-11 00:16:31,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +30: [2023-05-11 00:16:31,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +30: [2023-05-11 00:16:31,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +28: [2023-05-11 00:16:31,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +28: [2023-05-11 00:16:31,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +23: [2023-05-11 00:16:31,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +30: [2023-05-11 00:16:31,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +23: [2023-05-11 00:16:31,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +30: [2023-05-11 00:16:31,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +20: [2023-05-11 00:16:31,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +19: [2023-05-11 00:16:31,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +20: [2023-05-11 00:16:31,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +20: [2023-05-11 00:16:31,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +20: [2023-05-11 00:16:31,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +19: [2023-05-11 00:16:31,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +30: [2023-05-11 00:16:31,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +30: [2023-05-11 00:16:31,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +30: [2023-05-11 00:16:31,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +30: [2023-05-11 00:16:31,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +30: [2023-05-11 00:16:31,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +30: [2023-05-11 00:16:31,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +30: [2023-05-11 00:16:31,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +19: [2023-05-11 00:16:31,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +23: [2023-05-11 00:16:31,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +23: [2023-05-11 00:16:31,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +28: [2023-05-11 00:16:31,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +20: [2023-05-11 00:16:31,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +28: [2023-05-11 00:16:31,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +20: [2023-05-11 00:16:31,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +30: [2023-05-11 00:16:31,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +20: [2023-05-11 00:16:31,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +20: [2023-05-11 00:16:31,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... + 2: [2023-05-11 00:16:31,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +30: [2023-05-11 00:16:31,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +30: [2023-05-11 00:16:31,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +30: [2023-05-11 00:16:31,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +29: [2023-05-11 00:16:31,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 6: [2023-05-11 00:16:31,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 6: [2023-05-11 00:16:31,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 6: [2023-05-11 00:16:31,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 6: [2023-05-11 00:16:31,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 2: [2023-05-11 00:16:31,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt. +29: [2023-05-11 00:16:31,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +30: [2023-05-11 00:16:31,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt... +30: [2023-05-11 00:16:31,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +30: [2023-05-11 00:16:31,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +29: [2023-05-11 00:16:31,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +30: [2023-05-11 00:16:31,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +30: [2023-05-11 00:16:31,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +30: [2023-05-11 00:16:31,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +30: [2023-05-11 00:16:31,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +29: [2023-05-11 00:16:31,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +30: [2023-05-11 00:16:31,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +24: [2023-05-11 00:16:31,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +24: [2023-05-11 00:16:31,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +24: [2023-05-11 00:16:31,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +24: [2023-05-11 00:16:31,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +29: [2023-05-11 00:16:31,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +30: [2023-05-11 00:16:31,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +24: [2023-05-11 00:16:31,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +24: [2023-05-11 00:16:31,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +24: [2023-05-11 00:16:31,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +29: [2023-05-11 00:16:31,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +29: [2023-05-11 00:16:31,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +24: [2023-05-11 00:16:31,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +29: [2023-05-11 00:16:31,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +30: [2023-05-11 00:16:31,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +30: [2023-05-11 00:16:31,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +24: [2023-05-11 00:16:31,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +24: [2023-05-11 00:16:31,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +27: [2023-05-11 00:16:31,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +27: [2023-05-11 00:16:31,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +27: [2023-05-11 00:16:31,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +27: [2023-05-11 00:16:31,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +30: [2023-05-11 00:16:31,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 5: [2023-05-11 00:16:31,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 5: [2023-05-11 00:16:31,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 5: [2023-05-11 00:16:31,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +15: [2023-05-11 00:16:31,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +15: [2023-05-11 00:16:31,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +15: [2023-05-11 00:16:31,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +15: [2023-05-11 00:16:31,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +27: [2023-05-11 00:16:31,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +27: [2023-05-11 00:16:31,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +24: [2023-05-11 00:16:31,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +24: [2023-05-11 00:16:31,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +30: [2023-05-11 00:16:31,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +24: [2023-05-11 00:16:31,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +27: [2023-05-11 00:16:31,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +27: [2023-05-11 00:16:31,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +24: [2023-05-11 00:16:31,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +15: [2023-05-11 00:16:31,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +15: [2023-05-11 00:16:31,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +15: [2023-05-11 00:16:31,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +15: [2023-05-11 00:16:31,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +21: [2023-05-11 00:16:31,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +21: [2023-05-11 00:16:31,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +21: [2023-05-11 00:16:31,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +21: [2023-05-11 00:16:31,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +21: [2023-05-11 00:16:31,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +21: [2023-05-11 00:16:31,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +21: [2023-05-11 00:16:31,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +24: [2023-05-11 00:16:31,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +21: [2023-05-11 00:16:31,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +24: [2023-05-11 00:16:31,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +24: [2023-05-11 00:16:31,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +24: [2023-05-11 00:16:31,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +24: [2023-05-11 00:16:31,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +24: [2023-05-11 00:16:31,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +29: [2023-05-11 00:16:31,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +29: [2023-05-11 00:16:31,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +29: [2023-05-11 00:16:31,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +29: [2023-05-11 00:16:31,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +29: [2023-05-11 00:16:31,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +29: [2023-05-11 00:16:31,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +29: [2023-05-11 00:16:31,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +29: [2023-05-11 00:16:31,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +29: [2023-05-11 00:16:31,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +29: [2023-05-11 00:16:31,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +29: [2023-05-11 00:16:31,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +29: [2023-05-11 00:16:31,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +29: [2023-05-11 00:16:31,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +29: [2023-05-11 00:16:31,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +29: [2023-05-11 00:16:31,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +21: [2023-05-11 00:16:31,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +10: [2023-05-11 00:16:31,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +10: [2023-05-11 00:16:31,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +10: [2023-05-11 00:16:31,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +29: [2023-05-11 00:16:31,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +10: [2023-05-11 00:16:31,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +22: [2023-05-11 00:16:31,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +22: [2023-05-11 00:16:31,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +22: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +22: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +22: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +22: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +22: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +22: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 8: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 5: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 5: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 5: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 5: [2023-05-11 00:16:31,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 8: [2023-05-11 00:16:31,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +20: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +20: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +20: [2023-05-11 00:16:31,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +20: [2023-05-11 00:16:31,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +20: [2023-05-11 00:16:31,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +20: [2023-05-11 00:16:31,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +20: [2023-05-11 00:16:31,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +20: [2023-05-11 00:16:31,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +20: [2023-05-11 00:16:31,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +20: [2023-05-11 00:16:31,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +20: [2023-05-11 00:16:31,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +20: [2023-05-11 00:16:31,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +20: [2023-05-11 00:16:31,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +20: [2023-05-11 00:16:31,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 7: [2023-05-11 00:16:31,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +20: [2023-05-11 00:16:31,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 1: [2023-05-11 00:16:31,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +20: [2023-05-11 00:16:31,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 6: [2023-05-11 00:16:31,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +12: [2023-05-11 00:16:31,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +12: [2023-05-11 00:16:31,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 6: [2023-05-11 00:16:31,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 7: [2023-05-11 00:16:31,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +12: [2023-05-11 00:16:31,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 6: [2023-05-11 00:16:31,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +21: [2023-05-11 00:16:31,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +21: [2023-05-11 00:16:31,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +12: [2023-05-11 00:16:31,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +12: [2023-05-11 00:16:31,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +12: [2023-05-11 00:16:31,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +12: [2023-05-11 00:16:31,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +24: [2023-05-11 00:16:31,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +12: [2023-05-11 00:16:31,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +12: [2023-05-11 00:16:31,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +12: [2023-05-11 00:16:31,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +24: [2023-05-11 00:16:31,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +24: [2023-05-11 00:16:31,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +21: [2023-05-11 00:16:31,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +24: [2023-05-11 00:16:31,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +12: [2023-05-11 00:16:31,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +10: [2023-05-11 00:16:31,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +10: [2023-05-11 00:16:31,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +29: [2023-05-11 00:16:31,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +12: [2023-05-11 00:16:31,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +10: [2023-05-11 00:16:31,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +10: [2023-05-11 00:16:31,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +22: [2023-05-11 00:16:31,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +22: [2023-05-11 00:16:31,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +12: [2023-05-11 00:16:31,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +12: [2023-05-11 00:16:31,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +21: [2023-05-11 00:16:31,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +21: [2023-05-11 00:16:31,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +12: [2023-05-11 00:16:31,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +22: [2023-05-11 00:16:31,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +22: [2023-05-11 00:16:31,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +12: [2023-05-11 00:16:31,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 2: [2023-05-11 00:16:31,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +20: [2023-05-11 00:16:31,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +20: [2023-05-11 00:16:31,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +20: [2023-05-11 00:16:31,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 5: [2023-05-11 00:16:31,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +24: [2023-05-11 00:16:31,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +24: [2023-05-11 00:16:31,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +29: [2023-05-11 00:16:31,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +29: [2023-05-11 00:16:31,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +24: [2023-05-11 00:16:31,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +24: [2023-05-11 00:16:31,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +29: [2023-05-11 00:16:31,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +24: [2023-05-11 00:16:31,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +24: [2023-05-11 00:16:31,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +29: [2023-05-11 00:16:31,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +22: [2023-05-11 00:16:31,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +28: [2023-05-11 00:16:31,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +28: [2023-05-11 00:16:31,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +22: [2023-05-11 00:16:31,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +28: [2023-05-11 00:16:31,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +28: [2023-05-11 00:16:31,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +25: [2023-05-11 00:16:31,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +25: [2023-05-11 00:16:31,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +21: [2023-05-11 00:16:31,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +21: [2023-05-11 00:16:31,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +21: [2023-05-11 00:16:31,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 4: [2023-05-11 00:16:31,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +25: [2023-05-11 00:16:31,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +28: [2023-05-11 00:16:31,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +25: [2023-05-11 00:16:31,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +28: [2023-05-11 00:16:31,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +28: [2023-05-11 00:16:31,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +21: [2023-05-11 00:16:31,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 4: [2023-05-11 00:16:31,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +15: [2023-05-11 00:16:31,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +15: [2023-05-11 00:16:31,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +15: [2023-05-11 00:16:31,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +15: [2023-05-11 00:16:31,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +15: [2023-05-11 00:16:31,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +15: [2023-05-11 00:16:31,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +15: [2023-05-11 00:16:31,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +15: [2023-05-11 00:16:31,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +15: [2023-05-11 00:16:31,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +15: [2023-05-11 00:16:31,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +25: [2023-05-11 00:16:31,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +15: [2023-05-11 00:16:31,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +25: [2023-05-11 00:16:31,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +15: [2023-05-11 00:16:31,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +25: [2023-05-11 00:16:31,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +28: [2023-05-11 00:16:31,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +22: [2023-05-11 00:16:31,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +25: [2023-05-11 00:16:31,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +25: [2023-05-11 00:16:31,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 4: [2023-05-11 00:16:31,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +25: [2023-05-11 00:16:31,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 7: [2023-05-11 00:16:31,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +25: [2023-05-11 00:16:31,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +15: [2023-05-11 00:16:31,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +24: [2023-05-11 00:16:31,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +24: [2023-05-11 00:16:31,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +15: [2023-05-11 00:16:31,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +20: [2023-05-11 00:16:31,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +20: [2023-05-11 00:16:31,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +10: [2023-05-11 00:16:31,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +10: [2023-05-11 00:16:31,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +10: [2023-05-11 00:16:31,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +25: [2023-05-11 00:16:31,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +10: [2023-05-11 00:16:31,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +15: [2023-05-11 00:16:31,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +10: [2023-05-11 00:16:31,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +15: [2023-05-11 00:16:31,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +22: [2023-05-11 00:16:31,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +10: [2023-05-11 00:16:31,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +10: [2023-05-11 00:16:31,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +10: [2023-05-11 00:16:31,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +10: [2023-05-11 00:16:31,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +13: [2023-05-11 00:16:31,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +13: [2023-05-11 00:16:31,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +13: [2023-05-11 00:16:31,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 5: [2023-05-11 00:16:31,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 5: [2023-05-11 00:16:31,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +10: [2023-05-11 00:16:31,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +27: [2023-05-11 00:16:31,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +27: [2023-05-11 00:16:31,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +27: [2023-05-11 00:16:31,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +27: [2023-05-11 00:16:31,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +27: [2023-05-11 00:16:31,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +19: [2023-05-11 00:16:31,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +19: [2023-05-11 00:16:31,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +19: [2023-05-11 00:16:31,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +27: [2023-05-11 00:16:31,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +27: [2023-05-11 00:16:31,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +27: [2023-05-11 00:16:31,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +13: [2023-05-11 00:16:31,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +21: [2023-05-11 00:16:31,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +27: [2023-05-11 00:16:31,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +27: [2023-05-11 00:16:31,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +27: [2023-05-11 00:16:31,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +27: [2023-05-11 00:16:31,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +27: [2023-05-11 00:16:31,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 8: [2023-05-11 00:16:31,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +29: [2023-05-11 00:16:31,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +29: [2023-05-11 00:16:31,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +19: [2023-05-11 00:16:31,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +20: [2023-05-11 00:16:31,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +29: [2023-05-11 00:16:31,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 2: [2023-05-11 00:16:31,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +27: [2023-05-11 00:16:31,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +27: [2023-05-11 00:16:31,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +27: [2023-05-11 00:16:31,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 7: [2023-05-11 00:16:31,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +12: [2023-05-11 00:16:31,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +12: [2023-05-11 00:16:31,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +12: [2023-05-11 00:16:31,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +10: [2023-05-11 00:16:31,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +10: [2023-05-11 00:16:31,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 1: [2023-05-11 00:16:31,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 1: [2023-05-11 00:16:31,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +25: [2023-05-11 00:16:31,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 6: [2023-05-11 00:16:31,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 6: [2023-05-11 00:16:31,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +21: [2023-05-11 00:16:31,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +20: [2023-05-11 00:16:31,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +16: [2023-05-11 00:16:31,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +21: [2023-05-11 00:16:31,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +21: [2023-05-11 00:16:31,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +16: [2023-05-11 00:16:31,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +12: [2023-05-11 00:16:31,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +16: [2023-05-11 00:16:31,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +16: [2023-05-11 00:16:31,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +16: [2023-05-11 00:16:31,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +16: [2023-05-11 00:16:31,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +16: [2023-05-11 00:16:31,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +12: [2023-05-11 00:16:31,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +25: [2023-05-11 00:16:31,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +25: [2023-05-11 00:16:31,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +12: [2023-05-11 00:16:31,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +12: [2023-05-11 00:16:31,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +21: [2023-05-11 00:16:31,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +16: [2023-05-11 00:16:31,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 8: [2023-05-11 00:16:31,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +28: [2023-05-11 00:16:31,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +13: [2023-05-11 00:16:31,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +28: [2023-05-11 00:16:31,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +15: [2023-05-11 00:16:31,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +19: [2023-05-11 00:16:31,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +19: [2023-05-11 00:16:31,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +13: [2023-05-11 00:16:31,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +19: [2023-05-11 00:16:31,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +13: [2023-05-11 00:16:31,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +19: [2023-05-11 00:16:31,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +25: [2023-05-11 00:16:31,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +21: [2023-05-11 00:16:31,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +25: [2023-05-11 00:16:31,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +25: [2023-05-11 00:16:31,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +13: [2023-05-11 00:16:31,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +27: [2023-05-11 00:16:31,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +27: [2023-05-11 00:16:31,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +10: [2023-05-11 00:16:31,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 6: [2023-05-11 00:16:31,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +10: [2023-05-11 00:16:31,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +12: [2023-05-11 00:16:31,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +27: [2023-05-11 00:16:31,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +12: [2023-05-11 00:16:31,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +12: [2023-05-11 00:16:31,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +10: [2023-05-11 00:16:31,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +25: [2023-05-11 00:16:31,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +20: [2023-05-11 00:16:31,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +12: [2023-05-11 00:16:31,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +12: [2023-05-11 00:16:31,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +12: [2023-05-11 00:16:31,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +16: [2023-05-11 00:16:31,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +10: [2023-05-11 00:16:31,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +28: [2023-05-11 00:16:31,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +15: [2023-05-11 00:16:31,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +12: [2023-05-11 00:16:31,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +28: [2023-05-11 00:16:31,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +10: [2023-05-11 00:16:31,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +28: [2023-05-11 00:16:31,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +12: [2023-05-11 00:16:31,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 8: [2023-05-11 00:16:31,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 4: [2023-05-11 00:16:31,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +21: [2023-05-11 00:16:31,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +10: [2023-05-11 00:16:31,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +15: [2023-05-11 00:16:31,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +21: [2023-05-11 00:16:31,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +25: [2023-05-11 00:16:31,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +25: [2023-05-11 00:16:31,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +16: [2023-05-11 00:16:31,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +25: [2023-05-11 00:16:31,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 8: [2023-05-11 00:16:31,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +25: [2023-05-11 00:16:31,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +16: [2023-05-11 00:16:31,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +21: [2023-05-11 00:16:31,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +21: [2023-05-11 00:16:31,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 6: [2023-05-11 00:16:31,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +28: [2023-05-11 00:16:31,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +10: [2023-05-11 00:16:31,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +10: [2023-05-11 00:16:31,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +15: [2023-05-11 00:16:31,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +27: [2023-05-11 00:16:31,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +12: [2023-05-11 00:16:31,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +27: [2023-05-11 00:16:31,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +27: [2023-05-11 00:16:31,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +28: [2023-05-11 00:16:31,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 9: [2023-05-11 00:16:31,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +15: [2023-05-11 00:16:31,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 9: [2023-05-11 00:16:31,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 9: [2023-05-11 00:16:31,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 9: [2023-05-11 00:16:31,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +21: [2023-05-11 00:16:31,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +21: [2023-05-11 00:16:31,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +10: [2023-05-11 00:16:31,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +16: [2023-05-11 00:16:31,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +15: [2023-05-11 00:16:31,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 8: [2023-05-11 00:16:31,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +10: [2023-05-11 00:16:31,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +27: [2023-05-11 00:16:31,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +25: [2023-05-11 00:16:31,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +16: [2023-05-11 00:16:31,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +16: [2023-05-11 00:16:31,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +28: [2023-05-11 00:16:31,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 1: [2023-05-11 00:16:31,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +25: [2023-05-11 00:16:31,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +15: [2023-05-11 00:16:31,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +10: [2023-05-11 00:16:31,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +25: [2023-05-11 00:16:31,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +25: [2023-05-11 00:16:31,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +25: [2023-05-11 00:16:31,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +25: [2023-05-11 00:16:31,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 9: [2023-05-11 00:16:31,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +10: [2023-05-11 00:16:31,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +21: [2023-05-11 00:16:31,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +15: [2023-05-11 00:16:31,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +21: [2023-05-11 00:16:31,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +23: [2023-05-11 00:16:31,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +23: [2023-05-11 00:16:31,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +23: [2023-05-11 00:16:31,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +23: [2023-05-11 00:16:31,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +23: [2023-05-11 00:16:31,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +23: [2023-05-11 00:16:31,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +27: [2023-05-11 00:16:31,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +23: [2023-05-11 00:16:31,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +23: [2023-05-11 00:16:31,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 1: [2023-05-11 00:16:31,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 9: [2023-05-11 00:16:31,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +16: [2023-05-11 00:16:31,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +11: [2023-05-11 00:16:31,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +11: [2023-05-11 00:16:31,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +25: [2023-05-11 00:16:31,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +11: [2023-05-11 00:16:31,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +11: [2023-05-11 00:16:31,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +11: [2023-05-11 00:16:31,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +11: [2023-05-11 00:16:31,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +11: [2023-05-11 00:16:31,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +11: [2023-05-11 00:16:31,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +31: [2023-05-11 00:16:31,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +31: [2023-05-11 00:16:31,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +31: [2023-05-11 00:16:31,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +31: [2023-05-11 00:16:31,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +31: [2023-05-11 00:16:31,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +31: [2023-05-11 00:16:31,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +25: [2023-05-11 00:16:31,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +31: [2023-05-11 00:16:31,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +26: [2023-05-11 00:16:31,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +26: [2023-05-11 00:16:31,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +26: [2023-05-11 00:16:31,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +26: [2023-05-11 00:16:31,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +26: [2023-05-11 00:16:31,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +31: [2023-05-11 00:16:31,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +26: [2023-05-11 00:16:31,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +26: [2023-05-11 00:16:31,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +17: [2023-05-11 00:16:31,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +17: [2023-05-11 00:16:31,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +17: [2023-05-11 00:16:31,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +17: [2023-05-11 00:16:31,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +17: [2023-05-11 00:16:31,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +17: [2023-05-11 00:16:31,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +17: [2023-05-11 00:16:31,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +18: [2023-05-11 00:16:31,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +17: [2023-05-11 00:16:31,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +18: [2023-05-11 00:16:31,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +25: [2023-05-11 00:16:31,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 0: [2023-05-11 00:16:31,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +18: [2023-05-11 00:16:31,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 0: [2023-05-11 00:16:31,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +18: [2023-05-11 00:16:31,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +18: [2023-05-11 00:16:31,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +18: [2023-05-11 00:16:31,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +26: [2023-05-11 00:16:31,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +13: [2023-05-11 00:16:31,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +13: [2023-05-11 00:16:31,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +13: [2023-05-11 00:16:31,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +13: [2023-05-11 00:16:31,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +13: [2023-05-11 00:16:31,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +14: [2023-05-11 00:16:31,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +14: [2023-05-11 00:16:31,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +13: [2023-05-11 00:16:31,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +13: [2023-05-11 00:16:31,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +13: [2023-05-11 00:16:31,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +13: [2023-05-11 00:16:31,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +14: [2023-05-11 00:16:31,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +14: [2023-05-11 00:16:31,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +13: [2023-05-11 00:16:31,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +13: [2023-05-11 00:16:31,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +13: [2023-05-11 00:16:31,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +13: [2023-05-11 00:16:31,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +13: [2023-05-11 00:16:31,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +18: [2023-05-11 00:16:31,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +14: [2023-05-11 00:16:31,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +14: [2023-05-11 00:16:31,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +14: [2023-05-11 00:16:31,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +14: [2023-05-11 00:16:31,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +18: [2023-05-11 00:16:31,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +13: [2023-05-11 00:16:31,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +13: [2023-05-11 00:16:31,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +16: [2023-05-11 00:16:31,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +23: [2023-05-11 00:16:31,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +23: [2023-05-11 00:16:31,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +23: [2023-05-11 00:16:31,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +23: [2023-05-11 00:16:31,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +11: [2023-05-11 00:16:31,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +28: [2023-05-11 00:16:31,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +28: [2023-05-11 00:16:31,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +28: [2023-05-11 00:16:31,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +28: [2023-05-11 00:16:31,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 3: [2023-05-11 00:16:31,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +31: [2023-05-11 00:16:31,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +18: [2023-05-11 00:16:31,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +18: [2023-05-11 00:16:31,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +31: [2023-05-11 00:16:31,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +31: [2023-05-11 00:16:31,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +17: [2023-05-11 00:16:31,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +17: [2023-05-11 00:16:31,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +11: [2023-05-11 00:16:31,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +11: [2023-05-11 00:16:31,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +14: [2023-05-11 00:16:31,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +11: [2023-05-11 00:16:31,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +13: [2023-05-11 00:16:31,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +17: [2023-05-11 00:16:31,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +31: [2023-05-11 00:16:31,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +17: [2023-05-11 00:16:31,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +23: [2023-05-11 00:16:31,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +23: [2023-05-11 00:16:31,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +11: [2023-05-11 00:16:31,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +13: [2023-05-11 00:16:31,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +23: [2023-05-11 00:16:31,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +23: [2023-05-11 00:16:31,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +26: [2023-05-11 00:16:31,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +26: [2023-05-11 00:16:31,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +26: [2023-05-11 00:16:31,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 9: [2023-05-11 00:16:31,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +13: [2023-05-11 00:16:31,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +13: [2023-05-11 00:16:31,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +28: [2023-05-11 00:16:31,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +14: [2023-05-11 00:16:31,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +14: [2023-05-11 00:16:31,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +28: [2023-05-11 00:16:31,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +28: [2023-05-11 00:16:31,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +28: [2023-05-11 00:16:31,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 3: [2023-05-11 00:16:31,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 3: [2023-05-11 00:16:31,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +17: [2023-05-11 00:16:31,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +17: [2023-05-11 00:16:31,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +19: [2023-05-11 00:16:31,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +19: [2023-05-11 00:16:31,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +19: [2023-05-11 00:16:31,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +19: [2023-05-11 00:16:31,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +31: [2023-05-11 00:16:31,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +19: [2023-05-11 00:16:31,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +19: [2023-05-11 00:16:31,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 9: [2023-05-11 00:16:31,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +14: [2023-05-11 00:16:31,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +19: [2023-05-11 00:16:31,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +14: [2023-05-11 00:16:31,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +19: [2023-05-11 00:16:31,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +31: [2023-05-11 00:16:31,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +19: [2023-05-11 00:16:31,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +19: [2023-05-11 00:16:31,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +19: [2023-05-11 00:16:31,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +19: [2023-05-11 00:16:31,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +19: [2023-05-11 00:16:31,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +18: [2023-05-11 00:16:31,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +19: [2023-05-11 00:16:31,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +19: [2023-05-11 00:16:31,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +18: [2023-05-11 00:16:31,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +17: [2023-05-11 00:16:31,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +19: [2023-05-11 00:16:31,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 3: [2023-05-11 00:16:31,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +11: [2023-05-11 00:16:31,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +31: [2023-05-11 00:16:31,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +13: [2023-05-11 00:16:31,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +11: [2023-05-11 00:16:31,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +30: [2023-05-11 00:16:31,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +30: [2023-05-11 00:16:31,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +30: [2023-05-11 00:16:31,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +11: [2023-05-11 00:16:31,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +30: [2023-05-11 00:16:31,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +18: [2023-05-11 00:16:31,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +13: [2023-05-11 00:16:31,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +13: [2023-05-11 00:16:31,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +17: [2023-05-11 00:16:31,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +13: [2023-05-11 00:16:31,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +31: [2023-05-11 00:16:31,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +26: [2023-05-11 00:16:31,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +26: [2023-05-11 00:16:31,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +26: [2023-05-11 00:16:31,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 0: [2023-05-11 00:16:31,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +28: [2023-05-11 00:16:31,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +14: [2023-05-11 00:16:31,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +14: [2023-05-11 00:16:31,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +14: [2023-05-11 00:16:31,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +18: [2023-05-11 00:16:31,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +28: [2023-05-11 00:16:31,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +19: [2023-05-11 00:16:31,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +19: [2023-05-11 00:16:31,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +28: [2023-05-11 00:16:31,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +28: [2023-05-11 00:16:31,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +18: [2023-05-11 00:16:31,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +30: [2023-05-11 00:16:31,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +30: [2023-05-11 00:16:31,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +30: [2023-05-11 00:16:31,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +30: [2023-05-11 00:16:31,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +30: [2023-05-11 00:16:31,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +28: [2023-05-11 00:16:31,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +30: [2023-05-11 00:16:31,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +30: [2023-05-11 00:16:31,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +19: [2023-05-11 00:16:31,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 7: [2023-05-11 00:16:31,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 0: [2023-05-11 00:16:31,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +28: [2023-05-11 00:16:31,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +19: [2023-05-11 00:16:31,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 7: [2023-05-11 00:16:31,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +30: [2023-05-11 00:16:31,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +30: [2023-05-11 00:16:31,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +26: [2023-05-11 00:16:31,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +30: [2023-05-11 00:16:31,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +30: [2023-05-11 00:16:31,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +30: [2023-05-11 00:16:31,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +28: [2023-05-11 00:16:31,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +28: [2023-05-11 00:16:31,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +18: [2023-05-11 00:16:31,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +19: [2023-05-11 00:16:31,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 4: [2023-05-11 00:16:31,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 4: [2023-05-11 00:16:31,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 4: [2023-05-11 00:16:31,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 0: [2023-05-11 00:16:31,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +11: [2023-05-11 00:16:31,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +11: [2023-05-11 00:16:31,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +11: [2023-05-11 00:16:31,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 7: [2023-05-11 00:16:31,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +19: [2023-05-11 00:16:31,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +11: [2023-05-11 00:16:31,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +19: [2023-05-11 00:16:31,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +26: [2023-05-11 00:16:31,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 7: [2023-05-11 00:16:31,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +16: [2023-05-11 00:16:31,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +16: [2023-05-11 00:16:31,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 7: [2023-05-11 00:16:31,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +19: [2023-05-11 00:16:31,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +16: [2023-05-11 00:16:31,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +16: [2023-05-11 00:16:31,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +30: [2023-05-11 00:16:31,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 4: [2023-05-11 00:16:31,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +11: [2023-05-11 00:16:31,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 4: [2023-05-11 00:16:31,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +20: [2023-05-11 00:16:31,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +20: [2023-05-11 00:16:31,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +20: [2023-05-11 00:16:31,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +16: [2023-05-11 00:16:31,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +30: [2023-05-11 00:16:31,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 7: [2023-05-11 00:16:31,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +11: [2023-05-11 00:16:32,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +30: [2023-05-11 00:16:32,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 7: [2023-05-11 00:16:32,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 5: [2023-05-11 00:16:32,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 5: [2023-05-11 00:16:32,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +11: [2023-05-11 00:16:32,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 5: [2023-05-11 00:16:32,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 4: [2023-05-11 00:16:32,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +20: [2023-05-11 00:16:32,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 6: [2023-05-11 00:16:32,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 6: [2023-05-11 00:16:32,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 6: [2023-05-11 00:16:32,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +16: [2023-05-11 00:16:32,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +30: [2023-05-11 00:16:32,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +11: [2023-05-11 00:16:32,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +16: [2023-05-11 00:16:32,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +30: [2023-05-11 00:16:32,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +16: [2023-05-11 00:16:32,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +30: [2023-05-11 00:16:32,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +30: [2023-05-11 00:16:32,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +30: [2023-05-11 00:16:32,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 7: [2023-05-11 00:16:32,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +30: [2023-05-11 00:16:32,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +20: [2023-05-11 00:16:32,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +20: [2023-05-11 00:16:32,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +24: [2023-05-11 00:16:32,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +24: [2023-05-11 00:16:32,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +24: [2023-05-11 00:16:32,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +24: [2023-05-11 00:16:32,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +17: [2023-05-11 00:16:32,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +17: [2023-05-11 00:16:32,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +17: [2023-05-11 00:16:32,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +20: [2023-05-11 00:16:32,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +17: [2023-05-11 00:16:32,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 5: [2023-05-11 00:16:32,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +24: [2023-05-11 00:16:32,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +24: [2023-05-11 00:16:32,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +24: [2023-05-11 00:16:32,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +20: [2023-05-11 00:16:32,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +24: [2023-05-11 00:16:32,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 3: [2023-05-11 00:16:32,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 3: [2023-05-11 00:16:32,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 3: [2023-05-11 00:16:32,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 5: [2023-05-11 00:16:32,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 0: [2023-05-11 00:16:32,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 0: [2023-05-11 00:16:32,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 5: [2023-05-11 00:16:32,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +22: [2023-05-11 00:16:32,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +18: [2023-05-11 00:16:32,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +18: [2023-05-11 00:16:32,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +18: [2023-05-11 00:16:32,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +14: [2023-05-11 00:16:32,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +14: [2023-05-11 00:16:32,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +18: [2023-05-11 00:16:32,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +22: [2023-05-11 00:16:32,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +14: [2023-05-11 00:16:32,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +22: [2023-05-11 00:16:32,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +14: [2023-05-11 00:16:32,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 6: [2023-05-11 00:16:32,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +30: [2023-05-11 00:16:32,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 2: [2023-05-11 00:16:32,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 2: [2023-05-11 00:16:32,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. + 2: [2023-05-11 00:16:32,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +22: [2023-05-11 00:16:32,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +11: [2023-05-11 00:16:32,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +11: [2023-05-11 00:16:32,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +16: [2023-05-11 00:16:32,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +17: [2023-05-11 00:16:32,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +16: [2023-05-11 00:16:32,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 7: [2023-05-11 00:16:32,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +30: [2023-05-11 00:16:32,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +30: [2023-05-11 00:16:32,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +30: [2023-05-11 00:16:32,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +11: [2023-05-11 00:16:32,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +17: [2023-05-11 00:16:32,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +17: [2023-05-11 00:16:32,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +16: [2023-05-11 00:16:32,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +11: [2023-05-11 00:16:32,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +16: [2023-05-11 00:16:32,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +30: [2023-05-11 00:16:32,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +30: [2023-05-11 00:16:32,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 0: [2023-05-11 00:16:32,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +18: [2023-05-11 00:16:32,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +30: [2023-05-11 00:16:32,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +14: [2023-05-11 00:16:32,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +22: [2023-05-11 00:16:32,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 4: [2023-05-11 00:16:32,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +22: [2023-05-11 00:16:32,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +14: [2023-05-11 00:16:32,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +18: [2023-05-11 00:16:32,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +18: [2023-05-11 00:16:32,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +17: [2023-05-11 00:16:32,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 8: [2023-05-11 00:16:32,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 8: [2023-05-11 00:16:32,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 8: [2023-05-11 00:16:32,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 8: [2023-05-11 00:16:32,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +22: [2023-05-11 00:16:32,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +14: [2023-05-11 00:16:32,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +14: [2023-05-11 00:16:32,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +16: [2023-05-11 00:16:32,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +16: [2023-05-11 00:16:32,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +18: [2023-05-11 00:16:32,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +24: [2023-05-11 00:16:32,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +24: [2023-05-11 00:16:32,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +24: [2023-05-11 00:16:32,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +11: [2023-05-11 00:16:32,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +24: [2023-05-11 00:16:32,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +22: [2023-05-11 00:16:32,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +11: [2023-05-11 00:16:32,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 2: [2023-05-11 00:16:32,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +24: [2023-05-11 00:16:32,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 7: [2023-05-11 00:16:32,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 7: [2023-05-11 00:16:32,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +11: [2023-05-11 00:16:32,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +17: [2023-05-11 00:16:32,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +17: [2023-05-11 00:16:32,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 2: [2023-05-11 00:16:32,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 8: [2023-05-11 00:16:32,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +24: [2023-05-11 00:16:32,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +11: [2023-05-11 00:16:32,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +14: [2023-05-11 00:16:32,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +24: [2023-05-11 00:16:32,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +14: [2023-05-11 00:16:32,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 0: [2023-05-11 00:16:32,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +17: [2023-05-11 00:16:32,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +17: [2023-05-11 00:16:32,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 0: [2023-05-11 00:16:32,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 3: [2023-05-11 00:16:32,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +16: [2023-05-11 00:16:32,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +24: [2023-05-11 00:16:32,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +24: [2023-05-11 00:16:32,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +18: [2023-05-11 00:16:32,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +18: [2023-05-11 00:16:32,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +22: [2023-05-11 00:16:32,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +22: [2023-05-11 00:16:32,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 3: [2023-05-11 00:16:32,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 3: [2023-05-11 00:16:32,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +24: [2023-05-11 00:16:32,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +24: [2023-05-11 00:16:32,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +24: [2023-05-11 00:16:32,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +18: [2023-05-11 00:16:32,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +18: [2023-05-11 00:16:32,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +18: [2023-05-11 00:16:32,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +18: [2023-05-11 00:16:32,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +16: [2023-05-11 00:16:32,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 2: [2023-05-11 00:16:32,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +22: [2023-05-11 00:16:32,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +22: [2023-05-11 00:16:32,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 2: [2023-05-11 00:16:32,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +22: [2023-05-11 00:16:32,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +22: [2023-05-11 00:16:32,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +22: [2023-05-11 00:16:32,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +22: [2023-05-11 00:16:32,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 3: [2023-05-11 00:16:32,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +17: [2023-05-11 00:16:32,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +17: [2023-05-11 00:16:32,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +22: [2023-05-11 00:16:32,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +22: [2023-05-11 00:16:32,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +22: [2023-05-11 00:16:32,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +22: [2023-05-11 00:16:32,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +14: [2023-05-11 00:16:32,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +14: [2023-05-11 00:16:32,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +22: [2023-05-11 00:16:32,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +22: [2023-05-11 00:16:32,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +22: [2023-05-11 00:16:32,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 0: [2023-05-11 00:16:32,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +22: [2023-05-11 00:16:32,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +14: [2023-05-11 00:16:32,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +14: [2023-05-11 00:16:32,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +24: [2023-05-11 00:16:32,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +24: [2023-05-11 00:16:32,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +17: [2023-05-11 00:16:32,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +17: [2023-05-11 00:16:32,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +18: [2023-05-11 00:16:32,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +14: [2023-05-11 00:16:32,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +24: [2023-05-11 00:16:32,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +24: [2023-05-11 00:16:32,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +18: [2023-05-11 00:16:32,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +24: [2023-05-11 00:16:32,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 2: [2023-05-11 00:16:32,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +23: [2023-05-11 00:16:32,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +23: [2023-05-11 00:16:32,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +23: [2023-05-11 00:16:32,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +23: [2023-05-11 00:16:32,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +14: [2023-05-11 00:16:32,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +27: [2023-05-11 00:16:32,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +26: [2023-05-11 00:16:32,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +26: [2023-05-11 00:16:32,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +27: [2023-05-11 00:16:32,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +27: [2023-05-11 00:16:32,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +27: [2023-05-11 00:16:32,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +24: [2023-05-11 00:16:32,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +26: [2023-05-11 00:16:32,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +26: [2023-05-11 00:16:32,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +24: [2023-05-11 00:16:32,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +31: [2023-05-11 00:16:32,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +31: [2023-05-11 00:16:32,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +31: [2023-05-11 00:16:32,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +29: [2023-05-11 00:16:32,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +29: [2023-05-11 00:16:32,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +29: [2023-05-11 00:16:32,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +29: [2023-05-11 00:16:32,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +31: [2023-05-11 00:16:32,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt. +13: [2023-05-11 00:16:32,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +13: [2023-05-11 00:16:32,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +13: [2023-05-11 00:16:32,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +13: [2023-05-11 00:16:32,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 1: [2023-05-11 00:16:32,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 1: [2023-05-11 00:16:32,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 1: [2023-05-11 00:16:32,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +24: [2023-05-11 00:16:32,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 1: [2023-05-11 00:16:32,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +24: [2023-05-11 00:16:32,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +24: [2023-05-11 00:16:32,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +22: [2023-05-11 00:16:32,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +22: [2023-05-11 00:16:32,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +22: [2023-05-11 00:16:32,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +26: [2023-05-11 00:16:32,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +23: [2023-05-11 00:16:32,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +27: [2023-05-11 00:16:32,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +23: [2023-05-11 00:16:32,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +23: [2023-05-11 00:16:32,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +27: [2023-05-11 00:16:32,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +26: [2023-05-11 00:16:32,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +27: [2023-05-11 00:16:32,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +31: [2023-05-11 00:16:32,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +26: [2023-05-11 00:16:32,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +29: [2023-05-11 00:16:32,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +29: [2023-05-11 00:16:32,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +13: [2023-05-11 00:16:32,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +23: [2023-05-11 00:16:32,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +26: [2023-05-11 00:16:32,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +29: [2023-05-11 00:16:32,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +29: [2023-05-11 00:16:32,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +13: [2023-05-11 00:16:32,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +13: [2023-05-11 00:16:32,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +27: [2023-05-11 00:16:32,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +31: [2023-05-11 00:16:32,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +22: [2023-05-11 00:16:32,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +13: [2023-05-11 00:16:32,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +31: [2023-05-11 00:16:32,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +31: [2023-05-11 00:16:32,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt... +22: [2023-05-11 00:16:32,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 2: [2023-05-11 00:16:32,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... + 6: [2023-05-11 00:16:32,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 6: [2023-05-11 00:16:32,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 5: [2023-05-11 00:16:32,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +20: [2023-05-11 00:16:32,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +20: [2023-05-11 00:16:32,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +20: [2023-05-11 00:16:32,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +20: [2023-05-11 00:16:32,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +20: [2023-05-11 00:16:32,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +20: [2023-05-11 00:16:32,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +20: [2023-05-11 00:16:32,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +20: [2023-05-11 00:16:32,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +20: [2023-05-11 00:16:32,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +20: [2023-05-11 00:16:32,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +20: [2023-05-11 00:16:32,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +20: [2023-05-11 00:16:32,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +20: [2023-05-11 00:16:32,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 6: [2023-05-11 00:16:32,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 5: [2023-05-11 00:16:32,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 8: [2023-05-11 00:16:32,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +20: [2023-05-11 00:16:32,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 8: [2023-05-11 00:16:32,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +27: [2023-05-11 00:16:32,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +27: [2023-05-11 00:16:32,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +22: [2023-05-11 00:16:32,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +20: [2023-05-11 00:16:32,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +29: [2023-05-11 00:16:32,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +29: [2023-05-11 00:16:32,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +29: [2023-05-11 00:16:32,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +29: [2023-05-11 00:16:32,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +29: [2023-05-11 00:16:32,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +20: [2023-05-11 00:16:32,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +29: [2023-05-11 00:16:32,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 6: [2023-05-11 00:16:32,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +29: [2023-05-11 00:16:32,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 8: [2023-05-11 00:16:32,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +22: [2023-05-11 00:16:32,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +27: [2023-05-11 00:16:32,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +27: [2023-05-11 00:16:32,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +27: [2023-05-11 00:16:32,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +26: [2023-05-11 00:16:32,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +31: [2023-05-11 00:16:32,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +27: [2023-05-11 00:16:32,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +31: [2023-05-11 00:16:32,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +27: [2023-05-11 00:16:32,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +29: [2023-05-11 00:16:32,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +26: [2023-05-11 00:16:32,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +26: [2023-05-11 00:16:32,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +26: [2023-05-11 00:16:32,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +24: [2023-05-11 00:16:32,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +27: [2023-05-11 00:16:32,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +27: [2023-05-11 00:16:32,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +23: [2023-05-11 00:16:32,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +23: [2023-05-11 00:16:32,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +23: [2023-05-11 00:16:32,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +29: [2023-05-11 00:16:32,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +29: [2023-05-11 00:16:32,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +26: [2023-05-11 00:16:32,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +26: [2023-05-11 00:16:32,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +26: [2023-05-11 00:16:32,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +26: [2023-05-11 00:16:32,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +24: [2023-05-11 00:16:32,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +16: [2023-05-11 00:16:32,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +16: [2023-05-11 00:16:32,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +27: [2023-05-11 00:16:32,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 6: [2023-05-11 00:16:32,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +22: [2023-05-11 00:16:32,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +23: [2023-05-11 00:16:32,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +23: [2023-05-11 00:16:32,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +23: [2023-05-11 00:16:32,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +16: [2023-05-11 00:16:32,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +16: [2023-05-11 00:16:32,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +29: [2023-05-11 00:16:32,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +29: [2023-05-11 00:16:32,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +29: [2023-05-11 00:16:32,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +29: [2023-05-11 00:16:32,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 8: [2023-05-11 00:16:32,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +20: [2023-05-11 00:16:32,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +27: [2023-05-11 00:16:32,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 5: [2023-05-11 00:16:32,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +16: [2023-05-11 00:16:32,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +16: [2023-05-11 00:16:32,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +20: [2023-05-11 00:16:32,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +16: [2023-05-11 00:16:32,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +16: [2023-05-11 00:16:32,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +27: [2023-05-11 00:16:32,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +27: [2023-05-11 00:16:32,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 1: [2023-05-11 00:16:32,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +23: [2023-05-11 00:16:32,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 8: [2023-05-11 00:16:32,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +31: [2023-05-11 00:16:32,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +31: [2023-05-11 00:16:32,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +29: [2023-05-11 00:16:32,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +23: [2023-05-11 00:16:32,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +20: [2023-05-11 00:16:32,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +27: [2023-05-11 00:16:32,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +29: [2023-05-11 00:16:32,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +20: [2023-05-11 00:16:32,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +31: [2023-05-11 00:16:32,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. +29: [2023-05-11 00:16:32,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +29: [2023-05-11 00:16:32,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +27: [2023-05-11 00:16:32,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +27: [2023-05-11 00:16:32,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +16: [2023-05-11 00:16:32,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +16: [2023-05-11 00:16:32,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +20: [2023-05-11 00:16:32,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +31: [2023-05-11 00:16:32,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +27: [2023-05-11 00:16:32,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +27: [2023-05-11 00:16:32,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +29: [2023-05-11 00:16:32,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +29: [2023-05-11 00:16:32,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +20: [2023-05-11 00:16:32,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +31: [2023-05-11 00:16:32,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +31: [2023-05-11 00:16:32,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt... +27: [2023-05-11 00:16:32,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +27: [2023-05-11 00:16:32,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +29: [2023-05-11 00:16:32,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +16: [2023-05-11 00:16:32,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +16: [2023-05-11 00:16:32,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +20: [2023-05-11 00:16:32,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +27: [2023-05-11 00:16:32,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +29: [2023-05-11 00:16:32,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +20: [2023-05-11 00:16:32,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +27: [2023-05-11 00:16:32,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +16: [2023-05-11 00:16:32,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +12: [2023-05-11 00:16:32,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +27: [2023-05-11 00:16:32,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +29: [2023-05-11 00:16:32,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +12: [2023-05-11 00:16:32,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +12: [2023-05-11 00:16:32,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +12: [2023-05-11 00:16:32,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +16: [2023-05-11 00:16:32,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +16: [2023-05-11 00:16:32,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +27: [2023-05-11 00:16:32,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +12: [2023-05-11 00:16:32,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +29: [2023-05-11 00:16:32,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +18: [2023-05-11 00:16:32,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +18: [2023-05-11 00:16:32,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +12: [2023-05-11 00:16:32,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +12: [2023-05-11 00:16:32,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +12: [2023-05-11 00:16:32,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +16: [2023-05-11 00:16:32,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +18: [2023-05-11 00:16:32,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +18: [2023-05-11 00:16:32,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +18: [2023-05-11 00:16:32,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +18: [2023-05-11 00:16:32,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +18: [2023-05-11 00:16:32,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +18: [2023-05-11 00:16:32,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 9: [2023-05-11 00:16:32,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 9: [2023-05-11 00:16:32,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +15: [2023-05-11 00:16:32,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +15: [2023-05-11 00:16:32,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 9: [2023-05-11 00:16:32,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +15: [2023-05-11 00:16:32,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +10: [2023-05-11 00:16:32,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +10: [2023-05-11 00:16:32,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +15: [2023-05-11 00:16:32,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 3: [2023-05-11 00:16:32,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +10: [2023-05-11 00:16:32,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +10: [2023-05-11 00:16:32,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 3: [2023-05-11 00:16:32,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +18: [2023-05-11 00:16:32,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +18: [2023-05-11 00:16:32,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +18: [2023-05-11 00:16:32,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +15: [2023-05-11 00:16:32,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +15: [2023-05-11 00:16:32,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +25: [2023-05-11 00:16:32,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 9: [2023-05-11 00:16:32,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +25: [2023-05-11 00:16:32,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +15: [2023-05-11 00:16:32,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +15: [2023-05-11 00:16:32,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +10: [2023-05-11 00:16:32,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +10: [2023-05-11 00:16:32,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +25: [2023-05-11 00:16:32,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 9: [2023-05-11 00:16:32,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +25: [2023-05-11 00:16:32,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +10: [2023-05-11 00:16:32,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +18: [2023-05-11 00:16:32,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +10: [2023-05-11 00:16:32,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +18: [2023-05-11 00:16:32,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +18: [2023-05-11 00:16:32,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +21: [2023-05-11 00:16:32,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +21: [2023-05-11 00:16:32,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +21: [2023-05-11 00:16:32,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +21: [2023-05-11 00:16:32,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 0: [2023-05-11 00:16:32,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 0: [2023-05-11 00:16:32,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 0: [2023-05-11 00:16:32,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 0: [2023-05-11 00:16:32,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +25: [2023-05-11 00:16:32,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +25: [2023-05-11 00:16:32,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +25: [2023-05-11 00:16:32,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +11: [2023-05-11 00:16:32,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +18: [2023-05-11 00:16:32,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +11: [2023-05-11 00:16:32,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +11: [2023-05-11 00:16:32,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +11: [2023-05-11 00:16:32,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +28: [2023-05-11 00:16:32,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +28: [2023-05-11 00:16:32,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +28: [2023-05-11 00:16:32,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +25: [2023-05-11 00:16:32,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +28: [2023-05-11 00:16:32,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +17: [2023-05-11 00:16:32,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +17: [2023-05-11 00:16:32,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +17: [2023-05-11 00:16:32,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +21: [2023-05-11 00:16:32,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +21: [2023-05-11 00:16:32,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 4: [2023-05-11 00:16:32,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 4: [2023-05-11 00:16:32,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +17: [2023-05-11 00:16:32,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 7: [2023-05-11 00:16:32,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 7: [2023-05-11 00:16:32,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 7: [2023-05-11 00:16:32,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +18: [2023-05-11 00:16:32,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +18: [2023-05-11 00:16:32,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +21: [2023-05-11 00:16:32,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +18: [2023-05-11 00:16:32,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +21: [2023-05-11 00:16:32,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +18: [2023-05-11 00:16:32,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +11: [2023-05-11 00:16:32,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +11: [2023-05-11 00:16:32,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +18: [2023-05-11 00:16:32,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +12: [2023-05-11 00:16:32,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +12: [2023-05-11 00:16:32,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +12: [2023-05-11 00:16:32,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +12: [2023-05-11 00:16:32,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +12: [2023-05-11 00:16:32,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +12: [2023-05-11 00:16:32,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +12: [2023-05-11 00:16:32,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +12: [2023-05-11 00:16:32,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +12: [2023-05-11 00:16:32,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +12: [2023-05-11 00:16:32,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +12: [2023-05-11 00:16:32,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +12: [2023-05-11 00:16:32,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +12: [2023-05-11 00:16:32,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +12: [2023-05-11 00:16:32,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +12: [2023-05-11 00:16:32,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 3: [2023-05-11 00:16:32,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +28: [2023-05-11 00:16:32,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +12: [2023-05-11 00:16:32,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +28: [2023-05-11 00:16:32,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +28: [2023-05-11 00:16:32,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +28: [2023-05-11 00:16:32,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +13: [2023-05-11 00:16:32,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +13: [2023-05-11 00:16:32,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +13: [2023-05-11 00:16:32,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +11: [2023-05-11 00:16:32,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +13: [2023-05-11 00:16:32,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +11: [2023-05-11 00:16:32,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +22: [2023-05-11 00:16:32,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +22: [2023-05-11 00:16:32,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +22: [2023-05-11 00:16:32,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +22: [2023-05-11 00:16:32,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +17: [2023-05-11 00:16:32,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +17: [2023-05-11 00:16:32,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +13: [2023-05-11 00:16:32,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +17: [2023-05-11 00:16:32,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +13: [2023-05-11 00:16:32,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +13: [2023-05-11 00:16:32,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +13: [2023-05-11 00:16:32,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +13: [2023-05-11 00:16:32,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +26: [2023-05-11 00:16:32,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +26: [2023-05-11 00:16:32,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +26: [2023-05-11 00:16:32,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +26: [2023-05-11 00:16:32,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +23: [2023-05-11 00:16:32,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +23: [2023-05-11 00:16:32,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +23: [2023-05-11 00:16:32,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +23: [2023-05-11 00:16:32,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +13: [2023-05-11 00:16:32,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +13: [2023-05-11 00:16:32,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +26: [2023-05-11 00:16:32,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +13: [2023-05-11 00:16:32,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +14: [2023-05-11 00:16:32,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +14: [2023-05-11 00:16:32,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +13: [2023-05-11 00:16:32,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +17: [2023-05-11 00:16:32,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +13: [2023-05-11 00:16:32,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +17: [2023-05-11 00:16:32,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +26: [2023-05-11 00:16:32,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +26: [2023-05-11 00:16:32,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +17: [2023-05-11 00:16:32,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +14: [2023-05-11 00:16:32,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +13: [2023-05-11 00:16:32,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +17: [2023-05-11 00:16:32,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +14: [2023-05-11 00:16:32,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +19: [2023-05-11 00:16:32,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +19: [2023-05-11 00:16:32,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 4: [2023-05-11 00:16:32,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +17: [2023-05-11 00:16:32,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +13: [2023-05-11 00:16:32,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +26: [2023-05-11 00:16:32,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +17: [2023-05-11 00:16:32,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +17: [2023-05-11 00:16:32,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +18: [2023-05-11 00:16:32,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +23: [2023-05-11 00:16:32,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +23: [2023-05-11 00:16:32,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +18: [2023-05-11 00:16:32,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +18: [2023-05-11 00:16:32,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +18: [2023-05-11 00:16:32,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +14: [2023-05-11 00:16:32,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +14: [2023-05-11 00:16:32,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +23: [2023-05-11 00:16:32,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +19: [2023-05-11 00:16:32,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +23: [2023-05-11 00:16:32,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +19: [2023-05-11 00:16:32,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 3: [2023-05-11 00:16:32,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +12: [2023-05-11 00:16:32,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +14: [2023-05-11 00:16:32,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +14: [2023-05-11 00:16:32,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +17: [2023-05-11 00:16:32,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +22: [2023-05-11 00:16:32,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +22: [2023-05-11 00:16:32,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +22: [2023-05-11 00:16:32,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +22: [2023-05-11 00:16:32,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +17: [2023-05-11 00:16:32,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +10: [2023-05-11 00:16:32,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +10: [2023-05-11 00:16:32,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +10: [2023-05-11 00:16:32,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +10: [2023-05-11 00:16:32,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +10: [2023-05-11 00:16:32,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +10: [2023-05-11 00:16:32,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +10: [2023-05-11 00:16:32,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +10: [2023-05-11 00:16:32,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +10: [2023-05-11 00:16:32,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +10: [2023-05-11 00:16:32,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +10: [2023-05-11 00:16:32,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +10: [2023-05-11 00:16:32,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +10: [2023-05-11 00:16:32,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +10: [2023-05-11 00:16:32,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 7: [2023-05-11 00:16:32,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +19: [2023-05-11 00:16:32,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +10: [2023-05-11 00:16:32,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +12: [2023-05-11 00:16:32,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +12: [2023-05-11 00:16:32,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +23: [2023-05-11 00:16:32,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +16: [2023-05-11 00:16:32,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +16: [2023-05-11 00:16:32,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +16: [2023-05-11 00:16:32,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +10: [2023-05-11 00:16:32,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +14: [2023-05-11 00:16:32,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +23: [2023-05-11 00:16:32,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +16: [2023-05-11 00:16:32,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 1: [2023-05-11 00:16:32,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +13: [2023-05-11 00:16:32,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +19: [2023-05-11 00:16:32,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +19: [2023-05-11 00:16:32,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +13: [2023-05-11 00:16:32,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +17: [2023-05-11 00:16:32,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +17: [2023-05-11 00:16:32,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +11: [2023-05-11 00:16:32,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +11: [2023-05-11 00:16:32,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +11: [2023-05-11 00:16:32,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +11: [2023-05-11 00:16:32,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +11: [2023-05-11 00:16:32,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +11: [2023-05-11 00:16:32,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +11: [2023-05-11 00:16:32,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +11: [2023-05-11 00:16:32,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +11: [2023-05-11 00:16:32,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +11: [2023-05-11 00:16:32,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +11: [2023-05-11 00:16:32,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +19: [2023-05-11 00:16:32,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +14: [2023-05-11 00:16:32,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +13: [2023-05-11 00:16:32,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +30: [2023-05-11 00:16:32,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +30: [2023-05-11 00:16:32,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 0: [2023-05-11 00:16:32,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +17: [2023-05-11 00:16:32,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +11: [2023-05-11 00:16:32,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +17: [2023-05-11 00:16:32,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 0: [2023-05-11 00:16:32,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +31: [2023-05-11 00:16:32,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +31: [2023-05-11 00:16:32,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +11: [2023-05-11 00:16:32,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +11: [2023-05-11 00:16:32,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +31: [2023-05-11 00:16:32,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +12: [2023-05-11 00:16:32,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +30: [2023-05-11 00:16:32,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 7: [2023-05-11 00:16:32,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +30: [2023-05-11 00:16:32,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +23: [2023-05-11 00:16:32,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +31: [2023-05-11 00:16:32,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +10: [2023-05-11 00:16:32,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +26: [2023-05-11 00:16:32,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +26: [2023-05-11 00:16:32,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +26: [2023-05-11 00:16:32,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +12: [2023-05-11 00:16:32,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +17: [2023-05-11 00:16:32,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +17: [2023-05-11 00:16:32,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +17: [2023-05-11 00:16:32,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +14: [2023-05-11 00:16:32,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +10: [2023-05-11 00:16:32,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +31: [2023-05-11 00:16:32,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +31: [2023-05-11 00:16:32,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +18: [2023-05-11 00:16:32,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +31: [2023-05-11 00:16:32,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +18: [2023-05-11 00:16:32,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 4: [2023-05-11 00:16:32,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +23: [2023-05-11 00:16:32,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +13: [2023-05-11 00:16:32,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +17: [2023-05-11 00:16:32,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +17: [2023-05-11 00:16:32,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +12: [2023-05-11 00:16:32,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +31: [2023-05-11 00:16:32,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +17: [2023-05-11 00:16:32,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +13: [2023-05-11 00:16:32,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +24: [2023-05-11 00:16:32,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +24: [2023-05-11 00:16:32,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 4: [2023-05-11 00:16:32,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +22: [2023-05-11 00:16:32,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +22: [2023-05-11 00:16:32,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +19: [2023-05-11 00:16:32,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +19: [2023-05-11 00:16:32,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +16: [2023-05-11 00:16:32,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +16: [2023-05-11 00:16:32,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +16: [2023-05-11 00:16:32,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +22: [2023-05-11 00:16:32,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +22: [2023-05-11 00:16:32,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +18: [2023-05-11 00:16:32,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +16: [2023-05-11 00:16:32,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 5: [2023-05-11 00:16:32,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +19: [2023-05-11 00:16:32,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +19: [2023-05-11 00:16:32,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 0: [2023-05-11 00:16:32,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 8: [2023-05-11 00:16:32,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 8: [2023-05-11 00:16:32,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 4: [2023-05-11 00:16:32,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 4: [2023-05-11 00:16:32,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +19: [2023-05-11 00:16:32,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +18: [2023-05-11 00:16:32,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +14: [2023-05-11 00:16:32,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +11: [2023-05-11 00:16:32,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +19: [2023-05-11 00:16:32,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +24: [2023-05-11 00:16:32,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 1: [2023-05-11 00:16:32,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +11: [2023-05-11 00:16:32,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +11: [2023-05-11 00:16:32,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +30: [2023-05-11 00:16:32,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +24: [2023-05-11 00:16:32,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +18: [2023-05-11 00:16:32,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +10: [2023-05-11 00:16:32,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +30: [2023-05-11 00:16:32,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +13: [2023-05-11 00:16:32,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +12: [2023-05-11 00:16:32,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +19: [2023-05-11 00:16:32,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +17: [2023-05-11 00:16:32,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +18: [2023-05-11 00:16:32,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +29: [2023-05-11 00:16:32,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +29: [2023-05-11 00:16:32,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +18: [2023-05-11 00:16:32,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +11: [2023-05-11 00:16:32,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +31: [2023-05-11 00:16:32,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +30: [2023-05-11 00:16:32,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +29: [2023-05-11 00:16:32,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +30: [2023-05-11 00:16:32,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +26: [2023-05-11 00:16:32,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +29: [2023-05-11 00:16:32,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +13: [2023-05-11 00:16:32,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +26: [2023-05-11 00:16:32,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +26: [2023-05-11 00:16:32,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +10: [2023-05-11 00:16:32,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +17: [2023-05-11 00:16:32,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +19: [2023-05-11 00:16:32,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +22: [2023-05-11 00:16:32,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +23: [2023-05-11 00:16:32,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +18: [2023-05-11 00:16:32,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +27: [2023-05-11 00:16:32,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +10: [2023-05-11 00:16:32,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +27: [2023-05-11 00:16:32,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +27: [2023-05-11 00:16:32,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 5: [2023-05-11 00:16:32,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 6: [2023-05-11 00:16:32,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 6: [2023-05-11 00:16:32,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 6: [2023-05-11 00:16:32,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 6: [2023-05-11 00:16:32,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +24: [2023-05-11 00:16:32,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +22: [2023-05-11 00:16:32,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +22: [2023-05-11 00:16:32,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +20: [2023-05-11 00:16:32,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +20: [2023-05-11 00:16:32,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +20: [2023-05-11 00:16:32,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +11: [2023-05-11 00:16:32,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +20: [2023-05-11 00:16:32,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 1: [2023-05-11 00:16:32,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +24: [2023-05-11 00:16:32,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +23: [2023-05-11 00:16:32,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +17: [2023-05-11 00:16:32,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +22: [2023-05-11 00:16:32,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +27: [2023-05-11 00:16:32,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +31: [2023-05-11 00:16:32,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +14: [2023-05-11 00:16:32,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +14: [2023-05-11 00:16:32,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +14: [2023-05-11 00:16:32,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 8: [2023-05-11 00:16:32,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 4: [2023-05-11 00:16:32,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 2: [2023-05-11 00:16:32,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 2: [2023-05-11 00:16:32,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +19: [2023-05-11 00:16:32,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +19: [2023-05-11 00:16:32,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +10: [2023-05-11 00:16:32,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +17: [2023-05-11 00:16:32,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +11: [2023-05-11 00:16:32,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +24: [2023-05-11 00:16:32,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +26: [2023-05-11 00:16:32,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +14: [2023-05-11 00:16:32,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +14: [2023-05-11 00:16:32,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +12: [2023-05-11 00:16:32,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +11: [2023-05-11 00:16:32,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +29: [2023-05-11 00:16:32,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +29: [2023-05-11 00:16:32,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +24: [2023-05-11 00:16:32,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +29: [2023-05-11 00:16:32,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +19: [2023-05-11 00:16:32,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +19: [2023-05-11 00:16:32,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 0: [2023-05-11 00:16:32,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +13: [2023-05-11 00:16:32,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +10: [2023-05-11 00:16:32,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +31: [2023-05-11 00:16:32,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +19: [2023-05-11 00:16:32,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +19: [2023-05-11 00:16:32,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +27: [2023-05-11 00:16:32,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +19: [2023-05-11 00:16:32,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +16: [2023-05-11 00:16:32,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +16: [2023-05-11 00:16:32,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +16: [2023-05-11 00:16:32,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +16: [2023-05-11 00:16:32,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +16: [2023-05-11 00:16:32,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +16: [2023-05-11 00:16:32,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +27: [2023-05-11 00:16:32,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +20: [2023-05-11 00:16:32,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +23: [2023-05-11 00:16:32,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +11: [2023-05-11 00:16:32,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +17: [2023-05-11 00:16:32,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +14: [2023-05-11 00:16:32,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +20: [2023-05-11 00:16:32,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +29: [2023-05-11 00:16:32,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +15: [2023-05-11 00:16:32,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +15: [2023-05-11 00:16:32,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +15: [2023-05-11 00:16:32,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +15: [2023-05-11 00:16:32,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +15: [2023-05-11 00:16:32,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +15: [2023-05-11 00:16:32,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 6: [2023-05-11 00:16:32,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +15: [2023-05-11 00:16:32,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +15: [2023-05-11 00:16:32,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +20: [2023-05-11 00:16:32,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +15: [2023-05-11 00:16:32,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +15: [2023-05-11 00:16:32,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +15: [2023-05-11 00:16:32,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +15: [2023-05-11 00:16:32,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +27: [2023-05-11 00:16:32,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +27: [2023-05-11 00:16:32,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +15: [2023-05-11 00:16:32,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +15: [2023-05-11 00:16:32,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +15: [2023-05-11 00:16:32,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +15: [2023-05-11 00:16:32,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +23: [2023-05-11 00:16:32,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +14: [2023-05-11 00:16:32,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +20: [2023-05-11 00:16:32,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +14: [2023-05-11 00:16:32,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +26: [2023-05-11 00:16:32,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +14: [2023-05-11 00:16:32,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +19: [2023-05-11 00:16:32,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +19: [2023-05-11 00:16:32,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +19: [2023-05-11 00:16:32,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +25: [2023-05-11 00:16:32,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +25: [2023-05-11 00:16:32,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +25: [2023-05-11 00:16:32,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +25: [2023-05-11 00:16:32,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +25: [2023-05-11 00:16:32,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +25: [2023-05-11 00:16:32,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +25: [2023-05-11 00:16:32,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +25: [2023-05-11 00:16:32,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +19: [2023-05-11 00:16:32,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +25: [2023-05-11 00:16:32,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +25: [2023-05-11 00:16:32,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 2: [2023-05-11 00:16:32,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +25: [2023-05-11 00:16:32,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +14: [2023-05-11 00:16:32,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +25: [2023-05-11 00:16:32,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +25: [2023-05-11 00:16:32,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +25: [2023-05-11 00:16:32,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +31: [2023-05-11 00:16:32,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +31: [2023-05-11 00:16:32,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +14: [2023-05-11 00:16:32,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +11: [2023-05-11 00:16:32,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +25: [2023-05-11 00:16:32,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +10: [2023-05-11 00:16:32,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +17: [2023-05-11 00:16:32,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +16: [2023-05-11 00:16:32,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +25: [2023-05-11 00:16:32,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +31: [2023-05-11 00:16:32,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +16: [2023-05-11 00:16:32,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +14: [2023-05-11 00:16:32,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +15: [2023-05-11 00:16:32,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +15: [2023-05-11 00:16:32,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +11: [2023-05-11 00:16:32,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +19: [2023-05-11 00:16:32,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +15: [2023-05-11 00:16:32,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +14: [2023-05-11 00:16:32,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +14: [2023-05-11 00:16:32,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +14: [2023-05-11 00:16:32,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 4: [2023-05-11 00:16:32,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +14: [2023-05-11 00:16:32,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +15: [2023-05-11 00:16:32,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +25: [2023-05-11 00:16:32,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +25: [2023-05-11 00:16:32,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +19: [2023-05-11 00:16:32,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +31: [2023-05-11 00:16:32,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +31: [2023-05-11 00:16:32,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +23: [2023-05-11 00:16:32,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +23: [2023-05-11 00:16:32,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +25: [2023-05-11 00:16:32,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +15: [2023-05-11 00:16:32,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +26: [2023-05-11 00:16:32,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +26: [2023-05-11 00:16:32,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +26: [2023-05-11 00:16:32,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +26: [2023-05-11 00:16:32,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +15: [2023-05-11 00:16:32,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +25: [2023-05-11 00:16:32,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +19: [2023-05-11 00:16:32,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +14: [2023-05-11 00:16:32,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +14: [2023-05-11 00:16:32,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +15: [2023-05-11 00:16:32,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +23: [2023-05-11 00:16:32,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 2: [2023-05-11 00:16:32,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +23: [2023-05-11 00:16:32,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 2: [2023-05-11 00:16:32,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 2: [2023-05-11 00:16:32,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +15: [2023-05-11 00:16:32,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +19: [2023-05-11 00:16:32,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 3: [2023-05-11 00:16:32,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +14: [2023-05-11 00:16:32,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +14: [2023-05-11 00:16:32,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 3: [2023-05-11 00:16:32,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. + 3: [2023-05-11 00:16:32,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +25: [2023-05-11 00:16:32,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +25: [2023-05-11 00:16:32,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +23: [2023-05-11 00:16:32,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +26: [2023-05-11 00:16:32,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +26: [2023-05-11 00:16:32,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +26: [2023-05-11 00:16:32,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +30: [2023-05-11 00:16:32,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +30: [2023-05-11 00:16:32,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +30: [2023-05-11 00:16:32,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +30: [2023-05-11 00:16:32,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +30: [2023-05-11 00:16:32,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +30: [2023-05-11 00:16:32,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +30: [2023-05-11 00:16:32,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +30: [2023-05-11 00:16:32,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +30: [2023-05-11 00:16:32,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +30: [2023-05-11 00:16:32,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +30: [2023-05-11 00:16:32,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +28: [2023-05-11 00:16:32,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +30: [2023-05-11 00:16:32,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +30: [2023-05-11 00:16:32,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +30: [2023-05-11 00:16:32,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +30: [2023-05-11 00:16:32,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +28: [2023-05-11 00:16:32,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +28: [2023-05-11 00:16:32,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +28: [2023-05-11 00:16:32,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +28: [2023-05-11 00:16:32,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +23: [2023-05-11 00:16:32,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +28: [2023-05-11 00:16:32,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +28: [2023-05-11 00:16:32,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +28: [2023-05-11 00:16:32,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +28: [2023-05-11 00:16:32,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +23: [2023-05-11 00:16:32,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +28: [2023-05-11 00:16:32,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +23: [2023-05-11 00:16:32,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +30: [2023-05-11 00:16:32,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +28: [2023-05-11 00:16:32,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +28: [2023-05-11 00:16:32,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +28: [2023-05-11 00:16:32,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +19: [2023-05-11 00:16:32,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +28: [2023-05-11 00:16:32,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +28: [2023-05-11 00:16:32,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +25: [2023-05-11 00:16:32,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +26: [2023-05-11 00:16:32,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 9: [2023-05-11 00:16:32,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 9: [2023-05-11 00:16:32,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 9: [2023-05-11 00:16:32,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +28: [2023-05-11 00:16:32,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 7: [2023-05-11 00:16:32,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +31: [2023-05-11 00:16:32,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +31: [2023-05-11 00:16:32,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +31: [2023-05-11 00:16:32,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +31: [2023-05-11 00:16:32,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt. +26: [2023-05-11 00:16:32,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +26: [2023-05-11 00:16:32,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 7: [2023-05-11 00:16:32,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +21: [2023-05-11 00:16:32,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +21: [2023-05-11 00:16:32,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +21: [2023-05-11 00:16:32,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +21: [2023-05-11 00:16:32,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +21: [2023-05-11 00:16:32,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +21: [2023-05-11 00:16:32,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +21: [2023-05-11 00:16:32,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +21: [2023-05-11 00:16:32,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +21: [2023-05-11 00:16:32,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +21: [2023-05-11 00:16:32,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +21: [2023-05-11 00:16:32,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +21: [2023-05-11 00:16:32,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +21: [2023-05-11 00:16:32,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +21: [2023-05-11 00:16:32,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +21: [2023-05-11 00:16:32,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 7: [2023-05-11 00:16:32,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +23: [2023-05-11 00:16:32,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +23: [2023-05-11 00:16:32,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +24: [2023-05-11 00:16:32,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +24: [2023-05-11 00:16:32,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +24: [2023-05-11 00:16:32,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +24: [2023-05-11 00:16:32,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +21: [2023-05-11 00:16:32,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +30: [2023-05-11 00:16:32,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +30: [2023-05-11 00:16:32,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +24: [2023-05-11 00:16:32,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +24: [2023-05-11 00:16:32,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +24: [2023-05-11 00:16:32,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +30: [2023-05-11 00:16:32,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +24: [2023-05-11 00:16:32,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +24: [2023-05-11 00:16:32,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +24: [2023-05-11 00:16:32,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +24: [2023-05-11 00:16:32,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +24: [2023-05-11 00:16:32,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +24: [2023-05-11 00:16:32,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +24: [2023-05-11 00:16:32,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +22: [2023-05-11 00:16:32,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +22: [2023-05-11 00:16:32,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +25: [2023-05-11 00:16:32,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +22: [2023-05-11 00:16:32,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +22: [2023-05-11 00:16:32,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +22: [2023-05-11 00:16:32,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +22: [2023-05-11 00:16:32,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +22: [2023-05-11 00:16:32,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +22: [2023-05-11 00:16:32,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +28: [2023-05-11 00:16:32,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +24: [2023-05-11 00:16:32,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +23: [2023-05-11 00:16:32,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +23: [2023-05-11 00:16:32,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 1: [2023-05-11 00:16:32,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +28: [2023-05-11 00:16:32,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 3: [2023-05-11 00:16:32,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 1: [2023-05-11 00:16:32,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +24: [2023-05-11 00:16:32,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +28: [2023-05-11 00:16:32,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +28: [2023-05-11 00:16:32,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +23: [2023-05-11 00:16:32,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +23: [2023-05-11 00:16:32,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +31: [2023-05-11 00:16:32,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +31: [2023-05-11 00:16:32,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +31: [2023-05-11 00:16:32,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... +31: [2023-05-11 00:16:32,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +21: [2023-05-11 00:16:32,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +26: [2023-05-11 00:16:32,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +24: [2023-05-11 00:16:32,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +26: [2023-05-11 00:16:32,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +30: [2023-05-11 00:16:32,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +26: [2023-05-11 00:16:32,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +22: [2023-05-11 00:16:32,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +30: [2023-05-11 00:16:32,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +30: [2023-05-11 00:16:32,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +22: [2023-05-11 00:16:32,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +28: [2023-05-11 00:16:32,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +26: [2023-05-11 00:16:32,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +28: [2023-05-11 00:16:32,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +28: [2023-05-11 00:16:32,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +23: [2023-05-11 00:16:32,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +28: [2023-05-11 00:16:32,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +22: [2023-05-11 00:16:32,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +22: [2023-05-11 00:16:32,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +21: [2023-05-11 00:16:32,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +21: [2023-05-11 00:16:32,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +21: [2023-05-11 00:16:32,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +21: [2023-05-11 00:16:32,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +23: [2023-05-11 00:16:32,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 9: [2023-05-11 00:16:32,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +24: [2023-05-11 00:16:32,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +24: [2023-05-11 00:16:32,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +26: [2023-05-11 00:16:32,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +30: [2023-05-11 00:16:32,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +22: [2023-05-11 00:16:32,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +26: [2023-05-11 00:16:32,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +24: [2023-05-11 00:16:32,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +22: [2023-05-11 00:16:32,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 7: [2023-05-11 00:16:32,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +31: [2023-05-11 00:16:32,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +21: [2023-05-11 00:16:32,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +21: [2023-05-11 00:16:32,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +22: [2023-05-11 00:16:32,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +31: [2023-05-11 00:16:32,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +30: [2023-05-11 00:16:32,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +22: [2023-05-11 00:16:32,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +24: [2023-05-11 00:16:32,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +21: [2023-05-11 00:16:32,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +24: [2023-05-11 00:16:32,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 1: [2023-05-11 00:16:32,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +31: [2023-05-11 00:16:32,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +31: [2023-05-11 00:16:32,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +31: [2023-05-11 00:16:32,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt. +31: [2023-05-11 00:16:32,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +31: [2023-05-11 00:16:32,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... +31: [2023-05-11 00:16:32,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt... + 5: [2023-05-11 00:16:32,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 5: [2023-05-11 00:16:32,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 5: [2023-05-11 00:16:32,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 5: [2023-05-11 00:16:32,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 7: [2023-05-11 00:16:32,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +24: [2023-05-11 00:16:32,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +24: [2023-05-11 00:16:32,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +29: [2023-05-11 00:16:32,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +29: [2023-05-11 00:16:32,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +29: [2023-05-11 00:16:32,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +29: [2023-05-11 00:16:32,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +29: [2023-05-11 00:16:32,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +29: [2023-05-11 00:16:32,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +29: [2023-05-11 00:16:32,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +29: [2023-05-11 00:16:32,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +29: [2023-05-11 00:16:32,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +29: [2023-05-11 00:16:32,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +29: [2023-05-11 00:16:32,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +29: [2023-05-11 00:16:32,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +29: [2023-05-11 00:16:32,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +29: [2023-05-11 00:16:32,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +29: [2023-05-11 00:16:32,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 5: [2023-05-11 00:16:32,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +18: [2023-05-11 00:16:32,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +18: [2023-05-11 00:16:32,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +18: [2023-05-11 00:16:32,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +18: [2023-05-11 00:16:32,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +29: [2023-05-11 00:16:32,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 6: [2023-05-11 00:16:32,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 6: [2023-05-11 00:16:32,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +18: [2023-05-11 00:16:32,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +18: [2023-05-11 00:16:32,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +18: [2023-05-11 00:16:32,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +18: [2023-05-11 00:16:32,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 6: [2023-05-11 00:16:32,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +16: [2023-05-11 00:16:32,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +16: [2023-05-11 00:16:32,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +16: [2023-05-11 00:16:32,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +16: [2023-05-11 00:16:32,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +16: [2023-05-11 00:16:32,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +16: [2023-05-11 00:16:32,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +16: [2023-05-11 00:16:32,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +16: [2023-05-11 00:16:32,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 6: [2023-05-11 00:16:32,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 6: [2023-05-11 00:16:32,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +29: [2023-05-11 00:16:32,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +18: [2023-05-11 00:16:32,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +18: [2023-05-11 00:16:32,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +20: [2023-05-11 00:16:32,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +20: [2023-05-11 00:16:32,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +20: [2023-05-11 00:16:32,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +20: [2023-05-11 00:16:32,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +20: [2023-05-11 00:16:32,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +20: [2023-05-11 00:16:32,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +20: [2023-05-11 00:16:32,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +20: [2023-05-11 00:16:32,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +20: [2023-05-11 00:16:32,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +20: [2023-05-11 00:16:32,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +20: [2023-05-11 00:16:32,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +20: [2023-05-11 00:16:32,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +20: [2023-05-11 00:16:32,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +20: [2023-05-11 00:16:32,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +18: [2023-05-11 00:16:32,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +29: [2023-05-11 00:16:32,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +29: [2023-05-11 00:16:32,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +20: [2023-05-11 00:16:32,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +29: [2023-05-11 00:16:32,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +18: [2023-05-11 00:16:32,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +20: [2023-05-11 00:16:32,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 6: [2023-05-11 00:16:32,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +13: [2023-05-11 00:16:32,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +13: [2023-05-11 00:16:32,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +13: [2023-05-11 00:16:32,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +16: [2023-05-11 00:16:32,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +16: [2023-05-11 00:16:32,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +16: [2023-05-11 00:16:32,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +16: [2023-05-11 00:16:32,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +13: [2023-05-11 00:16:32,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +29: [2023-05-11 00:16:32,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +18: [2023-05-11 00:16:32,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +18: [2023-05-11 00:16:32,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +18: [2023-05-11 00:16:32,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +18: [2023-05-11 00:16:32,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +13: [2023-05-11 00:16:32,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +16: [2023-05-11 00:16:32,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +19: [2023-05-11 00:16:32,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +19: [2023-05-11 00:16:32,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +19: [2023-05-11 00:16:32,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +19: [2023-05-11 00:16:32,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +29: [2023-05-11 00:16:32,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +16: [2023-05-11 00:16:32,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +29: [2023-05-11 00:16:32,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +13: [2023-05-11 00:16:32,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +13: [2023-05-11 00:16:32,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +20: [2023-05-11 00:16:32,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +20: [2023-05-11 00:16:32,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +16: [2023-05-11 00:16:32,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +16: [2023-05-11 00:16:32,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +13: [2023-05-11 00:16:32,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +20: [2023-05-11 00:16:32,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +20: [2023-05-11 00:16:32,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +29: [2023-05-11 00:16:32,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 8: [2023-05-11 00:16:32,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 8: [2023-05-11 00:16:32,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 8: [2023-05-11 00:16:32,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 8: [2023-05-11 00:16:32,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +19: [2023-05-11 00:16:32,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +19: [2023-05-11 00:16:32,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +20: [2023-05-11 00:16:32,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +19: [2023-05-11 00:16:32,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +19: [2023-05-11 00:16:32,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +20: [2023-05-11 00:16:32,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +20: [2023-05-11 00:16:32,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +20: [2023-05-11 00:16:32,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +17: [2023-05-11 00:16:32,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +17: [2023-05-11 00:16:32,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +17: [2023-05-11 00:16:32,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +17: [2023-05-11 00:16:32,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 6: [2023-05-11 00:16:32,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +10: [2023-05-11 00:16:32,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +10: [2023-05-11 00:16:32,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +10: [2023-05-11 00:16:32,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +10: [2023-05-11 00:16:32,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 8: [2023-05-11 00:16:32,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 7: [2023-05-11 00:16:32,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 7: [2023-05-11 00:16:32,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 7: [2023-05-11 00:16:32,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +17: [2023-05-11 00:16:32,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +17: [2023-05-11 00:16:32,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +17: [2023-05-11 00:16:32,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +15: [2023-05-11 00:16:32,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +15: [2023-05-11 00:16:32,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +15: [2023-05-11 00:16:32,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +15: [2023-05-11 00:16:32,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +11: [2023-05-11 00:16:32,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +27: [2023-05-11 00:16:32,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +10: [2023-05-11 00:16:32,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +11: [2023-05-11 00:16:32,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +10: [2023-05-11 00:16:32,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +27: [2023-05-11 00:16:32,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +27: [2023-05-11 00:16:32,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +27: [2023-05-11 00:16:32,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +11: [2023-05-11 00:16:32,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +27: [2023-05-11 00:16:32,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +11: [2023-05-11 00:16:32,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +27: [2023-05-11 00:16:32,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +27: [2023-05-11 00:16:32,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +27: [2023-05-11 00:16:32,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +27: [2023-05-11 00:16:32,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +27: [2023-05-11 00:16:32,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +27: [2023-05-11 00:16:32,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +27: [2023-05-11 00:16:32,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +27: [2023-05-11 00:16:32,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +27: [2023-05-11 00:16:32,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +10: [2023-05-11 00:16:32,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +17: [2023-05-11 00:16:32,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +12: [2023-05-11 00:16:32,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +12: [2023-05-11 00:16:32,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +12: [2023-05-11 00:16:32,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +27: [2023-05-11 00:16:32,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +12: [2023-05-11 00:16:32,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +27: [2023-05-11 00:16:32,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +10: [2023-05-11 00:16:32,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +15: [2023-05-11 00:16:32,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +11: [2023-05-11 00:16:32,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +22: [2023-05-11 00:16:32,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +22: [2023-05-11 00:16:32,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +11: [2023-05-11 00:16:32,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +18: [2023-05-11 00:16:32,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +11: [2023-05-11 00:16:32,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +22: [2023-05-11 00:16:32,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +15: [2023-05-11 00:16:32,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +12: [2023-05-11 00:16:32,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +22: [2023-05-11 00:16:32,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +18: [2023-05-11 00:16:32,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +15: [2023-05-11 00:16:32,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +11: [2023-05-11 00:16:32,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +28: [2023-05-11 00:16:32,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +28: [2023-05-11 00:16:32,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +28: [2023-05-11 00:16:32,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +28: [2023-05-11 00:16:32,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +12: [2023-05-11 00:16:32,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +18: [2023-05-11 00:16:32,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +15: [2023-05-11 00:16:32,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +18: [2023-05-11 00:16:32,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +27: [2023-05-11 00:16:32,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +13: [2023-05-11 00:16:32,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +13: [2023-05-11 00:16:32,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +13: [2023-05-11 00:16:32,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +13: [2023-05-11 00:16:32,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +13: [2023-05-11 00:16:32,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +13: [2023-05-11 00:16:32,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +13: [2023-05-11 00:16:32,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +13: [2023-05-11 00:16:32,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +13: [2023-05-11 00:16:32,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +13: [2023-05-11 00:16:32,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +13: [2023-05-11 00:16:32,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +27: [2023-05-11 00:16:32,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +13: [2023-05-11 00:16:32,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +13: [2023-05-11 00:16:32,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +13: [2023-05-11 00:16:32,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 8: [2023-05-11 00:16:32,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +12: [2023-05-11 00:16:32,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +12: [2023-05-11 00:16:32,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +13: [2023-05-11 00:16:32,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +13: [2023-05-11 00:16:32,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +17: [2023-05-11 00:16:32,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +17: [2023-05-11 00:16:32,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +17: [2023-05-11 00:16:32,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +17: [2023-05-11 00:16:32,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +17: [2023-05-11 00:16:32,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +17: [2023-05-11 00:16:32,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +17: [2023-05-11 00:16:32,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +17: [2023-05-11 00:16:32,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +27: [2023-05-11 00:16:32,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +17: [2023-05-11 00:16:32,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +17: [2023-05-11 00:16:32,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +17: [2023-05-11 00:16:32,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +17: [2023-05-11 00:16:32,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +17: [2023-05-11 00:16:32,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +17: [2023-05-11 00:16:32,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +18: [2023-05-11 00:16:32,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 7: [2023-05-11 00:16:32,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +28: [2023-05-11 00:16:32,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +22: [2023-05-11 00:16:32,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +22: [2023-05-11 00:16:32,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +22: [2023-05-11 00:16:32,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +28: [2023-05-11 00:16:32,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +14: [2023-05-11 00:16:32,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +14: [2023-05-11 00:16:32,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +14: [2023-05-11 00:16:32,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +18: [2023-05-11 00:16:32,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +18: [2023-05-11 00:16:32,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +28: [2023-05-11 00:16:32,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 4: [2023-05-11 00:16:32,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 9: [2023-05-11 00:16:32,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +18: [2023-05-11 00:16:32,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +26: [2023-05-11 00:16:32,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +22: [2023-05-11 00:16:32,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +27: [2023-05-11 00:16:32,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +28: [2023-05-11 00:16:32,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +26: [2023-05-11 00:16:32,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +14: [2023-05-11 00:16:32,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +17: [2023-05-11 00:16:32,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 0: [2023-05-11 00:16:32,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 0: [2023-05-11 00:16:32,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +27: [2023-05-11 00:16:32,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +27: [2023-05-11 00:16:32,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +13: [2023-05-11 00:16:32,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 2: [2023-05-11 00:16:32,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +17: [2023-05-11 00:16:32,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +13: [2023-05-11 00:16:32,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 1: [2023-05-11 00:16:32,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +26: [2023-05-11 00:16:32,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 2: [2023-05-11 00:16:32,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +26: [2023-05-11 00:16:32,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 2: [2023-05-11 00:16:32,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +10: [2023-05-11 00:16:32,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +10: [2023-05-11 00:16:32,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +21: [2023-05-11 00:16:32,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +10: [2023-05-11 00:16:32,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +10: [2023-05-11 00:16:32,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +10: [2023-05-11 00:16:32,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +10: [2023-05-11 00:16:32,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +13: [2023-05-11 00:16:32,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +10: [2023-05-11 00:16:32,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +10: [2023-05-11 00:16:32,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +10: [2023-05-11 00:16:32,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +10: [2023-05-11 00:16:32,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +10: [2023-05-11 00:16:32,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +10: [2023-05-11 00:16:32,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +10: [2023-05-11 00:16:32,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +10: [2023-05-11 00:16:32,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +14: [2023-05-11 00:16:32,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +26: [2023-05-11 00:16:32,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +14: [2023-05-11 00:16:32,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +14: [2023-05-11 00:16:32,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +21: [2023-05-11 00:16:32,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +10: [2023-05-11 00:16:32,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +27: [2023-05-11 00:16:32,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +10: [2023-05-11 00:16:32,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +17: [2023-05-11 00:16:32,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +18: [2023-05-11 00:16:32,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +17: [2023-05-11 00:16:32,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +17: [2023-05-11 00:16:32,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +21: [2023-05-11 00:16:32,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +22: [2023-05-11 00:16:32,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +22: [2023-05-11 00:16:32,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +21: [2023-05-11 00:16:32,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +17: [2023-05-11 00:16:32,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +26: [2023-05-11 00:16:32,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 2: [2023-05-11 00:16:32,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +18: [2023-05-11 00:16:32,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 9: [2023-05-11 00:16:32,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +12: [2023-05-11 00:16:32,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +12: [2023-05-11 00:16:32,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +12: [2023-05-11 00:16:32,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +12: [2023-05-11 00:16:32,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +12: [2023-05-11 00:16:32,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +12: [2023-05-11 00:16:32,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +12: [2023-05-11 00:16:32,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +12: [2023-05-11 00:16:32,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +12: [2023-05-11 00:16:32,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +12: [2023-05-11 00:16:32,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +12: [2023-05-11 00:16:32,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +23: [2023-05-11 00:16:32,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +23: [2023-05-11 00:16:32,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +23: [2023-05-11 00:16:32,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 9: [2023-05-11 00:16:32,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +14: [2023-05-11 00:16:32,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +18: [2023-05-11 00:16:32,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +18: [2023-05-11 00:16:32,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +18: [2023-05-11 00:16:32,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +18: [2023-05-11 00:16:32,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +18: [2023-05-11 00:16:32,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 9: [2023-05-11 00:16:32,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +13: [2023-05-11 00:16:32,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +18: [2023-05-11 00:16:32,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +12: [2023-05-11 00:16:32,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +22: [2023-05-11 00:16:32,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +22: [2023-05-11 00:16:32,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 4: [2023-05-11 00:16:32,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +23: [2023-05-11 00:16:32,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 2: [2023-05-11 00:16:32,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 4: [2023-05-11 00:16:32,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 4: [2023-05-11 00:16:32,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +21: [2023-05-11 00:16:32,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +13: [2023-05-11 00:16:32,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +13: [2023-05-11 00:16:32,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +26: [2023-05-11 00:16:32,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +26: [2023-05-11 00:16:32,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +27: [2023-05-11 00:16:32,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +21: [2023-05-11 00:16:32,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +17: [2023-05-11 00:16:32,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +21: [2023-05-11 00:16:32,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +21: [2023-05-11 00:16:32,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +12: [2023-05-11 00:16:32,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +12: [2023-05-11 00:16:32,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +17: [2023-05-11 00:16:32,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +10: [2023-05-11 00:16:32,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +17: [2023-05-11 00:16:32,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +17: [2023-05-11 00:16:32,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +23: [2023-05-11 00:16:32,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +10: [2023-05-11 00:16:32,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +12: [2023-05-11 00:16:32,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +12: [2023-05-11 00:16:32,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +13: [2023-05-11 00:16:32,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +23: [2023-05-11 00:16:32,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +22: [2023-05-11 00:16:32,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +22: [2023-05-11 00:16:32,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +10: [2023-05-11 00:16:32,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +23: [2023-05-11 00:16:32,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +22: [2023-05-11 00:16:32,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 2: [2023-05-11 00:16:32,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +23: [2023-05-11 00:16:32,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +22: [2023-05-11 00:16:32,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 4: [2023-05-11 00:16:32,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +16: [2023-05-11 00:16:32,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +16: [2023-05-11 00:16:32,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +24: [2023-05-11 00:16:32,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +24: [2023-05-11 00:16:32,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +24: [2023-05-11 00:16:32,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +24: [2023-05-11 00:16:32,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +10: [2023-05-11 00:16:32,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +12: [2023-05-11 00:16:32,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +12: [2023-05-11 00:16:32,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +12: [2023-05-11 00:16:32,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +13: [2023-05-11 00:16:32,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +16: [2023-05-11 00:16:32,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +16: [2023-05-11 00:16:32,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +10: [2023-05-11 00:16:32,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +10: [2023-05-11 00:16:32,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +25: [2023-05-11 00:16:32,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +25: [2023-05-11 00:16:32,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +25: [2023-05-11 00:16:32,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +25: [2023-05-11 00:16:32,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +25: [2023-05-11 00:16:32,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +25: [2023-05-11 00:16:32,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +25: [2023-05-11 00:16:32,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +10: [2023-05-11 00:16:32,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +25: [2023-05-11 00:16:32,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +16: [2023-05-11 00:16:32,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +24: [2023-05-11 00:16:32,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +16: [2023-05-11 00:16:32,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +12: [2023-05-11 00:16:32,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +12: [2023-05-11 00:16:32,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +10: [2023-05-11 00:16:32,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +24: [2023-05-11 00:16:32,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +24: [2023-05-11 00:16:32,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +16: [2023-05-11 00:16:32,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +16: [2023-05-11 00:16:32,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +14: [2023-05-11 00:16:32,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +14: [2023-05-11 00:16:32,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +14: [2023-05-11 00:16:32,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +14: [2023-05-11 00:16:32,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +23: [2023-05-11 00:16:32,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +23: [2023-05-11 00:16:32,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +23: [2023-05-11 00:16:32,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +14: [2023-05-11 00:16:32,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +14: [2023-05-11 00:16:32,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +14: [2023-05-11 00:16:32,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +14: [2023-05-11 00:16:32,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +14: [2023-05-11 00:16:32,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +14: [2023-05-11 00:16:32,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +14: [2023-05-11 00:16:32,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +24: [2023-05-11 00:16:32,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +14: [2023-05-11 00:16:32,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +14: [2023-05-11 00:16:32,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +14: [2023-05-11 00:16:32,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +14: [2023-05-11 00:16:32,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +23: [2023-05-11 00:16:32,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +23: [2023-05-11 00:16:32,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +23: [2023-05-11 00:16:32,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +14: [2023-05-11 00:16:32,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +23: [2023-05-11 00:16:32,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +23: [2023-05-11 00:16:32,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +23: [2023-05-11 00:16:32,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +23: [2023-05-11 00:16:32,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +12: [2023-05-11 00:16:32,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +23: [2023-05-11 00:16:32,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +23: [2023-05-11 00:16:32,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +15: [2023-05-11 00:16:32,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +15: [2023-05-11 00:16:32,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +15: [2023-05-11 00:16:32,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +15: [2023-05-11 00:16:32,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +15: [2023-05-11 00:16:32,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +15: [2023-05-11 00:16:32,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +15: [2023-05-11 00:16:32,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +15: [2023-05-11 00:16:32,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +12: [2023-05-11 00:16:32,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +15: [2023-05-11 00:16:32,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 1: [2023-05-11 00:16:32,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 4: [2023-05-11 00:16:32,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +15: [2023-05-11 00:16:32,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +15: [2023-05-11 00:16:32,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +15: [2023-05-11 00:16:32,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +15: [2023-05-11 00:16:32,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +15: [2023-05-11 00:16:32,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +15: [2023-05-11 00:16:32,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +30: [2023-05-11 00:16:32,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +30: [2023-05-11 00:16:32,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +30: [2023-05-11 00:16:32,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +30: [2023-05-11 00:16:32,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +15: [2023-05-11 00:16:32,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +23: [2023-05-11 00:16:32,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +23: [2023-05-11 00:16:32,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +25: [2023-05-11 00:16:32,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +26: [2023-05-11 00:16:32,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +26: [2023-05-11 00:16:32,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +26: [2023-05-11 00:16:32,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +25: [2023-05-11 00:16:32,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +26: [2023-05-11 00:16:32,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +26: [2023-05-11 00:16:32,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +26: [2023-05-11 00:16:32,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +26: [2023-05-11 00:16:32,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +26: [2023-05-11 00:16:32,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +26: [2023-05-11 00:16:32,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +26: [2023-05-11 00:16:32,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +26: [2023-05-11 00:16:32,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +26: [2023-05-11 00:16:32,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +26: [2023-05-11 00:16:32,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +26: [2023-05-11 00:16:32,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +26: [2023-05-11 00:16:32,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +26: [2023-05-11 00:16:32,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 3: [2023-05-11 00:16:32,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 3: [2023-05-11 00:16:32,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 3: [2023-05-11 00:16:32,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 3: [2023-05-11 00:16:32,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. + 1: [2023-05-11 00:16:32,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 1: [2023-05-11 00:16:32,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 1: [2023-05-11 00:16:32,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +23: [2023-05-11 00:16:32,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +25: [2023-05-11 00:16:32,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +12: [2023-05-11 00:16:32,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +16: [2023-05-11 00:16:32,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +16: [2023-05-11 00:16:32,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +14: [2023-05-11 00:16:32,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +23: [2023-05-11 00:16:32,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +21: [2023-05-11 00:16:32,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +21: [2023-05-11 00:16:32,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +21: [2023-05-11 00:16:32,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +21: [2023-05-11 00:16:32,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +21: [2023-05-11 00:16:32,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +21: [2023-05-11 00:16:32,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +21: [2023-05-11 00:16:32,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +21: [2023-05-11 00:16:32,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +21: [2023-05-11 00:16:32,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +21: [2023-05-11 00:16:32,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 2: [2023-05-11 00:16:32,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +21: [2023-05-11 00:16:32,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +21: [2023-05-11 00:16:32,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +21: [2023-05-11 00:16:32,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +21: [2023-05-11 00:16:32,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +31: [2023-05-11 00:16:32,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +31: [2023-05-11 00:16:32,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +31: [2023-05-11 00:16:32,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +25: [2023-05-11 00:16:32,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +16: [2023-05-11 00:16:32,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +19: [2023-05-11 00:16:32,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +19: [2023-05-11 00:16:32,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +19: [2023-05-11 00:16:32,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +16: [2023-05-11 00:16:32,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +19: [2023-05-11 00:16:32,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +19: [2023-05-11 00:16:32,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +19: [2023-05-11 00:16:32,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +19: [2023-05-11 00:16:32,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +21: [2023-05-11 00:16:32,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 5: [2023-05-11 00:16:32,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 5: [2023-05-11 00:16:32,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 5: [2023-05-11 00:16:32,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +30: [2023-05-11 00:16:32,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +19: [2023-05-11 00:16:32,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +19: [2023-05-11 00:16:32,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +30: [2023-05-11 00:16:32,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +19: [2023-05-11 00:16:32,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +19: [2023-05-11 00:16:32,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +30: [2023-05-11 00:16:32,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +19: [2023-05-11 00:16:32,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +25: [2023-05-11 00:16:32,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +30: [2023-05-11 00:16:32,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +23: [2023-05-11 00:16:32,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +23: [2023-05-11 00:16:32,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +19: [2023-05-11 00:16:32,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +21: [2023-05-11 00:16:32,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +19: [2023-05-11 00:16:32,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +23: [2023-05-11 00:16:32,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +31: [2023-05-11 00:16:32,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt. +19: [2023-05-11 00:16:32,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +19: [2023-05-11 00:16:32,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 2: [2023-05-11 00:16:32,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 2: [2023-05-11 00:16:32,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 2: [2023-05-11 00:16:32,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 2: [2023-05-11 00:16:32,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 4: [2023-05-11 00:16:32,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +16: [2023-05-11 00:16:32,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +20: [2023-05-11 00:16:32,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +20: [2023-05-11 00:16:32,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +16: [2023-05-11 00:16:32,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 1: [2023-05-11 00:16:32,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +14: [2023-05-11 00:16:32,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +23: [2023-05-11 00:16:32,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +25: [2023-05-11 00:16:32,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +14: [2023-05-11 00:16:32,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +14: [2023-05-11 00:16:32,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +16: [2023-05-11 00:16:32,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +14: [2023-05-11 00:16:32,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +16: [2023-05-11 00:16:32,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +15: [2023-05-11 00:16:32,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +21: [2023-05-11 00:16:32,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +31: [2023-05-11 00:16:32,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +25: [2023-05-11 00:16:32,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +20: [2023-05-11 00:16:32,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +15: [2023-05-11 00:16:32,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +31: [2023-05-11 00:16:32,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +31: [2023-05-11 00:16:32,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +23: [2023-05-11 00:16:32,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +23: [2023-05-11 00:16:32,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +31: [2023-05-11 00:16:32,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +15: [2023-05-11 00:16:32,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +20: [2023-05-11 00:16:32,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 9: [2023-05-11 00:16:32,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 9: [2023-05-11 00:16:32,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 4: [2023-05-11 00:16:32,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +23: [2023-05-11 00:16:32,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +20: [2023-05-11 00:16:32,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +20: [2023-05-11 00:16:32,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 9: [2023-05-11 00:16:32,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +26: [2023-05-11 00:16:32,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +26: [2023-05-11 00:16:32,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +26: [2023-05-11 00:16:32,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +21: [2023-05-11 00:16:32,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +19: [2023-05-11 00:16:32,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +21: [2023-05-11 00:16:32,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +23: [2023-05-11 00:16:32,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 5: [2023-05-11 00:16:32,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +26: [2023-05-11 00:16:32,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +21: [2023-05-11 00:16:32,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +14: [2023-05-11 00:16:32,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +15: [2023-05-11 00:16:32,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +19: [2023-05-11 00:16:32,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +20: [2023-05-11 00:16:32,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +15: [2023-05-11 00:16:32,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +15: [2023-05-11 00:16:32,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +14: [2023-05-11 00:16:32,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +25: [2023-05-11 00:16:32,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +14: [2023-05-11 00:16:32,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +15: [2023-05-11 00:16:32,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 6: [2023-05-11 00:16:32,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 3: [2023-05-11 00:16:32,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +20: [2023-05-11 00:16:32,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +26: [2023-05-11 00:16:32,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +19: [2023-05-11 00:16:32,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +19: [2023-05-11 00:16:32,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 3: [2023-05-11 00:16:32,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 3: [2023-05-11 00:16:32,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 6: [2023-05-11 00:16:32,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +26: [2023-05-11 00:16:32,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +21: [2023-05-11 00:16:32,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +19: [2023-05-11 00:16:32,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +21: [2023-05-11 00:16:32,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +15: [2023-05-11 00:16:32,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +19: [2023-05-11 00:16:32,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +26: [2023-05-11 00:16:32,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +26: [2023-05-11 00:16:32,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 6: [2023-05-11 00:16:32,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 6: [2023-05-11 00:16:32,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +19: [2023-05-11 00:16:32,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +19: [2023-05-11 00:16:32,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +21: [2023-05-11 00:16:32,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +11: [2023-05-11 00:16:32,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +11: [2023-05-11 00:16:32,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +11: [2023-05-11 00:16:32,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +11: [2023-05-11 00:16:32,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +11: [2023-05-11 00:16:32,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +11: [2023-05-11 00:16:32,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +11: [2023-05-11 00:16:32,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +11: [2023-05-11 00:16:32,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +11: [2023-05-11 00:16:32,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +11: [2023-05-11 00:16:32,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +11: [2023-05-11 00:16:32,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +11: [2023-05-11 00:16:32,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +11: [2023-05-11 00:16:32,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +11: [2023-05-11 00:16:32,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +11: [2023-05-11 00:16:32,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +11: [2023-05-11 00:16:32,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 9: [2023-05-11 00:16:32,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +31: [2023-05-11 00:16:32,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +31: [2023-05-11 00:16:32,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +31: [2023-05-11 00:16:32,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +31: [2023-05-11 00:16:32,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 0: [2023-05-11 00:16:32,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 0: [2023-05-11 00:16:32,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +31: [2023-05-11 00:16:32,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 0: [2023-05-11 00:16:32,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +31: [2023-05-11 00:16:32,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +31: [2023-05-11 00:16:32,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +31: [2023-05-11 00:16:32,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +31: [2023-05-11 00:16:32,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +31: [2023-05-11 00:16:32,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +31: [2023-05-11 00:16:32,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... +31: [2023-05-11 00:16:32,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +31: [2023-05-11 00:16:32,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +31: [2023-05-11 00:16:32,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +31: [2023-05-11 00:16:32,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +31: [2023-05-11 00:16:32,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt... + 0: [2023-05-11 00:16:32,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt... +22: [2023-05-11 00:16:32,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +22: [2023-05-11 00:16:32,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +22: [2023-05-11 00:16:32,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +22: [2023-05-11 00:16:32,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +22: [2023-05-11 00:16:32,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +22: [2023-05-11 00:16:32,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +22: [2023-05-11 00:16:32,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +22: [2023-05-11 00:16:32,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +21: [2023-05-11 00:16:32,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +11: [2023-05-11 00:16:32,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +11: [2023-05-11 00:16:32,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +11: [2023-05-11 00:16:32,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +11: [2023-05-11 00:16:32,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +31: [2023-05-11 00:16:32,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +22: [2023-05-11 00:16:32,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +31: [2023-05-11 00:16:32,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +31: [2023-05-11 00:16:32,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +31: [2023-05-11 00:16:32,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +11: [2023-05-11 00:16:32,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +28: [2023-05-11 00:16:32,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +28: [2023-05-11 00:16:32,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +28: [2023-05-11 00:16:32,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +28: [2023-05-11 00:16:32,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +28: [2023-05-11 00:16:32,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +28: [2023-05-11 00:16:32,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +28: [2023-05-11 00:16:32,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +28: [2023-05-11 00:16:32,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +28: [2023-05-11 00:16:32,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +28: [2023-05-11 00:16:32,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +28: [2023-05-11 00:16:32,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +28: [2023-05-11 00:16:32,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +28: [2023-05-11 00:16:32,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +11: [2023-05-11 00:16:32,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +22: [2023-05-11 00:16:32,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +22: [2023-05-11 00:16:32,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +28: [2023-05-11 00:16:32,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +28: [2023-05-11 00:16:32,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +24: [2023-05-11 00:16:32,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +24: [2023-05-11 00:16:32,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +24: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +24: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +28: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +30: [2023-05-11 00:16:32,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +30: [2023-05-11 00:16:32,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +30: [2023-05-11 00:16:32,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +30: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +30: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +24: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +24: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +24: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +30: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +30: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +24: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +24: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +30: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +24: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +30: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +24: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +24: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +24: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +24: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +30: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +30: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +30: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +30: [2023-05-11 00:16:32,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +11: [2023-05-11 00:16:32,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +30: [2023-05-11 00:16:32,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +24: [2023-05-11 00:16:32,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +30: [2023-05-11 00:16:32,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +30: [2023-05-11 00:16:32,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +24: [2023-05-11 00:16:32,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +11: [2023-05-11 00:16:32,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +22: [2023-05-11 00:16:32,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +31: [2023-05-11 00:16:32,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +31: [2023-05-11 00:16:32,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +22: [2023-05-11 00:16:32,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +22: [2023-05-11 00:16:32,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +22: [2023-05-11 00:16:32,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +28: [2023-05-11 00:16:32,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +31: [2023-05-11 00:16:32,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +31: [2023-05-11 00:16:32,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +28: [2023-05-11 00:16:32,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +28: [2023-05-11 00:16:32,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +30: [2023-05-11 00:16:32,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +30: [2023-05-11 00:16:32,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +22: [2023-05-11 00:16:32,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +24: [2023-05-11 00:16:32,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +24: [2023-05-11 00:16:32,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +24: [2023-05-11 00:16:32,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +24: [2023-05-11 00:16:32,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +28: [2023-05-11 00:16:32,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt. +28: [2023-05-11 00:16:32,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +30: [2023-05-11 00:16:32,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +28: [2023-05-11 00:16:32,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 8: [2023-05-11 00:16:32,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 8: [2023-05-11 00:16:32,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 8: [2023-05-11 00:16:32,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +30: [2023-05-11 00:16:32,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +30: [2023-05-11 00:16:32,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +28: [2023-05-11 00:16:32,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +30: [2023-05-11 00:16:32,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +24: [2023-05-11 00:16:32,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +24: [2023-05-11 00:16:32,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +24: [2023-05-11 00:16:32,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +24: [2023-05-11 00:16:32,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +30: [2023-05-11 00:16:32,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +28: [2023-05-11 00:16:32,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +29: [2023-05-11 00:16:32,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +29: [2023-05-11 00:16:32,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +29: [2023-05-11 00:16:32,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +29: [2023-05-11 00:16:32,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 5: [2023-05-11 00:16:32,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +29: [2023-05-11 00:16:32,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +29: [2023-05-11 00:16:32,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 5: [2023-05-11 00:16:32,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 5: [2023-05-11 00:16:32,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +29: [2023-05-11 00:16:32,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +30: [2023-05-11 00:16:32,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +29: [2023-05-11 00:16:32,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +25: [2023-05-11 00:16:32,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +25: [2023-05-11 00:16:32,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +25: [2023-05-11 00:16:32,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +27: [2023-05-11 00:16:32,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +27: [2023-05-11 00:16:32,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +27: [2023-05-11 00:16:32,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +27: [2023-05-11 00:16:32,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +29: [2023-05-11 00:16:32,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +25: [2023-05-11 00:16:32,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 5: [2023-05-11 00:16:32,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +29: [2023-05-11 00:16:32,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +29: [2023-05-11 00:16:32,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +29: [2023-05-11 00:16:32,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +29: [2023-05-11 00:16:32,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +29: [2023-05-11 00:16:32,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +22: [2023-05-11 00:16:32,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +22: [2023-05-11 00:16:32,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +16: [2023-05-11 00:16:32,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +16: [2023-05-11 00:16:32,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +16: [2023-05-11 00:16:32,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +16: [2023-05-11 00:16:32,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +22: [2023-05-11 00:16:32,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 5: [2023-05-11 00:16:32,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +22: [2023-05-11 00:16:32,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +25: [2023-05-11 00:16:32,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +25: [2023-05-11 00:16:32,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +25: [2023-05-11 00:16:32,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +16: [2023-05-11 00:16:32,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +16: [2023-05-11 00:16:32,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +16: [2023-05-11 00:16:32,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +25: [2023-05-11 00:16:32,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +16: [2023-05-11 00:16:32,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +27: [2023-05-11 00:16:32,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +27: [2023-05-11 00:16:32,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +29: [2023-05-11 00:16:32,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +27: [2023-05-11 00:16:32,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +27: [2023-05-11 00:16:32,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +29: [2023-05-11 00:16:32,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +29: [2023-05-11 00:16:32,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +22: [2023-05-11 00:16:32,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +29: [2023-05-11 00:16:32,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +22: [2023-05-11 00:16:32,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +22: [2023-05-11 00:16:32,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +16: [2023-05-11 00:16:32,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +29: [2023-05-11 00:16:32,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +29: [2023-05-11 00:16:32,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +29: [2023-05-11 00:16:32,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +22: [2023-05-11 00:16:32,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +25: [2023-05-11 00:16:32,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +16: [2023-05-11 00:16:32,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +25: [2023-05-11 00:16:32,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +25: [2023-05-11 00:16:32,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +25: [2023-05-11 00:16:32,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +25: [2023-05-11 00:16:32,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +25: [2023-05-11 00:16:32,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +22: [2023-05-11 00:16:32,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +22: [2023-05-11 00:16:32,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +29: [2023-05-11 00:16:32,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +29: [2023-05-11 00:16:32,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +16: [2023-05-11 00:16:32,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +16: [2023-05-11 00:16:32,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +29: [2023-05-11 00:16:32,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +29: [2023-05-11 00:16:32,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +25: [2023-05-11 00:16:32,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +29: [2023-05-11 00:16:32,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +29: [2023-05-11 00:16:32,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +22: [2023-05-11 00:16:32,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +22: [2023-05-11 00:16:32,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +29: [2023-05-11 00:16:32,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +29: [2023-05-11 00:16:32,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +25: [2023-05-11 00:16:32,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +22: [2023-05-11 00:16:32,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +22: [2023-05-11 00:16:32,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +12: [2023-05-11 00:16:32,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +12: [2023-05-11 00:16:32,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +12: [2023-05-11 00:16:32,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +29: [2023-05-11 00:16:32,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +29: [2023-05-11 00:16:32,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +12: [2023-05-11 00:16:32,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +16: [2023-05-11 00:16:32,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +16: [2023-05-11 00:16:32,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +29: [2023-05-11 00:16:32,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +13: [2023-05-11 00:16:32,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +13: [2023-05-11 00:16:32,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +13: [2023-05-11 00:16:32,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +13: [2023-05-11 00:16:32,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +22: [2023-05-11 00:16:32,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +13: [2023-05-11 00:16:32,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +13: [2023-05-11 00:16:32,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +13: [2023-05-11 00:16:32,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +22: [2023-05-11 00:16:32,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +13: [2023-05-11 00:16:32,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +12: [2023-05-11 00:16:32,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +13: [2023-05-11 00:16:32,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +16: [2023-05-11 00:16:32,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +16: [2023-05-11 00:16:32,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +13: [2023-05-11 00:16:32,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +12: [2023-05-11 00:16:32,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +12: [2023-05-11 00:16:32,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +12: [2023-05-11 00:16:32,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +13: [2023-05-11 00:16:32,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +13: [2023-05-11 00:16:32,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +13: [2023-05-11 00:16:32,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +13: [2023-05-11 00:16:32,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +13: [2023-05-11 00:16:32,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +13: [2023-05-11 00:16:32,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +13: [2023-05-11 00:16:32,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +13: [2023-05-11 00:16:32,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 6: [2023-05-11 00:16:32,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 6: [2023-05-11 00:16:32,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 6: [2023-05-11 00:16:32,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +13: [2023-05-11 00:16:32,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +13: [2023-05-11 00:16:32,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 6: [2023-05-11 00:16:32,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +13: [2023-05-11 00:16:32,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +13: [2023-05-11 00:16:32,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +13: [2023-05-11 00:16:32,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +23: [2023-05-11 00:16:32,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +23: [2023-05-11 00:16:32,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +23: [2023-05-11 00:16:32,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +23: [2023-05-11 00:16:32,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +18: [2023-05-11 00:16:32,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +18: [2023-05-11 00:16:32,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +17: [2023-05-11 00:16:32,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +18: [2023-05-11 00:16:32,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +17: [2023-05-11 00:16:32,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +17: [2023-05-11 00:16:32,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +18: [2023-05-11 00:16:32,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +17: [2023-05-11 00:16:32,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +20: [2023-05-11 00:16:32,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +20: [2023-05-11 00:16:32,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +20: [2023-05-11 00:16:32,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +26: [2023-05-11 00:16:32,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +26: [2023-05-11 00:16:32,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +26: [2023-05-11 00:16:32,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +20: [2023-05-11 00:16:32,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +14: [2023-05-11 00:16:32,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +26: [2023-05-11 00:16:32,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +14: [2023-05-11 00:16:32,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +14: [2023-05-11 00:16:32,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +14: [2023-05-11 00:16:32,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +13: [2023-05-11 00:16:32,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +13: [2023-05-11 00:16:32,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 1: [2023-05-11 00:16:32,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +13: [2023-05-11 00:16:32,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +20: [2023-05-11 00:16:32,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +20: [2023-05-11 00:16:32,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +20: [2023-05-11 00:16:32,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +20: [2023-05-11 00:16:32,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +20: [2023-05-11 00:16:32,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +20: [2023-05-11 00:16:32,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +20: [2023-05-11 00:16:32,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +20: [2023-05-11 00:16:32,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +20: [2023-05-11 00:16:32,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 8: [2023-05-11 00:16:32,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 8: [2023-05-11 00:16:32,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +20: [2023-05-11 00:16:32,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +20: [2023-05-11 00:16:32,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 8: [2023-05-11 00:16:32,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 8: [2023-05-11 00:16:32,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +23: [2023-05-11 00:16:32,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +13: [2023-05-11 00:16:32,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +13: [2023-05-11 00:16:32,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +17: [2023-05-11 00:16:32,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +13: [2023-05-11 00:16:32,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +23: [2023-05-11 00:16:32,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +17: [2023-05-11 00:16:32,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +17: [2023-05-11 00:16:32,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +20: [2023-05-11 00:16:32,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 1: [2023-05-11 00:16:32,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +23: [2023-05-11 00:16:32,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +18: [2023-05-11 00:16:32,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 4: [2023-05-11 00:16:32,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 4: [2023-05-11 00:16:32,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 1: [2023-05-11 00:16:32,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +18: [2023-05-11 00:16:32,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 6: [2023-05-11 00:16:32,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +26: [2023-05-11 00:16:32,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +18: [2023-05-11 00:16:32,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +23: [2023-05-11 00:16:32,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +14: [2023-05-11 00:16:32,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +13: [2023-05-11 00:16:32,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +14: [2023-05-11 00:16:32,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +14: [2023-05-11 00:16:32,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +18: [2023-05-11 00:16:32,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +13: [2023-05-11 00:16:32,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 6: [2023-05-11 00:16:32,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +17: [2023-05-11 00:16:32,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +14: [2023-05-11 00:16:32,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +26: [2023-05-11 00:16:32,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +20: [2023-05-11 00:16:32,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +26: [2023-05-11 00:16:32,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +26: [2023-05-11 00:16:32,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +13: [2023-05-11 00:16:32,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +20: [2023-05-11 00:16:32,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +20: [2023-05-11 00:16:32,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +19: [2023-05-11 00:16:32,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +19: [2023-05-11 00:16:32,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +19: [2023-05-11 00:16:32,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +19: [2023-05-11 00:16:32,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +19: [2023-05-11 00:16:32,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +19: [2023-05-11 00:16:32,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +19: [2023-05-11 00:16:32,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +19: [2023-05-11 00:16:32,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +20: [2023-05-11 00:16:32,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +16: [2023-05-11 00:16:32,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 2: [2023-05-11 00:16:32,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +16: [2023-05-11 00:16:32,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +16: [2023-05-11 00:16:32,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +16: [2023-05-11 00:16:32,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 1: [2023-05-11 00:16:32,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 8: [2023-05-11 00:16:32,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +19: [2023-05-11 00:16:32,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +19: [2023-05-11 00:16:32,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +20: [2023-05-11 00:16:32,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +20: [2023-05-11 00:16:32,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +20: [2023-05-11 00:16:32,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 8: [2023-05-11 00:16:32,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +27: [2023-05-11 00:16:32,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +27: [2023-05-11 00:16:32,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +27: [2023-05-11 00:16:32,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +27: [2023-05-11 00:16:32,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +27: [2023-05-11 00:16:32,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +27: [2023-05-11 00:16:32,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +16: [2023-05-11 00:16:32,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +27: [2023-05-11 00:16:32,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +27: [2023-05-11 00:16:32,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +16: [2023-05-11 00:16:32,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +19: [2023-05-11 00:16:32,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +27: [2023-05-11 00:16:32,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +19: [2023-05-11 00:16:32,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +12: [2023-05-11 00:16:32,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +12: [2023-05-11 00:16:32,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +12: [2023-05-11 00:16:32,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +12: [2023-05-11 00:16:32,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +12: [2023-05-11 00:16:32,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +12: [2023-05-11 00:16:32,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +12: [2023-05-11 00:16:32,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +12: [2023-05-11 00:16:32,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +12: [2023-05-11 00:16:32,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +12: [2023-05-11 00:16:32,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +12: [2023-05-11 00:16:32,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +12: [2023-05-11 00:16:32,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +12: [2023-05-11 00:16:32,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +12: [2023-05-11 00:16:32,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +12: [2023-05-11 00:16:32,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +27: [2023-05-11 00:16:32,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 8: [2023-05-11 00:16:32,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +27: [2023-05-11 00:16:32,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +27: [2023-05-11 00:16:32,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +27: [2023-05-11 00:16:32,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +27: [2023-05-11 00:16:32,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +12: [2023-05-11 00:16:32,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +16: [2023-05-11 00:16:32,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +16: [2023-05-11 00:16:32,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +27: [2023-05-11 00:16:32,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +23: [2023-05-11 00:16:32,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +20: [2023-05-11 00:16:32,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +23: [2023-05-11 00:16:32,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +23: [2023-05-11 00:16:32,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +23: [2023-05-11 00:16:32,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +23: [2023-05-11 00:16:32,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +23: [2023-05-11 00:16:32,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +23: [2023-05-11 00:16:32,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +23: [2023-05-11 00:16:32,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +23: [2023-05-11 00:16:32,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +23: [2023-05-11 00:16:32,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +23: [2023-05-11 00:16:32,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +23: [2023-05-11 00:16:32,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +23: [2023-05-11 00:16:32,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +27: [2023-05-11 00:16:32,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +23: [2023-05-11 00:16:32,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +14: [2023-05-11 00:16:32,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +14: [2023-05-11 00:16:32,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +14: [2023-05-11 00:16:32,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +14: [2023-05-11 00:16:32,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +14: [2023-05-11 00:16:32,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +14: [2023-05-11 00:16:32,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +14: [2023-05-11 00:16:32,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +14: [2023-05-11 00:16:32,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +14: [2023-05-11 00:16:32,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +14: [2023-05-11 00:16:32,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +14: [2023-05-11 00:16:32,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +14: [2023-05-11 00:16:32,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 4: [2023-05-11 00:16:32,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +23: [2023-05-11 00:16:32,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +19: [2023-05-11 00:16:32,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +23: [2023-05-11 00:16:32,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +19: [2023-05-11 00:16:32,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +14: [2023-05-11 00:16:32,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +14: [2023-05-11 00:16:32,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +10: [2023-05-11 00:16:32,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 2: [2023-05-11 00:16:32,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 2: [2023-05-11 00:16:32,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 2: [2023-05-11 00:16:32,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +10: [2023-05-11 00:16:32,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 2: [2023-05-11 00:16:32,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 8: [2023-05-11 00:16:32,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +19: [2023-05-11 00:16:32,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +19: [2023-05-11 00:16:32,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +10: [2023-05-11 00:16:32,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 3: [2023-05-11 00:16:32,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 3: [2023-05-11 00:16:32,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 3: [2023-05-11 00:16:32,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +14: [2023-05-11 00:16:32,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +10: [2023-05-11 00:16:32,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +14: [2023-05-11 00:16:32,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +12: [2023-05-11 00:16:32,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +27: [2023-05-11 00:16:32,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +27: [2023-05-11 00:16:32,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +16: [2023-05-11 00:16:32,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +16: [2023-05-11 00:16:32,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +14: [2023-05-11 00:16:32,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +27: [2023-05-11 00:16:32,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +16: [2023-05-11 00:16:32,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +16: [2023-05-11 00:16:32,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 4: [2023-05-11 00:16:32,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +23: [2023-05-11 00:16:32,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +12: [2023-05-11 00:16:32,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +12: [2023-05-11 00:16:32,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +23: [2023-05-11 00:16:32,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +23: [2023-05-11 00:16:32,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 0: [2023-05-11 00:16:32,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 0: [2023-05-11 00:16:32,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 0: [2023-05-11 00:16:32,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 2: [2023-05-11 00:16:32,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 3: [2023-05-11 00:16:32,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +19: [2023-05-11 00:16:32,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +19: [2023-05-11 00:16:32,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 3: [2023-05-11 00:16:32,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +19: [2023-05-11 00:16:32,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 7: [2023-05-11 00:16:32,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 7: [2023-05-11 00:16:32,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 2: [2023-05-11 00:16:32,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +19: [2023-05-11 00:16:32,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +10: [2023-05-11 00:16:32,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +21: [2023-05-11 00:16:32,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +21: [2023-05-11 00:16:32,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +21: [2023-05-11 00:16:32,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +21: [2023-05-11 00:16:32,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +21: [2023-05-11 00:16:32,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +21: [2023-05-11 00:16:32,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +21: [2023-05-11 00:16:32,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +14: [2023-05-11 00:16:32,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +10: [2023-05-11 00:16:32,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +15: [2023-05-11 00:16:32,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +15: [2023-05-11 00:16:32,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +21: [2023-05-11 00:16:32,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +27: [2023-05-11 00:16:32,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +10: [2023-05-11 00:16:32,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +10: [2023-05-11 00:16:32,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +15: [2023-05-11 00:16:32,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 9: [2023-05-11 00:16:32,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +18: [2023-05-11 00:16:32,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +18: [2023-05-11 00:16:32,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +18: [2023-05-11 00:16:32,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +18: [2023-05-11 00:16:32,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +18: [2023-05-11 00:16:32,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +18: [2023-05-11 00:16:32,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +15: [2023-05-11 00:16:32,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 7: [2023-05-11 00:16:32,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +27: [2023-05-11 00:16:32,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 4: [2023-05-11 00:16:32,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +14: [2023-05-11 00:16:32,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +27: [2023-05-11 00:16:32,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +17: [2023-05-11 00:16:32,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +17: [2023-05-11 00:16:32,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +17: [2023-05-11 00:16:32,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +17: [2023-05-11 00:16:32,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +17: [2023-05-11 00:16:32,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +17: [2023-05-11 00:16:32,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +17: [2023-05-11 00:16:32,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +17: [2023-05-11 00:16:32,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +17: [2023-05-11 00:16:32,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +17: [2023-05-11 00:16:32,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +17: [2023-05-11 00:16:32,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +17: [2023-05-11 00:16:32,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +17: [2023-05-11 00:16:32,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +17: [2023-05-11 00:16:32,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +12: [2023-05-11 00:16:32,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +18: [2023-05-11 00:16:32,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +18: [2023-05-11 00:16:32,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +18: [2023-05-11 00:16:32,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +18: [2023-05-11 00:16:32,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +18: [2023-05-11 00:16:32,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +18: [2023-05-11 00:16:32,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +18: [2023-05-11 00:16:32,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +17: [2023-05-11 00:16:32,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +26: [2023-05-11 00:16:32,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +17: [2023-05-11 00:16:32,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +11: [2023-05-11 00:16:32,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +26: [2023-05-11 00:16:32,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +26: [2023-05-11 00:16:32,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +26: [2023-05-11 00:16:32,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +11: [2023-05-11 00:16:32,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +26: [2023-05-11 00:16:32,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +26: [2023-05-11 00:16:32,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +26: [2023-05-11 00:16:32,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +23: [2023-05-11 00:16:32,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +26: [2023-05-11 00:16:32,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +26: [2023-05-11 00:16:32,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +26: [2023-05-11 00:16:32,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +26: [2023-05-11 00:16:32,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +12: [2023-05-11 00:16:32,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +12: [2023-05-11 00:16:32,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +23: [2023-05-11 00:16:32,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +16: [2023-05-11 00:16:32,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +16: [2023-05-11 00:16:32,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +18: [2023-05-11 00:16:32,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +31: [2023-05-11 00:16:32,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +31: [2023-05-11 00:16:32,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +31: [2023-05-11 00:16:32,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 0: [2023-05-11 00:16:32,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +31: [2023-05-11 00:16:32,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +11: [2023-05-11 00:16:32,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. + 0: [2023-05-11 00:16:32,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +11: [2023-05-11 00:16:32,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt. +21: [2023-05-11 00:16:32,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 4: [2023-05-11 00:16:32,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +18: [2023-05-11 00:16:32,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +26: [2023-05-11 00:16:32,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +23: [2023-05-11 00:16:32,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +18: [2023-05-11 00:16:32,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +26: [2023-05-11 00:16:32,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +21: [2023-05-11 00:16:32,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +26: [2023-05-11 00:16:32,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +26: [2023-05-11 00:16:32,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +21: [2023-05-11 00:16:32,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +16: [2023-05-11 00:16:32,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +16: [2023-05-11 00:16:32,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +26: [2023-05-11 00:16:32,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +15: [2023-05-11 00:16:32,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +14: [2023-05-11 00:16:32,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +28: [2023-05-11 00:16:32,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 7: [2023-05-11 00:16:32,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +19: [2023-05-11 00:16:32,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +19: [2023-05-11 00:16:32,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +19: [2023-05-11 00:16:32,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +19: [2023-05-11 00:16:32,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +28: [2023-05-11 00:16:32,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +15: [2023-05-11 00:16:32,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 9: [2023-05-11 00:16:32,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 4: [2023-05-11 00:16:32,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 9: [2023-05-11 00:16:32,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 7: [2023-05-11 00:16:32,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +15: [2023-05-11 00:16:32,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +28: [2023-05-11 00:16:32,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +23: [2023-05-11 00:16:32,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +28: [2023-05-11 00:16:32,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +15: [2023-05-11 00:16:32,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 3: [2023-05-11 00:16:32,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 3: [2023-05-11 00:16:32,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +12: [2023-05-11 00:16:32,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +21: [2023-05-11 00:16:32,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +31: [2023-05-11 00:16:32,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +27: [2023-05-11 00:16:32,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +22: [2023-05-11 00:16:32,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +22: [2023-05-11 00:16:32,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +22: [2023-05-11 00:16:32,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +22: [2023-05-11 00:16:32,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +11: [2023-05-11 00:16:32,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +11: [2023-05-11 00:16:32,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +22: [2023-05-11 00:16:32,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +22: [2023-05-11 00:16:32,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +22: [2023-05-11 00:16:32,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +11: [2023-05-11 00:16:32,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +11: [2023-05-11 00:16:32,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +31: [2023-05-11 00:16:32,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +22: [2023-05-11 00:16:32,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +31: [2023-05-11 00:16:32,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 3: [2023-05-11 00:16:32,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 3: [2023-05-11 00:16:32,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +18: [2023-05-11 00:16:32,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +18: [2023-05-11 00:16:32,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +18: [2023-05-11 00:16:32,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +21: [2023-05-11 00:16:32,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +31: [2023-05-11 00:16:32,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +17: [2023-05-11 00:16:32,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +18: [2023-05-11 00:16:32,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +23: [2023-05-11 00:16:32,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +21: [2023-05-11 00:16:32,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +17: [2023-05-11 00:16:32,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +21: [2023-05-11 00:16:32,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +17: [2023-05-11 00:16:32,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +17: [2023-05-11 00:16:32,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +12: [2023-05-11 00:16:32,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +28: [2023-05-11 00:16:32,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +28: [2023-05-11 00:16:32,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +28: [2023-05-11 00:16:32,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 2: [2023-05-11 00:16:32,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +28: [2023-05-11 00:16:32,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +19: [2023-05-11 00:16:32,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 2: [2023-05-11 00:16:32,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +21: [2023-05-11 00:16:32,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +19: [2023-05-11 00:16:32,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 9: [2023-05-11 00:16:32,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 1: [2023-05-11 00:16:32,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +21: [2023-05-11 00:16:32,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +21: [2023-05-11 00:16:32,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +21: [2023-05-11 00:16:32,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +11: [2023-05-11 00:16:32,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +11: [2023-05-11 00:16:32,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +11: [2023-05-11 00:16:32,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +11: [2023-05-11 00:16:32,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +14: [2023-05-11 00:16:32,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +14: [2023-05-11 00:16:32,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +27: [2023-05-11 00:16:32,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +31: [2023-05-11 00:16:32,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +26: [2023-05-11 00:16:32,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +31: [2023-05-11 00:16:32,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +31: [2023-05-11 00:16:32,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 1: [2023-05-11 00:16:32,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +31: [2023-05-11 00:16:32,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +11: [2023-05-11 00:16:32,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +11: [2023-05-11 00:16:32,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +11: [2023-05-11 00:16:32,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +31: [2023-05-11 00:16:32,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +31: [2023-05-11 00:16:32,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 4: [2023-05-11 00:16:32,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 4: [2023-05-11 00:16:32,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +31: [2023-05-11 00:16:32,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +22: [2023-05-11 00:16:32,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +30: [2023-05-11 00:16:32,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +30: [2023-05-11 00:16:32,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +30: [2023-05-11 00:16:32,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +30: [2023-05-11 00:16:32,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +22: [2023-05-11 00:16:32,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +10: [2023-05-11 00:16:32,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +10: [2023-05-11 00:16:32,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +10: [2023-05-11 00:16:32,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +10: [2023-05-11 00:16:32,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +10: [2023-05-11 00:16:32,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +10: [2023-05-11 00:16:32,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +10: [2023-05-11 00:16:32,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +10: [2023-05-11 00:16:32,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +10: [2023-05-11 00:16:32,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +10: [2023-05-11 00:16:32,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +10: [2023-05-11 00:16:32,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +10: [2023-05-11 00:16:32,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +10: [2023-05-11 00:16:32,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +10: [2023-05-11 00:16:32,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +31: [2023-05-11 00:16:32,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +10: [2023-05-11 00:16:32,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +19: [2023-05-11 00:16:32,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +19: [2023-05-11 00:16:32,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +19: [2023-05-11 00:16:32,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +19: [2023-05-11 00:16:32,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +31: [2023-05-11 00:16:32,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +11: [2023-05-11 00:16:32,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 7: [2023-05-11 00:16:32,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +17: [2023-05-11 00:16:32,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +17: [2023-05-11 00:16:32,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +10: [2023-05-11 00:16:32,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +31: [2023-05-11 00:16:32,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +18: [2023-05-11 00:16:32,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +18: [2023-05-11 00:16:32,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +22: [2023-05-11 00:16:32,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +17: [2023-05-11 00:16:32,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +18: [2023-05-11 00:16:32,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +17: [2023-05-11 00:16:32,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 2: [2023-05-11 00:16:32,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +21: [2023-05-11 00:16:32,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +14: [2023-05-11 00:16:32,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +18: [2023-05-11 00:16:32,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +11: [2023-05-11 00:16:32,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +14: [2023-05-11 00:16:32,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +11: [2023-05-11 00:16:32,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 2: [2023-05-11 00:16:32,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +19: [2023-05-11 00:16:32,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +19: [2023-05-11 00:16:32,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +21: [2023-05-11 00:16:32,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +26: [2023-05-11 00:16:32,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +11: [2023-05-11 00:16:32,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +11: [2023-05-11 00:16:32,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +15: [2023-05-11 00:16:32,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +15: [2023-05-11 00:16:32,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +15: [2023-05-11 00:16:32,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +15: [2023-05-11 00:16:32,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +15: [2023-05-11 00:16:32,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +15: [2023-05-11 00:16:32,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +15: [2023-05-11 00:16:32,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +15: [2023-05-11 00:16:32,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +15: [2023-05-11 00:16:32,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +15: [2023-05-11 00:16:32,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +15: [2023-05-11 00:16:32,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +15: [2023-05-11 00:16:32,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +15: [2023-05-11 00:16:32,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +21: [2023-05-11 00:16:32,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +22: [2023-05-11 00:16:32,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +15: [2023-05-11 00:16:32,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +15: [2023-05-11 00:16:32,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +22: [2023-05-11 00:16:32,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +15: [2023-05-11 00:16:32,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +30: [2023-05-11 00:16:32,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +30: [2023-05-11 00:16:32,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +25: [2023-05-11 00:16:32,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +25: [2023-05-11 00:16:32,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +25: [2023-05-11 00:16:32,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +25: [2023-05-11 00:16:32,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +25: [2023-05-11 00:16:32,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +25: [2023-05-11 00:16:32,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +25: [2023-05-11 00:16:32,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +25: [2023-05-11 00:16:32,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +11: [2023-05-11 00:16:32,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +11: [2023-05-11 00:16:32,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +11: [2023-05-11 00:16:32,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +11: [2023-05-11 00:16:32,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +22: [2023-05-11 00:16:32,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 0: [2023-05-11 00:16:32,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +10: [2023-05-11 00:16:32,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 0: [2023-05-11 00:16:32,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +10: [2023-05-11 00:16:32,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +30: [2023-05-11 00:16:32,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +10: [2023-05-11 00:16:32,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +26: [2023-05-11 00:16:32,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +26: [2023-05-11 00:16:32,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... +26: [2023-05-11 00:16:32,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +21: [2023-05-11 00:16:32,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +30: [2023-05-11 00:16:32,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... + 0: [2023-05-11 00:16:32,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +22: [2023-05-11 00:16:32,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +31: [2023-05-11 00:16:32,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +11: [2023-05-11 00:16:32,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +15: [2023-05-11 00:16:32,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +15: [2023-05-11 00:16:32,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +11: [2023-05-11 00:16:32,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 9: [2023-05-11 00:16:32,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +10: [2023-05-11 00:16:32,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +21: [2023-05-11 00:16:32,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +31: [2023-05-11 00:16:32,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +31: [2023-05-11 00:16:32,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +31: [2023-05-11 00:16:32,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +31: [2023-05-11 00:16:32,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +21: [2023-05-11 00:16:32,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +21: [2023-05-11 00:16:32,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +10: [2023-05-11 00:16:32,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 1: [2023-05-11 00:16:32,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +25: [2023-05-11 00:16:32,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 1: [2023-05-11 00:16:32,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 9: [2023-05-11 00:16:32,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +26: [2023-05-11 00:16:32,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +15: [2023-05-11 00:16:32,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +21: [2023-05-11 00:16:32,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +10: [2023-05-11 00:16:32,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +25: [2023-05-11 00:16:32,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +21: [2023-05-11 00:16:32,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +26: [2023-05-11 00:16:32,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +21: [2023-05-11 00:16:32,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +21: [2023-05-11 00:16:32,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +26: [2023-05-11 00:16:32,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +15: [2023-05-11 00:16:32,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +21: [2023-05-11 00:16:32,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +11: [2023-05-11 00:16:32,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +25: [2023-05-11 00:16:32,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +22: [2023-05-11 00:16:32,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +25: [2023-05-11 00:16:32,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +21: [2023-05-11 00:16:32,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +31: [2023-05-11 00:16:32,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 9: [2023-05-11 00:16:32,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +11: [2023-05-11 00:16:32,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 9: [2023-05-11 00:16:32,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 5: [2023-05-11 00:16:32,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 5: [2023-05-11 00:16:32,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 5: [2023-05-11 00:16:32,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +10: [2023-05-11 00:16:32,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +11: [2023-05-11 00:16:32,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +15: [2023-05-11 00:16:32,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +31: [2023-05-11 00:16:32,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 7: [2023-05-11 00:16:32,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +31: [2023-05-11 00:16:32,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +31: [2023-05-11 00:16:32,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +11: [2023-05-11 00:16:32,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +15: [2023-05-11 00:16:32,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +25: [2023-05-11 00:16:32,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +31: [2023-05-11 00:16:32,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt... +15: [2023-05-11 00:16:32,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +25: [2023-05-11 00:16:32,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 7: [2023-05-11 00:16:32,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 0: [2023-05-11 00:16:32,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +11: [2023-05-11 00:16:32,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. +10: [2023-05-11 00:16:32,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +31: [2023-05-11 00:16:32,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 1: [2023-05-11 00:16:32,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +25: [2023-05-11 00:16:32,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +15: [2023-05-11 00:16:32,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +30: [2023-05-11 00:16:32,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +30: [2023-05-11 00:16:32,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +30: [2023-05-11 00:16:32,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +30: [2023-05-11 00:16:32,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +30: [2023-05-11 00:16:32,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +30: [2023-05-11 00:16:32,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +30: [2023-05-11 00:16:32,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +30: [2023-05-11 00:16:32,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +30: [2023-05-11 00:16:32,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +30: [2023-05-11 00:16:32,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 5: [2023-05-11 00:16:32,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +30: [2023-05-11 00:16:32,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +25: [2023-05-11 00:16:32,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +30: [2023-05-11 00:16:32,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +31: [2023-05-11 00:16:32,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +31: [2023-05-11 00:16:32,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +30: [2023-05-11 00:16:32,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +30: [2023-05-11 00:16:32,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +30: [2023-05-11 00:16:32,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +30: [2023-05-11 00:16:32,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +11: [2023-05-11 00:16:32,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +31: [2023-05-11 00:16:32,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +30: [2023-05-11 00:16:32,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 5: [2023-05-11 00:16:32,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +30: [2023-05-11 00:16:32,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +30: [2023-05-11 00:16:32,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +29: [2023-05-11 00:16:32,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +29: [2023-05-11 00:16:32,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +29: [2023-05-11 00:16:32,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +29: [2023-05-11 00:16:32,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +29: [2023-05-11 00:16:32,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +29: [2023-05-11 00:16:32,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +29: [2023-05-11 00:16:32,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +29: [2023-05-11 00:16:32,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +16: [2023-05-11 00:16:32,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +16: [2023-05-11 00:16:32,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +16: [2023-05-11 00:16:32,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +16: [2023-05-11 00:16:32,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 0: [2023-05-11 00:16:32,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 5: [2023-05-11 00:16:32,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +16: [2023-05-11 00:16:32,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +16: [2023-05-11 00:16:32,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +16: [2023-05-11 00:16:32,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +16: [2023-05-11 00:16:32,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +30: [2023-05-11 00:16:32,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +30: [2023-05-11 00:16:32,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 6: [2023-05-11 00:16:32,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 0: [2023-05-11 00:16:32,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +30: [2023-05-11 00:16:32,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 6: [2023-05-11 00:16:32,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +29: [2023-05-11 00:16:32,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +16: [2023-05-11 00:16:32,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +16: [2023-05-11 00:16:32,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +16: [2023-05-11 00:16:32,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +30: [2023-05-11 00:16:32,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +29: [2023-05-11 00:16:32,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +29: [2023-05-11 00:16:32,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 6: [2023-05-11 00:16:32,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +16: [2023-05-11 00:16:32,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 6: [2023-05-11 00:16:32,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +29: [2023-05-11 00:16:32,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 5: [2023-05-11 00:16:32,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +29: [2023-05-11 00:16:32,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +16: [2023-05-11 00:16:32,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +30: [2023-05-11 00:16:32,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +29: [2023-05-11 00:16:32,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +13: [2023-05-11 00:16:32,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +13: [2023-05-11 00:16:32,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +13: [2023-05-11 00:16:32,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +16: [2023-05-11 00:16:32,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +16: [2023-05-11 00:16:32,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +29: [2023-05-11 00:16:32,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +13: [2023-05-11 00:16:32,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +16: [2023-05-11 00:16:32,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 5: [2023-05-11 00:16:32,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +25: [2023-05-11 00:16:32,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +25: [2023-05-11 00:16:32,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +25: [2023-05-11 00:16:32,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +25: [2023-05-11 00:16:32,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +29: [2023-05-11 00:16:32,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +13: [2023-05-11 00:16:33,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +13: [2023-05-11 00:16:33,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +29: [2023-05-11 00:16:33,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +29: [2023-05-11 00:16:33,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +29: [2023-05-11 00:16:33,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +29: [2023-05-11 00:16:33,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +13: [2023-05-11 00:16:33,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +13: [2023-05-11 00:16:33,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +25: [2023-05-11 00:16:33,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +25: [2023-05-11 00:16:33,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +25: [2023-05-11 00:16:33,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +20: [2023-05-11 00:16:33,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +20: [2023-05-11 00:16:33,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +20: [2023-05-11 00:16:33,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +20: [2023-05-11 00:16:33,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +25: [2023-05-11 00:16:33,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +29: [2023-05-11 00:16:33,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +29: [2023-05-11 00:16:33,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +28: [2023-05-11 00:16:33,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +28: [2023-05-11 00:16:33,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +29: [2023-05-11 00:16:33,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +28: [2023-05-11 00:16:33,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +28: [2023-05-11 00:16:33,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +28: [2023-05-11 00:16:33,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +28: [2023-05-11 00:16:33,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +28: [2023-05-11 00:16:33,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +28: [2023-05-11 00:16:33,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +28: [2023-05-11 00:16:33,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +28: [2023-05-11 00:16:33,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +28: [2023-05-11 00:16:33,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +28: [2023-05-11 00:16:33,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +29: [2023-05-11 00:16:33,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +28: [2023-05-11 00:16:33,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +28: [2023-05-11 00:16:33,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +28: [2023-05-11 00:16:33,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +20: [2023-05-11 00:16:33,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +20: [2023-05-11 00:16:33,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +20: [2023-05-11 00:16:33,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +20: [2023-05-11 00:16:33,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +28: [2023-05-11 00:16:33,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +25: [2023-05-11 00:16:33,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +29: [2023-05-11 00:16:33,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +25: [2023-05-11 00:16:33,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +29: [2023-05-11 00:16:33,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +25: [2023-05-11 00:16:33,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +25: [2023-05-11 00:16:33,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +29: [2023-05-11 00:16:33,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +29: [2023-05-11 00:16:33,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +24: [2023-05-11 00:16:33,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +24: [2023-05-11 00:16:33,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +28: [2023-05-11 00:16:33,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +24: [2023-05-11 00:16:33,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +24: [2023-05-11 00:16:33,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +29: [2023-05-11 00:16:33,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +29: [2023-05-11 00:16:33,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +29: [2023-05-11 00:16:33,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +29: [2023-05-11 00:16:33,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 8: [2023-05-11 00:16:33,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 8: [2023-05-11 00:16:33,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +25: [2023-05-11 00:16:33,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +28: [2023-05-11 00:16:33,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +28: [2023-05-11 00:16:33,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +25: [2023-05-11 00:16:33,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 8: [2023-05-11 00:16:33,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +25: [2023-05-11 00:16:33,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +25: [2023-05-11 00:16:33,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +24: [2023-05-11 00:16:33,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +24: [2023-05-11 00:16:33,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +24: [2023-05-11 00:16:33,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +22: [2023-05-11 00:16:33,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +28: [2023-05-11 00:16:33,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +22: [2023-05-11 00:16:33,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +24: [2023-05-11 00:16:33,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +22: [2023-05-11 00:16:33,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +22: [2023-05-11 00:16:33,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +28: [2023-05-11 00:16:33,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +28: [2023-05-11 00:16:33,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +28: [2023-05-11 00:16:33,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +22: [2023-05-11 00:16:33,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +22: [2023-05-11 00:16:33,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +22: [2023-05-11 00:16:33,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +24: [2023-05-11 00:16:33,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +24: [2023-05-11 00:16:33,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +24: [2023-05-11 00:16:33,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +24: [2023-05-11 00:16:33,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +22: [2023-05-11 00:16:33,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +24: [2023-05-11 00:16:33,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +24: [2023-05-11 00:16:33,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +24: [2023-05-11 00:16:33,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +24: [2023-05-11 00:16:33,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +14: [2023-05-11 00:16:33,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +24: [2023-05-11 00:16:33,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +24: [2023-05-11 00:16:33,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +14: [2023-05-11 00:16:33,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +24: [2023-05-11 00:16:33,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +24: [2023-05-11 00:16:33,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +14: [2023-05-11 00:16:33,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +28: [2023-05-11 00:16:33,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +14: [2023-05-11 00:16:33,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +19: [2023-05-11 00:16:33,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +19: [2023-05-11 00:16:33,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +24: [2023-05-11 00:16:33,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +24: [2023-05-11 00:16:33,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +24: [2023-05-11 00:16:33,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +24: [2023-05-11 00:16:33,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +19: [2023-05-11 00:16:33,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +22: [2023-05-11 00:16:33,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +13: [2023-05-11 00:16:33,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +13: [2023-05-11 00:16:33,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +13: [2023-05-11 00:16:33,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +13: [2023-05-11 00:16:33,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +13: [2023-05-11 00:16:33,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +13: [2023-05-11 00:16:33,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +13: [2023-05-11 00:16:33,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +13: [2023-05-11 00:16:33,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +13: [2023-05-11 00:16:33,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +13: [2023-05-11 00:16:33,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +13: [2023-05-11 00:16:33,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +19: [2023-05-11 00:16:33,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +13: [2023-05-11 00:16:33,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +13: [2023-05-11 00:16:33,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +13: [2023-05-11 00:16:33,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +24: [2023-05-11 00:16:33,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +12: [2023-05-11 00:16:33,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +12: [2023-05-11 00:16:33,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +12: [2023-05-11 00:16:33,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +12: [2023-05-11 00:16:33,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +12: [2023-05-11 00:16:33,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +12: [2023-05-11 00:16:33,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +22: [2023-05-11 00:16:33,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +14: [2023-05-11 00:16:33,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +14: [2023-05-11 00:16:33,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +19: [2023-05-11 00:16:33,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +19: [2023-05-11 00:16:33,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +13: [2023-05-11 00:16:33,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +13: [2023-05-11 00:16:33,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +14: [2023-05-11 00:16:33,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +19: [2023-05-11 00:16:33,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +14: [2023-05-11 00:16:33,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +19: [2023-05-11 00:16:33,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +22: [2023-05-11 00:16:33,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +22: [2023-05-11 00:16:33,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +12: [2023-05-11 00:16:33,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +17: [2023-05-11 00:16:33,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +17: [2023-05-11 00:16:33,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +17: [2023-05-11 00:16:33,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +17: [2023-05-11 00:16:33,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +12: [2023-05-11 00:16:33,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +22: [2023-05-11 00:16:33,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +22: [2023-05-11 00:16:33,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +19: [2023-05-11 00:16:33,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +24: [2023-05-11 00:16:33,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +24: [2023-05-11 00:16:33,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +13: [2023-05-11 00:16:33,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +17: [2023-05-11 00:16:33,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +17: [2023-05-11 00:16:33,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +17: [2023-05-11 00:16:33,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +13: [2023-05-11 00:16:33,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +13: [2023-05-11 00:16:33,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +17: [2023-05-11 00:16:33,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +22: [2023-05-11 00:16:33,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +19: [2023-05-11 00:16:33,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +12: [2023-05-11 00:16:33,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +12: [2023-05-11 00:16:33,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +12: [2023-05-11 00:16:33,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +24: [2023-05-11 00:16:33,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +24: [2023-05-11 00:16:33,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +22: [2023-05-11 00:16:33,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 6: [2023-05-11 00:16:33,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +22: [2023-05-11 00:16:33,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +22: [2023-05-11 00:16:33,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +22: [2023-05-11 00:16:33,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +22: [2023-05-11 00:16:33,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +22: [2023-05-11 00:16:33,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 6: [2023-05-11 00:16:33,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +22: [2023-05-11 00:16:33,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +22: [2023-05-11 00:16:33,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +22: [2023-05-11 00:16:33,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +16: [2023-05-11 00:16:33,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +16: [2023-05-11 00:16:33,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 4: [2023-05-11 00:16:33,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +13: [2023-05-11 00:16:33,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +13: [2023-05-11 00:16:33,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 8: [2023-05-11 00:16:33,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 8: [2023-05-11 00:16:33,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 2: [2023-05-11 00:16:33,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 6: [2023-05-11 00:16:33,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +20: [2023-05-11 00:16:33,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +20: [2023-05-11 00:16:33,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +20: [2023-05-11 00:16:33,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +20: [2023-05-11 00:16:33,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +20: [2023-05-11 00:16:33,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +20: [2023-05-11 00:16:33,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +20: [2023-05-11 00:16:33,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +20: [2023-05-11 00:16:33,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +20: [2023-05-11 00:16:33,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +20: [2023-05-11 00:16:33,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +20: [2023-05-11 00:16:33,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +20: [2023-05-11 00:16:33,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +20: [2023-05-11 00:16:33,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +20: [2023-05-11 00:16:33,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 8: [2023-05-11 00:16:33,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +13: [2023-05-11 00:16:33,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +20: [2023-05-11 00:16:33,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 2: [2023-05-11 00:16:33,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +19: [2023-05-11 00:16:33,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +12: [2023-05-11 00:16:33,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +19: [2023-05-11 00:16:33,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +13: [2023-05-11 00:16:33,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +12: [2023-05-11 00:16:33,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +26: [2023-05-11 00:16:33,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +26: [2023-05-11 00:16:33,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +20: [2023-05-11 00:16:33,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +24: [2023-05-11 00:16:33,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +16: [2023-05-11 00:16:33,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +24: [2023-05-11 00:16:33,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +16: [2023-05-11 00:16:33,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +12: [2023-05-11 00:16:33,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +19: [2023-05-11 00:16:33,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +10: [2023-05-11 00:16:33,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +10: [2023-05-11 00:16:33,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +10: [2023-05-11 00:16:33,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +26: [2023-05-11 00:16:33,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +10: [2023-05-11 00:16:33,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 2: [2023-05-11 00:16:33,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 2: [2023-05-11 00:16:33,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 2: [2023-05-11 00:16:33,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +11: [2023-05-11 00:16:33,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +11: [2023-05-11 00:16:33,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +11: [2023-05-11 00:16:33,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +26: [2023-05-11 00:16:33,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +11: [2023-05-11 00:16:33,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 2: [2023-05-11 00:16:33,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +12: [2023-05-11 00:16:33,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +16: [2023-05-11 00:16:33,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +16: [2023-05-11 00:16:33,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +26: [2023-05-11 00:16:33,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +19: [2023-05-11 00:16:33,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +26: [2023-05-11 00:16:33,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +13: [2023-05-11 00:16:33,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +19: [2023-05-11 00:16:33,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +22: [2023-05-11 00:16:33,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +22: [2023-05-11 00:16:33,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +24: [2023-05-11 00:16:33,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +19: [2023-05-11 00:16:33,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +22: [2023-05-11 00:16:33,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +22: [2023-05-11 00:16:33,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +11: [2023-05-11 00:16:33,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +10: [2023-05-11 00:16:33,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +16: [2023-05-11 00:16:33,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +27: [2023-05-11 00:16:33,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +27: [2023-05-11 00:16:33,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +27: [2023-05-11 00:16:33,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +27: [2023-05-11 00:16:33,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +27: [2023-05-11 00:16:33,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +27: [2023-05-11 00:16:33,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +16: [2023-05-11 00:16:33,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +10: [2023-05-11 00:16:33,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +11: [2023-05-11 00:16:33,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +11: [2023-05-11 00:16:33,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +20: [2023-05-11 00:16:33,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +20: [2023-05-11 00:16:33,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +11: [2023-05-11 00:16:33,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +27: [2023-05-11 00:16:33,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +27: [2023-05-11 00:16:33,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +12: [2023-05-11 00:16:33,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +26: [2023-05-11 00:16:33,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +20: [2023-05-11 00:16:33,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +20: [2023-05-11 00:16:33,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +26: [2023-05-11 00:16:33,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +10: [2023-05-11 00:16:33,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +14: [2023-05-11 00:16:33,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +14: [2023-05-11 00:16:33,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +14: [2023-05-11 00:16:33,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +14: [2023-05-11 00:16:33,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +14: [2023-05-11 00:16:33,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +14: [2023-05-11 00:16:33,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +14: [2023-05-11 00:16:33,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +10: [2023-05-11 00:16:33,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +14: [2023-05-11 00:16:33,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +14: [2023-05-11 00:16:33,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +14: [2023-05-11 00:16:33,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +14: [2023-05-11 00:16:33,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +14: [2023-05-11 00:16:33,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +14: [2023-05-11 00:16:33,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +14: [2023-05-11 00:16:33,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 6: [2023-05-11 00:16:33,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +23: [2023-05-11 00:16:33,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +23: [2023-05-11 00:16:33,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +14: [2023-05-11 00:16:33,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +14: [2023-05-11 00:16:33,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +19: [2023-05-11 00:16:33,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +19: [2023-05-11 00:16:33,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +19: [2023-05-11 00:16:33,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +19: [2023-05-11 00:16:33,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 2: [2023-05-11 00:16:33,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +23: [2023-05-11 00:16:33,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 8: [2023-05-11 00:16:33,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +23: [2023-05-11 00:16:33,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +22: [2023-05-11 00:16:33,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +22: [2023-05-11 00:16:33,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +22: [2023-05-11 00:16:33,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +22: [2023-05-11 00:16:33,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +21: [2023-05-11 00:16:33,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +21: [2023-05-11 00:16:33,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +21: [2023-05-11 00:16:33,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +21: [2023-05-11 00:16:33,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +21: [2023-05-11 00:16:33,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +21: [2023-05-11 00:16:33,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +27: [2023-05-11 00:16:33,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +21: [2023-05-11 00:16:33,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +21: [2023-05-11 00:16:33,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +16: [2023-05-11 00:16:33,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +27: [2023-05-11 00:16:33,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +16: [2023-05-11 00:16:33,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +20: [2023-05-11 00:16:33,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +23: [2023-05-11 00:16:33,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +20: [2023-05-11 00:16:33,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +16: [2023-05-11 00:16:33,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +20: [2023-05-11 00:16:33,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +20: [2023-05-11 00:16:33,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +16: [2023-05-11 00:16:33,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +23: [2023-05-11 00:16:33,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +16: [2023-05-11 00:16:33,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +16: [2023-05-11 00:16:33,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 2: [2023-05-11 00:16:33,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +23: [2023-05-11 00:16:33,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +14: [2023-05-11 00:16:33,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +27: [2023-05-11 00:16:33,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +23: [2023-05-11 00:16:33,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +19: [2023-05-11 00:16:33,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +19: [2023-05-11 00:16:33,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +19: [2023-05-11 00:16:33,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +12: [2023-05-11 00:16:33,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +12: [2023-05-11 00:16:33,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +12: [2023-05-11 00:16:33,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +12: [2023-05-11 00:16:33,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 8: [2023-05-11 00:16:33,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +27: [2023-05-11 00:16:33,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +27: [2023-05-11 00:16:33,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +16: [2023-05-11 00:16:33,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +19: [2023-05-11 00:16:33,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +21: [2023-05-11 00:16:33,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +16: [2023-05-11 00:16:33,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +31: [2023-05-11 00:16:33,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +27: [2023-05-11 00:16:33,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +31: [2023-05-11 00:16:33,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +31: [2023-05-11 00:16:33,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +27: [2023-05-11 00:16:33,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +14: [2023-05-11 00:16:33,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +14: [2023-05-11 00:16:33,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 0: [2023-05-11 00:16:33,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 0: [2023-05-11 00:16:33,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 3: [2023-05-11 00:16:33,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +31: [2023-05-11 00:16:33,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +14: [2023-05-11 00:16:33,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 2: [2023-05-11 00:16:33,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +27: [2023-05-11 00:16:33,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +21: [2023-05-11 00:16:33,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +12: [2023-05-11 00:16:33,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +12: [2023-05-11 00:16:33,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +12: [2023-05-11 00:16:33,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +12: [2023-05-11 00:16:33,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +21: [2023-05-11 00:16:33,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +27: [2023-05-11 00:16:33,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +27: [2023-05-11 00:16:33,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +31: [2023-05-11 00:16:33,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 7: [2023-05-11 00:16:33,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 7: [2023-05-11 00:16:33,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 7: [2023-05-11 00:16:33,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +18: [2023-05-11 00:16:33,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +27: [2023-05-11 00:16:33,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +31: [2023-05-11 00:16:33,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +27: [2023-05-11 00:16:33,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 3: [2023-05-11 00:16:33,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +31: [2023-05-11 00:16:33,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 4: [2023-05-11 00:16:33,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +27: [2023-05-11 00:16:33,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +18: [2023-05-11 00:16:33,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. + 9: [2023-05-11 00:16:33,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +31: [2023-05-11 00:16:33,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +27: [2023-05-11 00:16:33,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +18: [2023-05-11 00:16:33,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +27: [2023-05-11 00:16:33,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +18: [2023-05-11 00:16:33,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +23: [2023-05-11 00:16:33,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +23: [2023-05-11 00:16:33,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +19: [2023-05-11 00:16:33,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +23: [2023-05-11 00:16:33,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +23: [2023-05-11 00:16:33,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +19: [2023-05-11 00:16:33,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +14: [2023-05-11 00:16:33,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +14: [2023-05-11 00:16:33,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +27: [2023-05-11 00:16:33,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +19: [2023-05-11 00:16:33,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +23: [2023-05-11 00:16:33,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +23: [2023-05-11 00:16:33,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +19: [2023-05-11 00:16:33,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +23: [2023-05-11 00:16:33,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +23: [2023-05-11 00:16:33,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +23: [2023-05-11 00:16:33,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +23: [2023-05-11 00:16:33,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +23: [2023-05-11 00:16:33,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +23: [2023-05-11 00:16:33,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 2: [2023-05-11 00:16:33,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 2: [2023-05-11 00:16:33,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +21: [2023-05-11 00:16:33,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +23: [2023-05-11 00:16:33,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +23: [2023-05-11 00:16:33,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt. +27: [2023-05-11 00:16:33,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +27: [2023-05-11 00:16:33,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +23: [2023-05-11 00:16:33,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +18: [2023-05-11 00:16:33,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +21: [2023-05-11 00:16:33,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +23: [2023-05-11 00:16:33,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 1: [2023-05-11 00:16:33,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +14: [2023-05-11 00:16:33,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +18: [2023-05-11 00:16:33,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 4: [2023-05-11 00:16:33,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +12: [2023-05-11 00:16:33,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +19: [2023-05-11 00:16:33,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +19: [2023-05-11 00:16:33,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +19: [2023-05-11 00:16:33,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +12: [2023-05-11 00:16:33,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +19: [2023-05-11 00:16:33,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +12: [2023-05-11 00:16:33,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +12: [2023-05-11 00:16:33,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +12: [2023-05-11 00:16:33,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +12: [2023-05-11 00:16:33,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +18: [2023-05-11 00:16:33,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +18: [2023-05-11 00:16:33,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +11: [2023-05-11 00:16:33,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +21: [2023-05-11 00:16:33,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +11: [2023-05-11 00:16:33,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +11: [2023-05-11 00:16:33,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +11: [2023-05-11 00:16:33,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +17: [2023-05-11 00:16:33,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +17: [2023-05-11 00:16:33,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +17: [2023-05-11 00:16:33,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +17: [2023-05-11 00:16:33,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +17: [2023-05-11 00:16:33,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +17: [2023-05-11 00:16:33,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +17: [2023-05-11 00:16:33,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +11: [2023-05-11 00:16:33,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +11: [2023-05-11 00:16:33,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +11: [2023-05-11 00:16:33,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +17: [2023-05-11 00:16:33,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +30: [2023-05-11 00:16:33,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +11: [2023-05-11 00:16:33,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +11: [2023-05-11 00:16:33,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +11: [2023-05-11 00:16:33,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +17: [2023-05-11 00:16:33,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +17: [2023-05-11 00:16:33,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +30: [2023-05-11 00:16:33,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +17: [2023-05-11 00:16:33,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +17: [2023-05-11 00:16:33,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +17: [2023-05-11 00:16:33,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +21: [2023-05-11 00:16:33,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +26: [2023-05-11 00:16:33,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +26: [2023-05-11 00:16:33,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +30: [2023-05-11 00:16:33,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +26: [2023-05-11 00:16:33,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +26: [2023-05-11 00:16:33,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +30: [2023-05-11 00:16:33,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +23: [2023-05-11 00:16:33,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +26: [2023-05-11 00:16:33,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +26: [2023-05-11 00:16:33,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +26: [2023-05-11 00:16:33,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +26: [2023-05-11 00:16:33,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +12: [2023-05-11 00:16:33,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +12: [2023-05-11 00:16:33,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +11: [2023-05-11 00:16:33,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +11: [2023-05-11 00:16:33,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +11: [2023-05-11 00:16:33,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +27: [2023-05-11 00:16:33,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +27: [2023-05-11 00:16:33,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +26: [2023-05-11 00:16:33,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +26: [2023-05-11 00:16:33,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +26: [2023-05-11 00:16:33,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +26: [2023-05-11 00:16:33,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +17: [2023-05-11 00:16:33,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +26: [2023-05-11 00:16:33,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +26: [2023-05-11 00:16:33,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +18: [2023-05-11 00:16:33,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +18: [2023-05-11 00:16:33,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +18: [2023-05-11 00:16:33,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +14: [2023-05-11 00:16:33,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +18: [2023-05-11 00:16:33,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +17: [2023-05-11 00:16:33,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +26: [2023-05-11 00:16:33,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +17: [2023-05-11 00:16:33,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +11: [2023-05-11 00:16:33,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +23: [2023-05-11 00:16:33,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +23: [2023-05-11 00:16:33,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 9: [2023-05-11 00:16:33,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +11: [2023-05-11 00:16:33,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +11: [2023-05-11 00:16:33,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +30: [2023-05-11 00:16:33,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +30: [2023-05-11 00:16:33,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +26: [2023-05-11 00:16:33,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +10: [2023-05-11 00:16:33,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +10: [2023-05-11 00:16:33,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +10: [2023-05-11 00:16:33,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +10: [2023-05-11 00:16:33,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +10: [2023-05-11 00:16:33,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +10: [2023-05-11 00:16:33,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +10: [2023-05-11 00:16:33,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +10: [2023-05-11 00:16:33,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +30: [2023-05-11 00:16:33,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +30: [2023-05-11 00:16:33,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +10: [2023-05-11 00:16:33,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +10: [2023-05-11 00:16:33,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 1: [2023-05-11 00:16:33,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +21: [2023-05-11 00:16:33,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +31: [2023-05-11 00:16:33,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +31: [2023-05-11 00:16:33,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +31: [2023-05-11 00:16:33,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +31: [2023-05-11 00:16:33,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +18: [2023-05-11 00:16:33,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +18: [2023-05-11 00:16:33,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +31: [2023-05-11 00:16:33,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +31: [2023-05-11 00:16:33,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +31: [2023-05-11 00:16:33,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +31: [2023-05-11 00:16:33,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +31: [2023-05-11 00:16:33,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +31: [2023-05-11 00:16:33,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +31: [2023-05-11 00:16:33,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +31: [2023-05-11 00:16:33,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +31: [2023-05-11 00:16:33,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 7: [2023-05-11 00:16:33,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +31: [2023-05-11 00:16:33,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 3: [2023-05-11 00:16:33,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 3: [2023-05-11 00:16:33,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +18: [2023-05-11 00:16:33,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +10: [2023-05-11 00:16:33,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +10: [2023-05-11 00:16:33,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +10: [2023-05-11 00:16:33,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +10: [2023-05-11 00:16:33,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +23: [2023-05-11 00:16:33,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +27: [2023-05-11 00:16:33,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +27: [2023-05-11 00:16:33,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 9: [2023-05-11 00:16:33,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +27: [2023-05-11 00:16:33,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 4: [2023-05-11 00:16:33,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +31: [2023-05-11 00:16:33,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +31: [2023-05-11 00:16:33,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 1: [2023-05-11 00:16:33,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +10: [2023-05-11 00:16:33,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 3: [2023-05-11 00:16:33,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +27: [2023-05-11 00:16:33,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +23: [2023-05-11 00:16:33,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +23: [2023-05-11 00:16:33,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +10: [2023-05-11 00:16:33,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 3: [2023-05-11 00:16:33,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 9: [2023-05-11 00:16:33,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +18: [2023-05-11 00:16:33,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +18: [2023-05-11 00:16:33,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +18: [2023-05-11 00:16:33,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +23: [2023-05-11 00:16:33,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +11: [2023-05-11 00:16:33,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +17: [2023-05-11 00:16:33,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 4: [2023-05-11 00:16:33,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +17: [2023-05-11 00:16:33,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +30: [2023-05-11 00:16:33,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +30: [2023-05-11 00:16:33,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +17: [2023-05-11 00:16:33,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +11: [2023-05-11 00:16:33,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 7: [2023-05-11 00:16:33,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +18: [2023-05-11 00:16:33,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +18: [2023-05-11 00:16:33,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 7: [2023-05-11 00:16:33,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +18: [2023-05-11 00:16:33,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +10: [2023-05-11 00:16:33,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +10: [2023-05-11 00:16:33,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +18: [2023-05-11 00:16:33,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +31: [2023-05-11 00:16:33,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +10: [2023-05-11 00:16:33,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +23: [2023-05-11 00:16:33,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +18: [2023-05-11 00:16:33,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +18: [2023-05-11 00:16:33,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +18: [2023-05-11 00:16:33,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +15: [2023-05-11 00:16:33,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +15: [2023-05-11 00:16:33,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +15: [2023-05-11 00:16:33,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +15: [2023-05-11 00:16:33,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +31: [2023-05-11 00:16:33,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +29: [2023-05-11 00:16:33,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +29: [2023-05-11 00:16:33,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +31: [2023-05-11 00:16:33,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +30: [2023-05-11 00:16:33,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +29: [2023-05-11 00:16:33,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +29: [2023-05-11 00:16:33,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +30: [2023-05-11 00:16:33,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +15: [2023-05-11 00:16:33,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +17: [2023-05-11 00:16:33,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +15: [2023-05-11 00:16:33,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +11: [2023-05-11 00:16:33,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +18: [2023-05-11 00:16:33,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +17: [2023-05-11 00:16:33,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +11: [2023-05-11 00:16:33,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +26: [2023-05-11 00:16:33,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +26: [2023-05-11 00:16:33,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +26: [2023-05-11 00:16:33,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +15: [2023-05-11 00:16:33,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +15: [2023-05-11 00:16:33,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +11: [2023-05-11 00:16:33,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +29: [2023-05-11 00:16:33,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +29: [2023-05-11 00:16:33,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +29: [2023-05-11 00:16:33,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +17: [2023-05-11 00:16:33,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +18: [2023-05-11 00:16:33,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +11: [2023-05-11 00:16:33,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +29: [2023-05-11 00:16:33,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +10: [2023-05-11 00:16:33,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +10: [2023-05-11 00:16:33,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +10: [2023-05-11 00:16:33,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +18: [2023-05-11 00:16:33,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +30: [2023-05-11 00:16:33,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +10: [2023-05-11 00:16:33,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +31: [2023-05-11 00:16:33,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +31: [2023-05-11 00:16:33,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +17: [2023-05-11 00:16:33,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +31: [2023-05-11 00:16:33,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 9: [2023-05-11 00:16:33,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 9: [2023-05-11 00:16:33,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +15: [2023-05-11 00:16:33,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +11: [2023-05-11 00:16:33,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +30: [2023-05-11 00:16:33,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +15: [2023-05-11 00:16:33,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +18: [2023-05-11 00:16:33,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +26: [2023-05-11 00:16:33,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +31: [2023-05-11 00:16:33,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +18: [2023-05-11 00:16:33,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +11: [2023-05-11 00:16:33,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +26: [2023-05-11 00:16:33,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +30: [2023-05-11 00:16:33,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +29: [2023-05-11 00:16:33,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +10: [2023-05-11 00:16:33,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +26: [2023-05-11 00:16:33,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +25: [2023-05-11 00:16:33,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +25: [2023-05-11 00:16:33,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +25: [2023-05-11 00:16:33,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +25: [2023-05-11 00:16:33,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +17: [2023-05-11 00:16:33,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +31: [2023-05-11 00:16:33,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +16: [2023-05-11 00:16:33,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +16: [2023-05-11 00:16:33,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +29: [2023-05-11 00:16:33,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +15: [2023-05-11 00:16:33,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +29: [2023-05-11 00:16:33,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +30: [2023-05-11 00:16:33,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +16: [2023-05-11 00:16:33,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 5: [2023-05-11 00:16:33,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 5: [2023-05-11 00:16:33,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +16: [2023-05-11 00:16:33,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +25: [2023-05-11 00:16:33,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +25: [2023-05-11 00:16:33,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +15: [2023-05-11 00:16:33,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +18: [2023-05-11 00:16:33,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +25: [2023-05-11 00:16:33,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +26: [2023-05-11 00:16:33,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. +25: [2023-05-11 00:16:33,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +16: [2023-05-11 00:16:33,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +16: [2023-05-11 00:16:33,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +29: [2023-05-11 00:16:33,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +16: [2023-05-11 00:16:33,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +16: [2023-05-11 00:16:33,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 7: [2023-05-11 00:16:33,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +29: [2023-05-11 00:16:33,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +25: [2023-05-11 00:16:33,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +25: [2023-05-11 00:16:33,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +15: [2023-05-11 00:16:33,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +29: [2023-05-11 00:16:33,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +26: [2023-05-11 00:16:33,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +18: [2023-05-11 00:16:33,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +16: [2023-05-11 00:16:33,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +16: [2023-05-11 00:16:33,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +15: [2023-05-11 00:16:33,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +25: [2023-05-11 00:16:33,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +29: [2023-05-11 00:16:33,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +25: [2023-05-11 00:16:33,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +16: [2023-05-11 00:16:33,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +16: [2023-05-11 00:16:33,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +15: [2023-05-11 00:16:33,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +15: [2023-05-11 00:16:33,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +25: [2023-05-11 00:16:33,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +30: [2023-05-11 00:16:33,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +29: [2023-05-11 00:16:33,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 0: [2023-05-11 00:16:33,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... + 0: [2023-05-11 00:16:33,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +30: [2023-05-11 00:16:33,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 0: [2023-05-11 00:16:33,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt... +30: [2023-05-11 00:16:33,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +30: [2023-05-11 00:16:33,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 0: [2023-05-11 00:16:33,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt... +25: [2023-05-11 00:16:33,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +16: [2023-05-11 00:16:33,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +16: [2023-05-11 00:16:33,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +25: [2023-05-11 00:16:33,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 1: [2023-05-11 00:16:33,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +30: [2023-05-11 00:16:33,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 1: [2023-05-11 00:16:33,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +30: [2023-05-11 00:16:33,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +30: [2023-05-11 00:16:33,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +15: [2023-05-11 00:16:33,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +15: [2023-05-11 00:16:33,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +15: [2023-05-11 00:16:33,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +15: [2023-05-11 00:16:33,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +30: [2023-05-11 00:16:33,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +25: [2023-05-11 00:16:33,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +16: [2023-05-11 00:16:33,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +16: [2023-05-11 00:16:33,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +15: [2023-05-11 00:16:33,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +15: [2023-05-11 00:16:33,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +15: [2023-05-11 00:16:33,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +15: [2023-05-11 00:16:33,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 1: [2023-05-11 00:16:33,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 0: [2023-05-11 00:16:33,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +30: [2023-05-11 00:16:33,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +30: [2023-05-11 00:16:33,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +30: [2023-05-11 00:16:33,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +30: [2023-05-11 00:16:33,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +28: [2023-05-11 00:16:33,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +28: [2023-05-11 00:16:33,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +28: [2023-05-11 00:16:33,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +28: [2023-05-11 00:16:33,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +30: [2023-05-11 00:16:33,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +30: [2023-05-11 00:16:33,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +30: [2023-05-11 00:16:33,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +30: [2023-05-11 00:16:33,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +28: [2023-05-11 00:16:33,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +15: [2023-05-11 00:16:33,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +28: [2023-05-11 00:16:33,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +28: [2023-05-11 00:16:33,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +15: [2023-05-11 00:16:33,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +28: [2023-05-11 00:16:33,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +15: [2023-05-11 00:16:33,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +15: [2023-05-11 00:16:33,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +15: [2023-05-11 00:16:33,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +15: [2023-05-11 00:16:33,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 0: [2023-05-11 00:16:33,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +28: [2023-05-11 00:16:33,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +28: [2023-05-11 00:16:33,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +28: [2023-05-11 00:16:33,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +15: [2023-05-11 00:16:33,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +15: [2023-05-11 00:16:33,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +28: [2023-05-11 00:16:33,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +28: [2023-05-11 00:16:33,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +28: [2023-05-11 00:16:33,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +28: [2023-05-11 00:16:33,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +24: [2023-05-11 00:16:33,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +28: [2023-05-11 00:16:33,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +24: [2023-05-11 00:16:33,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +24: [2023-05-11 00:16:33,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +24: [2023-05-11 00:16:33,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 5: [2023-05-11 00:16:33,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 5: [2023-05-11 00:16:33,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 5: [2023-05-11 00:16:33,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +24: [2023-05-11 00:16:33,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +24: [2023-05-11 00:16:33,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +24: [2023-05-11 00:16:33,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +24: [2023-05-11 00:16:33,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +28: [2023-05-11 00:16:33,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +28: [2023-05-11 00:16:33,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +28: [2023-05-11 00:16:33,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +28: [2023-05-11 00:16:33,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 5: [2023-05-11 00:16:33,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +24: [2023-05-11 00:16:33,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +24: [2023-05-11 00:16:33,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +24: [2023-05-11 00:16:33,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +28: [2023-05-11 00:16:33,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +28: [2023-05-11 00:16:33,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +28: [2023-05-11 00:16:33,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +28: [2023-05-11 00:16:33,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +24: [2023-05-11 00:16:33,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +24: [2023-05-11 00:16:33,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +12: [2023-05-11 00:16:33,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +12: [2023-05-11 00:16:33,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +12: [2023-05-11 00:16:33,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +12: [2023-05-11 00:16:33,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +12: [2023-05-11 00:16:33,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +12: [2023-05-11 00:16:33,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +12: [2023-05-11 00:16:33,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +24: [2023-05-11 00:16:33,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +12: [2023-05-11 00:16:33,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +24: [2023-05-11 00:16:33,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +28: [2023-05-11 00:16:33,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +28: [2023-05-11 00:16:33,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +13: [2023-05-11 00:16:33,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +13: [2023-05-11 00:16:33,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +13: [2023-05-11 00:16:33,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +13: [2023-05-11 00:16:33,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +23: [2023-05-11 00:16:33,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +23: [2023-05-11 00:16:33,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +24: [2023-05-11 00:16:33,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +23: [2023-05-11 00:16:33,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +29: [2023-05-11 00:16:33,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +29: [2023-05-11 00:16:33,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +23: [2023-05-11 00:16:33,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +29: [2023-05-11 00:16:33,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +29: [2023-05-11 00:16:33,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +13: [2023-05-11 00:16:33,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +13: [2023-05-11 00:16:33,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +13: [2023-05-11 00:16:33,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +13: [2023-05-11 00:16:33,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +24: [2023-05-11 00:16:33,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +24: [2023-05-11 00:16:33,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +24: [2023-05-11 00:16:33,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +24: [2023-05-11 00:16:33,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 8: [2023-05-11 00:16:33,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +13: [2023-05-11 00:16:33,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +13: [2023-05-11 00:16:33,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +28: [2023-05-11 00:16:33,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +28: [2023-05-11 00:16:33,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +13: [2023-05-11 00:16:33,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +13: [2023-05-11 00:16:33,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +12: [2023-05-11 00:16:33,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +19: [2023-05-11 00:16:33,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +19: [2023-05-11 00:16:33,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +19: [2023-05-11 00:16:33,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +19: [2023-05-11 00:16:33,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +12: [2023-05-11 00:16:33,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +12: [2023-05-11 00:16:33,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +12: [2023-05-11 00:16:33,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +29: [2023-05-11 00:16:33,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +14: [2023-05-11 00:16:33,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +14: [2023-05-11 00:16:33,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +14: [2023-05-11 00:16:33,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +14: [2023-05-11 00:16:33,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +13: [2023-05-11 00:16:33,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +14: [2023-05-11 00:16:33,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +14: [2023-05-11 00:16:33,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +10: [2023-05-11 00:16:33,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +10: [2023-05-11 00:16:33,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 8: [2023-05-11 00:16:33,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +29: [2023-05-11 00:16:33,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +23: [2023-05-11 00:16:33,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +23: [2023-05-11 00:16:33,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +23: [2023-05-11 00:16:33,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +29: [2023-05-11 00:16:33,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +29: [2023-05-11 00:16:33,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +24: [2023-05-11 00:16:33,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +14: [2023-05-11 00:16:33,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +24: [2023-05-11 00:16:33,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +24: [2023-05-11 00:16:33,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +13: [2023-05-11 00:16:33,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +13: [2023-05-11 00:16:33,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +14: [2023-05-11 00:16:33,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +19: [2023-05-11 00:16:33,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +19: [2023-05-11 00:16:33,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +19: [2023-05-11 00:16:33,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +19: [2023-05-11 00:16:33,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +13: [2023-05-11 00:16:33,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +12: [2023-05-11 00:16:33,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +12: [2023-05-11 00:16:33,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +22: [2023-05-11 00:16:33,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +22: [2023-05-11 00:16:33,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +28: [2023-05-11 00:16:33,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +19: [2023-05-11 00:16:33,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +19: [2023-05-11 00:16:33,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +23: [2023-05-11 00:16:33,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 9: [2023-05-11 00:16:33,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 9: [2023-05-11 00:16:33,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +22: [2023-05-11 00:16:33,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +22: [2023-05-11 00:16:33,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +24: [2023-05-11 00:16:33,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +28: [2023-05-11 00:16:33,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +19: [2023-05-11 00:16:33,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +10: [2023-05-11 00:16:33,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +10: [2023-05-11 00:16:33,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +22: [2023-05-11 00:16:33,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +22: [2023-05-11 00:16:33,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +28: [2023-05-11 00:16:33,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +19: [2023-05-11 00:16:33,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +22: [2023-05-11 00:16:33,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +22: [2023-05-11 00:16:33,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +12: [2023-05-11 00:16:33,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +28: [2023-05-11 00:16:33,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +10: [2023-05-11 00:16:33,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +13: [2023-05-11 00:16:33,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +12: [2023-05-11 00:16:33,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +12: [2023-05-11 00:16:33,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +16: [2023-05-11 00:16:33,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +12: [2023-05-11 00:16:33,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +14: [2023-05-11 00:16:33,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +12: [2023-05-11 00:16:33,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +26: [2023-05-11 00:16:33,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +26: [2023-05-11 00:16:33,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +26: [2023-05-11 00:16:33,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +26: [2023-05-11 00:16:33,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +19: [2023-05-11 00:16:33,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +26: [2023-05-11 00:16:33,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +26: [2023-05-11 00:16:33,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +26: [2023-05-11 00:16:33,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +10: [2023-05-11 00:16:33,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +16: [2023-05-11 00:16:33,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +16: [2023-05-11 00:16:33,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +12: [2023-05-11 00:16:33,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +12: [2023-05-11 00:16:33,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +16: [2023-05-11 00:16:33,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +29: [2023-05-11 00:16:33,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +26: [2023-05-11 00:16:33,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +29: [2023-05-11 00:16:33,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +24: [2023-05-11 00:16:33,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +24: [2023-05-11 00:16:33,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 9: [2023-05-11 00:16:33,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +13: [2023-05-11 00:16:33,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +22: [2023-05-11 00:16:33,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +10: [2023-05-11 00:16:33,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +10: [2023-05-11 00:16:33,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +19: [2023-05-11 00:16:33,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +14: [2023-05-11 00:16:33,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +24: [2023-05-11 00:16:33,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +24: [2023-05-11 00:16:33,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 6: [2023-05-11 00:16:33,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +29: [2023-05-11 00:16:33,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +29: [2023-05-11 00:16:33,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +22: [2023-05-11 00:16:33,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +19: [2023-05-11 00:16:33,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +20: [2023-05-11 00:16:33,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +20: [2023-05-11 00:16:33,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +13: [2023-05-11 00:16:33,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +19: [2023-05-11 00:16:33,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +19: [2023-05-11 00:16:33,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +14: [2023-05-11 00:16:33,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +13: [2023-05-11 00:16:33,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +13: [2023-05-11 00:16:33,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +13: [2023-05-11 00:16:33,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +12: [2023-05-11 00:16:33,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +20: [2023-05-11 00:16:33,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +20: [2023-05-11 00:16:33,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +21: [2023-05-11 00:16:33,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +21: [2023-05-11 00:16:33,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +20: [2023-05-11 00:16:33,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +20: [2023-05-11 00:16:33,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +21: [2023-05-11 00:16:33,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +21: [2023-05-11 00:16:33,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +21: [2023-05-11 00:16:33,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +21: [2023-05-11 00:16:33,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +21: [2023-05-11 00:16:33,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +16: [2023-05-11 00:16:33,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +21: [2023-05-11 00:16:33,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +16: [2023-05-11 00:16:33,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +19: [2023-05-11 00:16:33,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +13: [2023-05-11 00:16:33,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +20: [2023-05-11 00:16:33,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +20: [2023-05-11 00:16:33,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +13: [2023-05-11 00:16:33,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +13: [2023-05-11 00:16:33,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 6: [2023-05-11 00:16:33,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +29: [2023-05-11 00:16:33,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +19: [2023-05-11 00:16:33,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +29: [2023-05-11 00:16:33,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 9: [2023-05-11 00:16:33,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +16: [2023-05-11 00:16:33,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +12: [2023-05-11 00:16:33,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +13: [2023-05-11 00:16:33,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +13: [2023-05-11 00:16:33,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +13: [2023-05-11 00:16:33,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +27: [2023-05-11 00:16:33,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +22: [2023-05-11 00:16:33,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +27: [2023-05-11 00:16:33,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +27: [2023-05-11 00:16:33,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +27: [2023-05-11 00:16:33,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +16: [2023-05-11 00:16:33,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +27: [2023-05-11 00:16:33,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +27: [2023-05-11 00:16:33,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +19: [2023-05-11 00:16:33,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +24: [2023-05-11 00:16:33,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +27: [2023-05-11 00:16:33,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +22: [2023-05-11 00:16:33,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +19: [2023-05-11 00:16:33,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +24: [2023-05-11 00:16:33,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 9: [2023-05-11 00:16:33,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +12: [2023-05-11 00:16:33,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +27: [2023-05-11 00:16:33,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +12: [2023-05-11 00:16:33,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +29: [2023-05-11 00:16:33,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +19: [2023-05-11 00:16:33,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +14: [2023-05-11 00:16:33,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +29: [2023-05-11 00:16:33,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +24: [2023-05-11 00:16:33,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +24: [2023-05-11 00:16:33,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +13: [2023-05-11 00:16:33,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +22: [2023-05-11 00:16:33,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +21: [2023-05-11 00:16:33,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +20: [2023-05-11 00:16:33,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +12: [2023-05-11 00:16:33,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +11: [2023-05-11 00:16:33,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +11: [2023-05-11 00:16:33,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +11: [2023-05-11 00:16:33,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +11: [2023-05-11 00:16:33,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +11: [2023-05-11 00:16:33,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +11: [2023-05-11 00:16:33,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +11: [2023-05-11 00:16:33,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +11: [2023-05-11 00:16:33,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +13: [2023-05-11 00:16:33,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +20: [2023-05-11 00:16:33,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +12: [2023-05-11 00:16:33,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +12: [2023-05-11 00:16:33,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +14: [2023-05-11 00:16:33,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +14: [2023-05-11 00:16:33,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +26: [2023-05-11 00:16:33,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +26: [2023-05-11 00:16:33,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +26: [2023-05-11 00:16:33,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +22: [2023-05-11 00:16:33,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +17: [2023-05-11 00:16:33,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +17: [2023-05-11 00:16:33,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +17: [2023-05-11 00:16:33,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +31: [2023-05-11 00:16:33,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +17: [2023-05-11 00:16:33,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +31: [2023-05-11 00:16:33,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +17: [2023-05-11 00:16:33,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +17: [2023-05-11 00:16:33,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +17: [2023-05-11 00:16:33,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +17: [2023-05-11 00:16:33,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +16: [2023-05-11 00:16:33,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +16: [2023-05-11 00:16:33,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +31: [2023-05-11 00:16:33,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +12: [2023-05-11 00:16:33,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +16: [2023-05-11 00:16:33,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +16: [2023-05-11 00:16:33,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +16: [2023-05-11 00:16:33,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +16: [2023-05-11 00:16:33,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +27: [2023-05-11 00:16:33,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +27: [2023-05-11 00:16:33,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +27: [2023-05-11 00:16:33,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +12: [2023-05-11 00:16:33,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +22: [2023-05-11 00:16:33,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +19: [2023-05-11 00:16:33,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +23: [2023-05-11 00:16:33,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +23: [2023-05-11 00:16:33,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +23: [2023-05-11 00:16:33,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +23: [2023-05-11 00:16:33,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +23: [2023-05-11 00:16:33,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +23: [2023-05-11 00:16:33,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +23: [2023-05-11 00:16:33,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +23: [2023-05-11 00:16:33,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +23: [2023-05-11 00:16:33,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +21: [2023-05-11 00:16:33,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +21: [2023-05-11 00:16:33,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +12: [2023-05-11 00:16:33,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +23: [2023-05-11 00:16:33,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +23: [2023-05-11 00:16:33,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +23: [2023-05-11 00:16:33,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +12: [2023-05-11 00:16:33,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +23: [2023-05-11 00:16:33,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +23: [2023-05-11 00:16:33,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +21: [2023-05-11 00:16:33,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +19: [2023-05-11 00:16:33,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +31: [2023-05-11 00:16:33,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +31: [2023-05-11 00:16:33,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +19: [2023-05-11 00:16:33,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +31: [2023-05-11 00:16:33,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +31: [2023-05-11 00:16:33,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +13: [2023-05-11 00:16:33,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +23: [2023-05-11 00:16:33,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +13: [2023-05-11 00:16:33,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +23: [2023-05-11 00:16:33,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +21: [2023-05-11 00:16:33,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +18: [2023-05-11 00:16:33,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +18: [2023-05-11 00:16:33,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +18: [2023-05-11 00:16:33,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +18: [2023-05-11 00:16:33,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +19: [2023-05-11 00:16:33,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +31: [2023-05-11 00:16:33,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +20: [2023-05-11 00:16:33,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +18: [2023-05-11 00:16:33,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +18: [2023-05-11 00:16:33,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +18: [2023-05-11 00:16:33,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +14: [2023-05-11 00:16:33,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +20: [2023-05-11 00:16:33,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +14: [2023-05-11 00:16:33,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +11: [2023-05-11 00:16:33,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +22: [2023-05-11 00:16:33,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +11: [2023-05-11 00:16:33,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +18: [2023-05-11 00:16:33,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +26: [2023-05-11 00:16:33,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +26: [2023-05-11 00:16:33,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 2: [2023-05-11 00:16:33,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 2: [2023-05-11 00:16:33,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 2: [2023-05-11 00:16:33,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 8: [2023-05-11 00:16:33,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 8: [2023-05-11 00:16:33,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 8: [2023-05-11 00:16:33,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +19: [2023-05-11 00:16:33,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +27: [2023-05-11 00:16:33,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +11: [2023-05-11 00:16:33,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +11: [2023-05-11 00:16:33,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +26: [2023-05-11 00:16:33,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +19: [2023-05-11 00:16:33,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +19: [2023-05-11 00:16:33,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +21: [2023-05-11 00:16:33,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +17: [2023-05-11 00:16:33,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +17: [2023-05-11 00:16:33,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +20: [2023-05-11 00:16:33,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +21: [2023-05-11 00:16:33,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +16: [2023-05-11 00:16:33,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +19: [2023-05-11 00:16:33,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +17: [2023-05-11 00:16:33,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +21: [2023-05-11 00:16:33,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +27: [2023-05-11 00:16:33,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +27: [2023-05-11 00:16:33,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +31: [2023-05-11 00:16:33,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +19: [2023-05-11 00:16:33,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +19: [2023-05-11 00:16:33,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +16: [2023-05-11 00:16:33,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +23: [2023-05-11 00:16:33,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +26: [2023-05-11 00:16:33,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +10: [2023-05-11 00:16:33,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +10: [2023-05-11 00:16:33,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +10: [2023-05-11 00:16:33,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +10: [2023-05-11 00:16:33,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +20: [2023-05-11 00:16:33,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +23: [2023-05-11 00:16:33,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +10: [2023-05-11 00:16:33,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +10: [2023-05-11 00:16:33,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +10: [2023-05-11 00:16:33,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +10: [2023-05-11 00:16:33,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +11: [2023-05-11 00:16:33,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +10: [2023-05-11 00:16:33,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +23: [2023-05-11 00:16:33,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +10: [2023-05-11 00:16:33,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +10: [2023-05-11 00:16:33,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +10: [2023-05-11 00:16:33,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +10: [2023-05-11 00:16:33,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +10: [2023-05-11 00:16:33,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +10: [2023-05-11 00:16:33,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +31: [2023-05-11 00:16:33,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +18: [2023-05-11 00:16:33,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +11: [2023-05-11 00:16:33,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +10: [2023-05-11 00:16:33,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +27: [2023-05-11 00:16:33,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +11: [2023-05-11 00:16:33,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +11: [2023-05-11 00:16:33,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +18: [2023-05-11 00:16:33,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +18: [2023-05-11 00:16:33,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +17: [2023-05-11 00:16:33,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +20: [2023-05-11 00:16:33,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +23: [2023-05-11 00:16:33,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +26: [2023-05-11 00:16:33,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +31: [2023-05-11 00:16:33,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +17: [2023-05-11 00:16:33,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +14: [2023-05-11 00:16:33,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +14: [2023-05-11 00:16:33,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +14: [2023-05-11 00:16:33,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 1: [2023-05-11 00:16:33,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +20: [2023-05-11 00:16:33,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +14: [2023-05-11 00:16:33,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +17: [2023-05-11 00:16:33,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +31: [2023-05-11 00:16:33,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +10: [2023-05-11 00:16:33,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +23: [2023-05-11 00:16:33,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +23: [2023-05-11 00:16:33,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +27: [2023-05-11 00:16:33,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +17: [2023-05-11 00:16:33,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +10: [2023-05-11 00:16:33,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +31: [2023-05-11 00:16:33,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +18: [2023-05-11 00:16:33,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +18: [2023-05-11 00:16:33,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +18: [2023-05-11 00:16:33,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +25: [2023-05-11 00:16:33,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +25: [2023-05-11 00:16:33,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 7: [2023-05-11 00:16:33,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +10: [2023-05-11 00:16:33,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +23: [2023-05-11 00:16:33,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +25: [2023-05-11 00:16:33,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +25: [2023-05-11 00:16:33,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 8: [2023-05-11 00:16:33,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +26: [2023-05-11 00:16:33,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +26: [2023-05-11 00:16:33,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +26: [2023-05-11 00:16:33,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +14: [2023-05-11 00:16:33,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +26: [2023-05-11 00:16:33,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +31: [2023-05-11 00:16:33,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +14: [2023-05-11 00:16:33,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +17: [2023-05-11 00:16:33,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +14: [2023-05-11 00:16:33,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +14: [2023-05-11 00:16:33,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +10: [2023-05-11 00:16:33,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +31: [2023-05-11 00:16:33,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +18: [2023-05-11 00:16:33,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +10: [2023-05-11 00:16:33,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +25: [2023-05-11 00:16:33,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +23: [2023-05-11 00:16:33,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 7: [2023-05-11 00:16:33,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +25: [2023-05-11 00:16:33,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +26: [2023-05-11 00:16:33,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +26: [2023-05-11 00:16:33,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +10: [2023-05-11 00:16:33,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +10: [2023-05-11 00:16:33,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +26: [2023-05-11 00:16:33,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +25: [2023-05-11 00:16:33,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +25: [2023-05-11 00:16:33,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +31: [2023-05-11 00:16:33,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 2: [2023-05-11 00:16:33,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +26: [2023-05-11 00:16:33,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +18: [2023-05-11 00:16:33,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +18: [2023-05-11 00:16:33,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +18: [2023-05-11 00:16:33,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +18: [2023-05-11 00:16:33,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +18: [2023-05-11 00:16:33,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 8: [2023-05-11 00:16:33,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +14: [2023-05-11 00:16:33,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +14: [2023-05-11 00:16:33,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 8: [2023-05-11 00:16:33,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +10: [2023-05-11 00:16:33,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +14: [2023-05-11 00:16:33,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +14: [2023-05-11 00:16:33,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +20: [2023-05-11 00:16:33,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +20: [2023-05-11 00:16:33,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 1: [2023-05-11 00:16:33,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +25: [2023-05-11 00:16:33,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +26: [2023-05-11 00:16:33,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +26: [2023-05-11 00:16:33,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +25: [2023-05-11 00:16:33,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +20: [2023-05-11 00:16:33,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 1: [2023-05-11 00:16:33,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +20: [2023-05-11 00:16:33,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +25: [2023-05-11 00:16:33,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +25: [2023-05-11 00:16:33,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 2: [2023-05-11 00:16:33,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +15: [2023-05-11 00:16:33,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +15: [2023-05-11 00:16:33,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +15: [2023-05-11 00:16:33,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +15: [2023-05-11 00:16:33,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +26: [2023-05-11 00:16:33,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +26: [2023-05-11 00:16:33,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +18: [2023-05-11 00:16:33,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +15: [2023-05-11 00:16:33,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +30: [2023-05-11 00:16:33,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +30: [2023-05-11 00:16:33,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +15: [2023-05-11 00:16:33,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +30: [2023-05-11 00:16:33,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +30: [2023-05-11 00:16:33,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +26: [2023-05-11 00:16:33,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +26: [2023-05-11 00:16:33,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +18: [2023-05-11 00:16:33,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +25: [2023-05-11 00:16:33,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +25: [2023-05-11 00:16:33,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +15: [2023-05-11 00:16:33,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +15: [2023-05-11 00:16:33,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +14: [2023-05-11 00:16:33,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +14: [2023-05-11 00:16:33,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +14: [2023-05-11 00:16:33,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +18: [2023-05-11 00:16:33,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +18: [2023-05-11 00:16:33,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +20: [2023-05-11 00:16:33,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +14: [2023-05-11 00:16:33,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +17: [2023-05-11 00:16:33,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +17: [2023-05-11 00:16:33,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +17: [2023-05-11 00:16:33,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +17: [2023-05-11 00:16:33,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +20: [2023-05-11 00:16:33,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +20: [2023-05-11 00:16:33,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +20: [2023-05-11 00:16:33,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +22: [2023-05-11 00:16:33,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +22: [2023-05-11 00:16:33,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +28: [2023-05-11 00:16:33,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +28: [2023-05-11 00:16:33,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +30: [2023-05-11 00:16:33,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +15: [2023-05-11 00:16:33,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +30: [2023-05-11 00:16:33,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +30: [2023-05-11 00:16:33,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +22: [2023-05-11 00:16:33,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +30: [2023-05-11 00:16:33,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +30: [2023-05-11 00:16:33,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +30: [2023-05-11 00:16:33,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +30: [2023-05-11 00:16:33,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +22: [2023-05-11 00:16:33,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +25: [2023-05-11 00:16:33,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +25: [2023-05-11 00:16:33,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +25: [2023-05-11 00:16:33,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +25: [2023-05-11 00:16:33,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +15: [2023-05-11 00:16:33,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +30: [2023-05-11 00:16:33,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +29: [2023-05-11 00:16:33,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +29: [2023-05-11 00:16:33,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +29: [2023-05-11 00:16:33,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +29: [2023-05-11 00:16:33,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +25: [2023-05-11 00:16:33,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +25: [2023-05-11 00:16:33,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +25: [2023-05-11 00:16:33,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +29: [2023-05-11 00:16:33,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +30: [2023-05-11 00:16:33,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +29: [2023-05-11 00:16:33,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +30: [2023-05-11 00:16:33,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +29: [2023-05-11 00:16:33,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +25: [2023-05-11 00:16:33,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +25: [2023-05-11 00:16:33,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +28: [2023-05-11 00:16:33,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +17: [2023-05-11 00:16:33,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +29: [2023-05-11 00:16:33,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +25: [2023-05-11 00:16:33,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +17: [2023-05-11 00:16:33,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +17: [2023-05-11 00:16:33,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 4: [2023-05-11 00:16:33,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +17: [2023-05-11 00:16:33,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +28: [2023-05-11 00:16:33,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +26: [2023-05-11 00:16:33,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +28: [2023-05-11 00:16:33,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 4: [2023-05-11 00:16:33,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 1: [2023-05-11 00:16:33,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +28: [2023-05-11 00:16:33,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +18: [2023-05-11 00:16:33,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +22: [2023-05-11 00:16:33,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +26: [2023-05-11 00:16:33,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +18: [2023-05-11 00:16:33,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +22: [2023-05-11 00:16:33,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +22: [2023-05-11 00:16:33,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +15: [2023-05-11 00:16:33,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +30: [2023-05-11 00:16:33,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +30: [2023-05-11 00:16:33,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +22: [2023-05-11 00:16:33,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +30: [2023-05-11 00:16:33,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 5: [2023-05-11 00:16:33,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +15: [2023-05-11 00:16:33,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +18: [2023-05-11 00:16:33,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +28: [2023-05-11 00:16:33,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +20: [2023-05-11 00:16:33,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +20: [2023-05-11 00:16:33,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +25: [2023-05-11 00:16:33,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +20: [2023-05-11 00:16:33,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +20: [2023-05-11 00:16:33,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 5: [2023-05-11 00:16:33,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +15: [2023-05-11 00:16:33,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +20: [2023-05-11 00:16:33,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +20: [2023-05-11 00:16:33,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 5: [2023-05-11 00:16:33,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +18: [2023-05-11 00:16:33,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +28: [2023-05-11 00:16:33,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +30: [2023-05-11 00:16:33,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +30: [2023-05-11 00:16:33,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +18: [2023-05-11 00:16:33,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +11: [2023-05-11 00:16:33,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +11: [2023-05-11 00:16:33,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +11: [2023-05-11 00:16:33,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +11: [2023-05-11 00:16:33,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +27: [2023-05-11 00:16:33,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +27: [2023-05-11 00:16:33,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +27: [2023-05-11 00:16:33,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +27: [2023-05-11 00:16:33,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +25: [2023-05-11 00:16:33,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +25: [2023-05-11 00:16:33,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +25: [2023-05-11 00:16:33,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +17: [2023-05-11 00:16:33,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +17: [2023-05-11 00:16:33,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +18: [2023-05-11 00:16:33,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +29: [2023-05-11 00:16:33,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +15: [2023-05-11 00:16:33,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +17: [2023-05-11 00:16:33,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +18: [2023-05-11 00:16:33,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +17: [2023-05-11 00:16:33,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 4: [2023-05-11 00:16:33,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +22: [2023-05-11 00:16:33,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +17: [2023-05-11 00:16:33,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +22: [2023-05-11 00:16:33,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +17: [2023-05-11 00:16:33,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +18: [2023-05-11 00:16:33,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +22: [2023-05-11 00:16:33,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +22: [2023-05-11 00:16:33,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +17: [2023-05-11 00:16:33,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +17: [2023-05-11 00:16:33,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +29: [2023-05-11 00:16:33,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +29: [2023-05-11 00:16:33,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +22: [2023-05-11 00:16:33,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +22: [2023-05-11 00:16:33,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +30: [2023-05-11 00:16:33,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +30: [2023-05-11 00:16:33,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +30: [2023-05-11 00:16:33,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +30: [2023-05-11 00:16:33,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +15: [2023-05-11 00:16:33,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +11: [2023-05-11 00:16:33,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +11: [2023-05-11 00:16:33,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +11: [2023-05-11 00:16:33,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +30: [2023-05-11 00:16:33,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +25: [2023-05-11 00:16:33,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +27: [2023-05-11 00:16:33,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +25: [2023-05-11 00:16:33,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +29: [2023-05-11 00:16:33,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +27: [2023-05-11 00:16:33,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +27: [2023-05-11 00:16:33,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +15: [2023-05-11 00:16:33,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +20: [2023-05-11 00:16:33,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +11: [2023-05-11 00:16:33,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +20: [2023-05-11 00:16:33,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +25: [2023-05-11 00:16:33,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +27: [2023-05-11 00:16:33,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +25: [2023-05-11 00:16:33,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +30: [2023-05-11 00:16:33,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +30: [2023-05-11 00:16:33,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 0: [2023-05-11 00:16:33,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 7: [2023-05-11 00:16:33,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +21: [2023-05-11 00:16:33,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 7: [2023-05-11 00:16:33,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +29: [2023-05-11 00:16:33,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +29: [2023-05-11 00:16:33,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +22: [2023-05-11 00:16:33,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +21: [2023-05-11 00:16:33,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 7: [2023-05-11 00:16:33,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +22: [2023-05-11 00:16:33,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +21: [2023-05-11 00:16:33,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 3: [2023-05-11 00:16:33,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 3: [2023-05-11 00:16:33,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +21: [2023-05-11 00:16:33,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +30: [2023-05-11 00:16:33,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 6: [2023-05-11 00:16:33,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 6: [2023-05-11 00:16:33,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 6: [2023-05-11 00:16:33,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 3: [2023-05-11 00:16:33,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 3: [2023-05-11 00:16:33,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +31: [2023-05-11 00:16:33,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +31: [2023-05-11 00:16:33,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +31: [2023-05-11 00:16:33,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +31: [2023-05-11 00:16:33,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 7: [2023-05-11 00:16:33,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +11: [2023-05-11 00:16:33,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +21: [2023-05-11 00:16:33,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +27: [2023-05-11 00:16:33,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +27: [2023-05-11 00:16:33,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 4: [2023-05-11 00:16:33,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 4: [2023-05-11 00:16:33,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +29: [2023-05-11 00:16:33,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +11: [2023-05-11 00:16:33,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +11: [2023-05-11 00:16:33,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +11: [2023-05-11 00:16:33,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +11: [2023-05-11 00:16:33,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +11: [2023-05-11 00:16:33,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +21: [2023-05-11 00:16:33,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +21: [2023-05-11 00:16:33,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +21: [2023-05-11 00:16:33,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 5: [2023-05-11 00:16:33,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +30: [2023-05-11 00:16:33,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +30: [2023-05-11 00:16:33,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 5: [2023-05-11 00:16:33,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +30: [2023-05-11 00:16:33,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +15: [2023-05-11 00:16:33,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +30: [2023-05-11 00:16:33,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +15: [2023-05-11 00:16:33,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +31: [2023-05-11 00:16:33,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +31: [2023-05-11 00:16:33,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +11: [2023-05-11 00:16:33,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +11: [2023-05-11 00:16:33,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +30: [2023-05-11 00:16:33,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +15: [2023-05-11 00:16:33,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +16: [2023-05-11 00:16:33,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +16: [2023-05-11 00:16:33,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +16: [2023-05-11 00:16:33,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +16: [2023-05-11 00:16:33,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +16: [2023-05-11 00:16:33,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +16: [2023-05-11 00:16:33,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +15: [2023-05-11 00:16:33,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +27: [2023-05-11 00:16:33,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +27: [2023-05-11 00:16:33,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +31: [2023-05-11 00:16:33,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +27: [2023-05-11 00:16:33,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +27: [2023-05-11 00:16:33,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +16: [2023-05-11 00:16:33,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +16: [2023-05-11 00:16:33,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +29: [2023-05-11 00:16:33,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +31: [2023-05-11 00:16:33,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +29: [2023-05-11 00:16:33,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +29: [2023-05-11 00:16:33,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +29: [2023-05-11 00:16:33,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 4: [2023-05-11 00:16:33,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 4: [2023-05-11 00:16:33,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +15: [2023-05-11 00:16:33,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +21: [2023-05-11 00:16:33,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +29: [2023-05-11 00:16:33,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +21: [2023-05-11 00:16:33,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 7: [2023-05-11 00:16:33,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +21: [2023-05-11 00:16:33,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 7: [2023-05-11 00:16:33,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +21: [2023-05-11 00:16:33,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +15: [2023-05-11 00:16:33,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +21: [2023-05-11 00:16:33,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +21: [2023-05-11 00:16:33,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 7: [2023-05-11 00:16:33,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 7: [2023-05-11 00:16:33,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +15: [2023-05-11 00:16:33,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +31: [2023-05-11 00:16:33,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 5: [2023-05-11 00:16:33,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +15: [2023-05-11 00:16:33,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +31: [2023-05-11 00:16:33,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 3: [2023-05-11 00:16:33,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +16: [2023-05-11 00:16:33,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +29: [2023-05-11 00:16:33,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 3: [2023-05-11 00:16:33,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 3: [2023-05-11 00:16:33,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 5: [2023-05-11 00:16:33,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +27: [2023-05-11 00:16:33,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 6: [2023-05-11 00:16:33,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 6: [2023-05-11 00:16:33,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +29: [2023-05-11 00:16:33,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +27: [2023-05-11 00:16:33,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +29: [2023-05-11 00:16:33,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +29: [2023-05-11 00:16:33,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +21: [2023-05-11 00:16:33,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +21: [2023-05-11 00:16:33,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +16: [2023-05-11 00:16:33,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +31: [2023-05-11 00:16:33,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +31: [2023-05-11 00:16:33,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +31: [2023-05-11 00:16:33,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +31: [2023-05-11 00:16:33,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 4: [2023-05-11 00:16:33,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +16: [2023-05-11 00:16:33,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +16: [2023-05-11 00:16:33,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +24: [2023-05-11 00:16:33,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 5: [2023-05-11 00:16:33,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +16: [2023-05-11 00:16:33,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +15: [2023-05-11 00:16:33,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. +24: [2023-05-11 00:16:33,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +15: [2023-05-11 00:16:33,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +16: [2023-05-11 00:16:33,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +16: [2023-05-11 00:16:33,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +15: [2023-05-11 00:16:33,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +15: [2023-05-11 00:16:33,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 5: [2023-05-11 00:16:33,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +15: [2023-05-11 00:16:33,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +15: [2023-05-11 00:16:33,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 4: [2023-05-11 00:16:33,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +16: [2023-05-11 00:16:33,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +16: [2023-05-11 00:16:33,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +15: [2023-05-11 00:16:33,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +15: [2023-05-11 00:16:33,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 4: [2023-05-11 00:16:33,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +24: [2023-05-11 00:16:33,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +24: [2023-05-11 00:16:33,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +29: [2023-05-11 00:16:33,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +29: [2023-05-11 00:16:33,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 5: [2023-05-11 00:16:33,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +12: [2023-05-11 00:16:33,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +12: [2023-05-11 00:16:33,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +16: [2023-05-11 00:16:33,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +12: [2023-05-11 00:16:33,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +24: [2023-05-11 00:16:33,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +29: [2023-05-11 00:16:33,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +29: [2023-05-11 00:16:33,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +16: [2023-05-11 00:16:33,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +29: [2023-05-11 00:16:33,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +29: [2023-05-11 00:16:33,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +12: [2023-05-11 00:16:33,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +12: [2023-05-11 00:16:33,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +12: [2023-05-11 00:16:33,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +12: [2023-05-11 00:16:33,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +16: [2023-05-11 00:16:33,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +12: [2023-05-11 00:16:33,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +16: [2023-05-11 00:16:33,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +16: [2023-05-11 00:16:33,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +29: [2023-05-11 00:16:33,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +29: [2023-05-11 00:16:33,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +24: [2023-05-11 00:16:33,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +16: [2023-05-11 00:16:33,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +31: [2023-05-11 00:16:33,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +24: [2023-05-11 00:16:33,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +16: [2023-05-11 00:16:33,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +24: [2023-05-11 00:16:33,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +31: [2023-05-11 00:16:33,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 8: [2023-05-11 00:16:33,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +12: [2023-05-11 00:16:33,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +12: [2023-05-11 00:16:33,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +16: [2023-05-11 00:16:33,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +16: [2023-05-11 00:16:33,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +16: [2023-05-11 00:16:33,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +16: [2023-05-11 00:16:33,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +16: [2023-05-11 00:16:33,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +16: [2023-05-11 00:16:33,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +12: [2023-05-11 00:16:33,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +24: [2023-05-11 00:16:33,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +24: [2023-05-11 00:16:33,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +24: [2023-05-11 00:16:33,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +24: [2023-05-11 00:16:33,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +12: [2023-05-11 00:16:33,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +24: [2023-05-11 00:16:33,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +24: [2023-05-11 00:16:33,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 8: [2023-05-11 00:16:33,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +24: [2023-05-11 00:16:33,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +24: [2023-05-11 00:16:33,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +24: [2023-05-11 00:16:33,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +24: [2023-05-11 00:16:33,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +12: [2023-05-11 00:16:33,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +24: [2023-05-11 00:16:33,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +24: [2023-05-11 00:16:33,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +24: [2023-05-11 00:16:33,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +24: [2023-05-11 00:16:33,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +24: [2023-05-11 00:16:33,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +24: [2023-05-11 00:16:33,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +16: [2023-05-11 00:16:33,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +12: [2023-05-11 00:16:33,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +16: [2023-05-11 00:16:33,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 0: [2023-05-11 00:16:33,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 0: [2023-05-11 00:16:33,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +14: [2023-05-11 00:16:33,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +14: [2023-05-11 00:16:33,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +24: [2023-05-11 00:16:33,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. + 3: [2023-05-11 00:16:33,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt. +14: [2023-05-11 00:16:33,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 2: [2023-05-11 00:16:33,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 2: [2023-05-11 00:16:33,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +28: [2023-05-11 00:16:33,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +28: [2023-05-11 00:16:33,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +28: [2023-05-11 00:16:33,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +28: [2023-05-11 00:16:33,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +28: [2023-05-11 00:16:33,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +28: [2023-05-11 00:16:33,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +28: [2023-05-11 00:16:33,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +28: [2023-05-11 00:16:33,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +28: [2023-05-11 00:16:33,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +28: [2023-05-11 00:16:33,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +28: [2023-05-11 00:16:33,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +28: [2023-05-11 00:16:33,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +28: [2023-05-11 00:16:33,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +12: [2023-05-11 00:16:33,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +28: [2023-05-11 00:16:33,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +14: [2023-05-11 00:16:33,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 8: [2023-05-11 00:16:33,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +28: [2023-05-11 00:16:33,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +28: [2023-05-11 00:16:33,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +12: [2023-05-11 00:16:33,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 2: [2023-05-11 00:16:33,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +24: [2023-05-11 00:16:33,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +14: [2023-05-11 00:16:33,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +14: [2023-05-11 00:16:33,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... +24: [2023-05-11 00:16:33,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +24: [2023-05-11 00:16:33,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +14: [2023-05-11 00:16:33,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +14: [2023-05-11 00:16:33,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +24: [2023-05-11 00:16:33,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +28: [2023-05-11 00:16:33,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +23: [2023-05-11 00:16:33,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +23: [2023-05-11 00:16:33,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +23: [2023-05-11 00:16:33,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +23: [2023-05-11 00:16:33,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 3: [2023-05-11 00:16:33,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +24: [2023-05-11 00:16:33,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +24: [2023-05-11 00:16:33,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +28: [2023-05-11 00:16:33,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +28: [2023-05-11 00:16:33,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 8: [2023-05-11 00:16:33,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 8: [2023-05-11 00:16:33,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 0: [2023-05-11 00:16:33,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 8: [2023-05-11 00:16:33,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +24: [2023-05-11 00:16:33,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +21: [2023-05-11 00:16:33,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +21: [2023-05-11 00:16:33,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +23: [2023-05-11 00:16:33,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +18: [2023-05-11 00:16:33,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +23: [2023-05-11 00:16:33,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +22: [2023-05-11 00:16:33,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +23: [2023-05-11 00:16:33,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +22: [2023-05-11 00:16:33,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +23: [2023-05-11 00:16:33,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +28: [2023-05-11 00:16:33,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +22: [2023-05-11 00:16:33,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +22: [2023-05-11 00:16:33,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +28: [2023-05-11 00:16:33,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +28: [2023-05-11 00:16:33,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +21: [2023-05-11 00:16:33,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +21: [2023-05-11 00:16:33,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +21: [2023-05-11 00:16:33,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +21: [2023-05-11 00:16:33,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 0: [2023-05-11 00:16:33,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... +22: [2023-05-11 00:16:33,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +22: [2023-05-11 00:16:33,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +22: [2023-05-11 00:16:33,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +22: [2023-05-11 00:16:33,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +18: [2023-05-11 00:16:33,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +25: [2023-05-11 00:16:33,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +25: [2023-05-11 00:16:33,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +25: [2023-05-11 00:16:33,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +25: [2023-05-11 00:16:33,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +21: [2023-05-11 00:16:33,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +21: [2023-05-11 00:16:33,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +17: [2023-05-11 00:16:33,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +17: [2023-05-11 00:16:33,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +17: [2023-05-11 00:16:33,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +17: [2023-05-11 00:16:33,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 8: [2023-05-11 00:16:33,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +18: [2023-05-11 00:16:33,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +18: [2023-05-11 00:16:33,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 8: [2023-05-11 00:16:33,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +18: [2023-05-11 00:16:33,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +21: [2023-05-11 00:16:33,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt... + 2: [2023-05-11 00:16:33,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +20: [2023-05-11 00:16:33,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +20: [2023-05-11 00:16:33,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +20: [2023-05-11 00:16:33,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +28: [2023-05-11 00:16:33,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +20: [2023-05-11 00:16:33,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +13: [2023-05-11 00:16:33,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +13: [2023-05-11 00:16:33,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +13: [2023-05-11 00:16:33,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 2: [2023-05-11 00:16:33,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +25: [2023-05-11 00:16:33,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +22: [2023-05-11 00:16:33,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +25: [2023-05-11 00:16:33,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +25: [2023-05-11 00:16:33,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +26: [2023-05-11 00:16:33,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +26: [2023-05-11 00:16:33,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +18: [2023-05-11 00:16:33,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +13: [2023-05-11 00:16:33,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +13: [2023-05-11 00:16:33,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +13: [2023-05-11 00:16:33,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +13: [2023-05-11 00:16:33,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +13: [2023-05-11 00:16:33,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +22: [2023-05-11 00:16:33,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +13: [2023-05-11 00:16:33,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +13: [2023-05-11 00:16:33,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +18: [2023-05-11 00:16:33,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +25: [2023-05-11 00:16:33,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +26: [2023-05-11 00:16:33,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 6: [2023-05-11 00:16:33,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +18: [2023-05-11 00:16:33,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +17: [2023-05-11 00:16:33,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +21: [2023-05-11 00:16:33,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +26: [2023-05-11 00:16:33,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +17: [2023-05-11 00:16:33,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +22: [2023-05-11 00:16:33,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +14: [2023-05-11 00:16:33,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +14: [2023-05-11 00:16:33,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +14: [2023-05-11 00:16:33,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +14: [2023-05-11 00:16:33,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 2: [2023-05-11 00:16:33,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +13: [2023-05-11 00:16:33,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +20: [2023-05-11 00:16:33,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 2: [2023-05-11 00:16:33,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 6: [2023-05-11 00:16:33,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +17: [2023-05-11 00:16:33,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +14: [2023-05-11 00:16:33,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +14: [2023-05-11 00:16:33,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +14: [2023-05-11 00:16:33,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +26: [2023-05-11 00:16:33,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +14: [2023-05-11 00:16:33,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +14: [2023-05-11 00:16:33,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +20: [2023-05-11 00:16:33,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +14: [2023-05-11 00:16:33,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +21: [2023-05-11 00:16:33,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +26: [2023-05-11 00:16:33,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +28: [2023-05-11 00:16:33,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +20: [2023-05-11 00:16:33,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +26: [2023-05-11 00:16:33,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +26: [2023-05-11 00:16:33,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +13: [2023-05-11 00:16:33,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +17: [2023-05-11 00:16:33,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +14: [2023-05-11 00:16:33,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +14: [2023-05-11 00:16:33,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +14: [2023-05-11 00:16:33,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +14: [2023-05-11 00:16:33,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +14: [2023-05-11 00:16:33,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +13: [2023-05-11 00:16:33,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +22: [2023-05-11 00:16:33,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +26: [2023-05-11 00:16:33,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +26: [2023-05-11 00:16:33,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +26: [2023-05-11 00:16:33,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +13: [2023-05-11 00:16:33,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +14: [2023-05-11 00:16:33,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +13: [2023-05-11 00:16:33,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +26: [2023-05-11 00:16:33,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +20: [2023-05-11 00:16:33,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +26: [2023-05-11 00:16:33,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +22: [2023-05-11 00:16:33,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +13: [2023-05-11 00:16:33,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +22: [2023-05-11 00:16:33,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +22: [2023-05-11 00:16:33,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +22: [2023-05-11 00:16:33,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +26: [2023-05-11 00:16:33,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +21: [2023-05-11 00:16:33,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +22: [2023-05-11 00:16:33,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +26: [2023-05-11 00:16:33,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +26: [2023-05-11 00:16:33,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +22: [2023-05-11 00:16:33,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 8: [2023-05-11 00:16:33,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +25: [2023-05-11 00:16:33,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +22: [2023-05-11 00:16:33,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +25: [2023-05-11 00:16:33,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +14: [2023-05-11 00:16:33,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +25: [2023-05-11 00:16:33,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +25: [2023-05-11 00:16:33,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +22: [2023-05-11 00:16:33,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +20: [2023-05-11 00:16:33,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +20: [2023-05-11 00:16:33,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +20: [2023-05-11 00:16:33,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +20: [2023-05-11 00:16:33,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +20: [2023-05-11 00:16:33,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +20: [2023-05-11 00:16:33,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +21: [2023-05-11 00:16:33,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +20: [2023-05-11 00:16:33,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +20: [2023-05-11 00:16:33,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 2: [2023-05-11 00:16:33,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +13: [2023-05-11 00:16:33,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +13: [2023-05-11 00:16:33,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 6: [2023-05-11 00:16:33,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +20: [2023-05-11 00:16:33,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +13: [2023-05-11 00:16:33,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +13: [2023-05-11 00:16:33,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +22: [2023-05-11 00:16:33,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +13: [2023-05-11 00:16:33,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +22: [2023-05-11 00:16:33,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +25: [2023-05-11 00:16:33,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +14: [2023-05-11 00:16:33,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +22: [2023-05-11 00:16:33,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +25: [2023-05-11 00:16:33,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +20: [2023-05-11 00:16:33,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +22: [2023-05-11 00:16:33,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +14: [2023-05-11 00:16:33,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +26: [2023-05-11 00:16:33,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +13: [2023-05-11 00:16:33,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +13: [2023-05-11 00:16:33,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +25: [2023-05-11 00:16:33,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +13: [2023-05-11 00:16:33,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +13: [2023-05-11 00:16:33,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +20: [2023-05-11 00:16:33,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +14: [2023-05-11 00:16:33,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +20: [2023-05-11 00:16:33,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +26: [2023-05-11 00:16:33,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +21: [2023-05-11 00:16:33,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +26: [2023-05-11 00:16:33,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +26: [2023-05-11 00:16:33,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +25: [2023-05-11 00:16:33,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +21: [2023-05-11 00:16:33,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +26: [2023-05-11 00:16:33,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 2: [2023-05-11 00:16:33,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +26: [2023-05-11 00:16:33,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 2: [2023-05-11 00:16:33,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +13: [2023-05-11 00:16:33,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +11: [2023-05-11 00:16:33,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +20: [2023-05-11 00:16:33,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +20: [2023-05-11 00:16:33,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +11: [2023-05-11 00:16:33,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +13: [2023-05-11 00:16:33,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +13: [2023-05-11 00:16:33,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +20: [2023-05-11 00:16:33,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +13: [2023-05-11 00:16:33,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +20: [2023-05-11 00:16:33,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +17: [2023-05-11 00:16:33,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +17: [2023-05-11 00:16:33,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +17: [2023-05-11 00:16:33,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +17: [2023-05-11 00:16:33,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +17: [2023-05-11 00:16:33,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +17: [2023-05-11 00:16:33,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +17: [2023-05-11 00:16:33,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +17: [2023-05-11 00:16:33,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +17: [2023-05-11 00:16:33,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +17: [2023-05-11 00:16:33,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +26: [2023-05-11 00:16:33,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +17: [2023-05-11 00:16:33,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +21: [2023-05-11 00:16:33,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +26: [2023-05-11 00:16:33,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +14: [2023-05-11 00:16:33,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +17: [2023-05-11 00:16:33,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +17: [2023-05-11 00:16:33,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +11: [2023-05-11 00:16:33,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +11: [2023-05-11 00:16:33,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +17: [2023-05-11 00:16:33,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +13: [2023-05-11 00:16:33,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +13: [2023-05-11 00:16:33,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +14: [2023-05-11 00:16:33,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 2: [2023-05-11 00:16:33,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +22: [2023-05-11 00:16:33,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +17: [2023-05-11 00:16:33,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +22: [2023-05-11 00:16:33,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +22: [2023-05-11 00:16:33,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +22: [2023-05-11 00:16:33,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +17: [2023-05-11 00:16:33,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +11: [2023-05-11 00:16:33,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +11: [2023-05-11 00:16:33,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +11: [2023-05-11 00:16:33,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +26: [2023-05-11 00:16:33,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +26: [2023-05-11 00:16:33,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +26: [2023-05-11 00:16:33,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +11: [2023-05-11 00:16:33,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +20: [2023-05-11 00:16:33,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +20: [2023-05-11 00:16:33,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +19: [2023-05-11 00:16:33,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +19: [2023-05-11 00:16:33,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +20: [2023-05-11 00:16:33,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +22: [2023-05-11 00:16:33,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +22: [2023-05-11 00:16:33,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +27: [2023-05-11 00:16:33,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +27: [2023-05-11 00:16:33,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +14: [2023-05-11 00:16:33,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +19: [2023-05-11 00:16:33,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +27: [2023-05-11 00:16:33,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +11: [2023-05-11 00:16:33,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +27: [2023-05-11 00:16:33,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +26: [2023-05-11 00:16:33,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +13: [2023-05-11 00:16:33,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +19: [2023-05-11 00:16:33,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +26: [2023-05-11 00:16:33,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +22: [2023-05-11 00:16:33,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +22: [2023-05-11 00:16:33,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +19: [2023-05-11 00:16:33,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +19: [2023-05-11 00:16:33,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +27: [2023-05-11 00:16:33,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +27: [2023-05-11 00:16:33,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +27: [2023-05-11 00:16:33,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +26: [2023-05-11 00:16:33,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +17: [2023-05-11 00:16:33,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +27: [2023-05-11 00:16:33,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +23: [2023-05-11 00:16:33,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +17: [2023-05-11 00:16:33,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +23: [2023-05-11 00:16:33,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +17: [2023-05-11 00:16:33,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +23: [2023-05-11 00:16:33,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +23: [2023-05-11 00:16:33,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +23: [2023-05-11 00:16:33,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +23: [2023-05-11 00:16:33,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +19: [2023-05-11 00:16:33,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +31: [2023-05-11 00:16:33,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +31: [2023-05-11 00:16:33,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +31: [2023-05-11 00:16:33,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +31: [2023-05-11 00:16:33,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +23: [2023-05-11 00:16:33,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +23: [2023-05-11 00:16:33,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +23: [2023-05-11 00:16:33,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +23: [2023-05-11 00:16:33,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +11: [2023-05-11 00:16:33,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +11: [2023-05-11 00:16:33,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +23: [2023-05-11 00:16:33,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +23: [2023-05-11 00:16:33,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +23: [2023-05-11 00:16:33,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +23: [2023-05-11 00:16:33,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +27: [2023-05-11 00:16:33,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +19: [2023-05-11 00:16:33,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +31: [2023-05-11 00:16:33,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +27: [2023-05-11 00:16:33,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +20: [2023-05-11 00:16:33,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +23: [2023-05-11 00:16:33,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +23: [2023-05-11 00:16:33,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +14: [2023-05-11 00:16:33,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +31: [2023-05-11 00:16:33,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +31: [2023-05-11 00:16:33,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +31: [2023-05-11 00:16:33,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +20: [2023-05-11 00:16:33,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +11: [2023-05-11 00:16:33,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +20: [2023-05-11 00:16:33,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 6: [2023-05-11 00:16:33,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 6: [2023-05-11 00:16:33,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +27: [2023-05-11 00:16:33,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +27: [2023-05-11 00:16:33,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +27: [2023-05-11 00:16:33,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +20: [2023-05-11 00:16:33,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +17: [2023-05-11 00:16:33,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +19: [2023-05-11 00:16:33,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +17: [2023-05-11 00:16:33,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +11: [2023-05-11 00:16:33,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +17: [2023-05-11 00:16:33,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +11: [2023-05-11 00:16:33,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +26: [2023-05-11 00:16:33,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +21: [2023-05-11 00:16:33,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +21: [2023-05-11 00:16:33,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +21: [2023-05-11 00:16:33,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +23: [2023-05-11 00:16:33,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 9: [2023-05-11 00:16:33,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 9: [2023-05-11 00:16:33,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +21: [2023-05-11 00:16:33,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +27: [2023-05-11 00:16:33,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +23: [2023-05-11 00:16:33,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +23: [2023-05-11 00:16:33,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +17: [2023-05-11 00:16:33,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +31: [2023-05-11 00:16:33,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +15: [2023-05-11 00:16:33,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +19: [2023-05-11 00:16:33,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +15: [2023-05-11 00:16:33,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +15: [2023-05-11 00:16:33,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +15: [2023-05-11 00:16:33,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +20: [2023-05-11 00:16:33,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +31: [2023-05-11 00:16:33,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +31: [2023-05-11 00:16:33,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +15: [2023-05-11 00:16:33,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +15: [2023-05-11 00:16:33,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +27: [2023-05-11 00:16:33,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +27: [2023-05-11 00:16:33,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 3: [2023-05-11 00:16:33,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 3: [2023-05-11 00:16:33,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +27: [2023-05-11 00:16:33,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +26: [2023-05-11 00:16:33,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +19: [2023-05-11 00:16:33,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 1: [2023-05-11 00:16:33,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +19: [2023-05-11 00:16:33,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +15: [2023-05-11 00:16:33,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +15: [2023-05-11 00:16:33,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +23: [2023-05-11 00:16:33,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +11: [2023-05-11 00:16:33,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +27: [2023-05-11 00:16:33,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +21: [2023-05-11 00:16:33,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +27: [2023-05-11 00:16:33,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +27: [2023-05-11 00:16:33,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 9: [2023-05-11 00:16:33,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 7: [2023-05-11 00:16:33,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +23: [2023-05-11 00:16:33,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +23: [2023-05-11 00:16:33,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +21: [2023-05-11 00:16:33,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +21: [2023-05-11 00:16:33,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 7: [2023-05-11 00:16:33,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +21: [2023-05-11 00:16:33,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +17: [2023-05-11 00:16:33,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 4: [2023-05-11 00:16:33,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +31: [2023-05-11 00:16:33,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +19: [2023-05-11 00:16:33,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +19: [2023-05-11 00:16:33,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 9: [2023-05-11 00:16:33,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +19: [2023-05-11 00:16:33,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 3: [2023-05-11 00:16:33,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +31: [2023-05-11 00:16:33,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +23: [2023-05-11 00:16:33,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +19: [2023-05-11 00:16:33,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +11: [2023-05-11 00:16:33,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 4: [2023-05-11 00:16:33,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 3: [2023-05-11 00:16:33,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +27: [2023-05-11 00:16:33,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +11: [2023-05-11 00:16:33,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +11: [2023-05-11 00:16:33,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 6: [2023-05-11 00:16:33,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +31: [2023-05-11 00:16:33,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +27: [2023-05-11 00:16:33,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +31: [2023-05-11 00:16:33,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +31: [2023-05-11 00:16:33,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +15: [2023-05-11 00:16:33,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +15: [2023-05-11 00:16:33,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +19: [2023-05-11 00:16:33,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +31: [2023-05-11 00:16:33,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +19: [2023-05-11 00:16:33,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 6: [2023-05-11 00:16:33,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +11: [2023-05-11 00:16:33,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +27: [2023-05-11 00:16:33,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +11: [2023-05-11 00:16:33,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +31: [2023-05-11 00:16:33,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +27: [2023-05-11 00:16:33,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +21: [2023-05-11 00:16:33,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +31: [2023-05-11 00:16:33,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. +23: [2023-05-11 00:16:33,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +21: [2023-05-11 00:16:33,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +19: [2023-05-11 00:16:33,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +19: [2023-05-11 00:16:33,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +31: [2023-05-11 00:16:33,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +19: [2023-05-11 00:16:33,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +11: [2023-05-11 00:16:33,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +11: [2023-05-11 00:16:33,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +15: [2023-05-11 00:16:33,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 4: [2023-05-11 00:16:33,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +27: [2023-05-11 00:16:33,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +18: [2023-05-11 00:16:33,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +18: [2023-05-11 00:16:33,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +31: [2023-05-11 00:16:33,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +15: [2023-05-11 00:16:33,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +18: [2023-05-11 00:16:33,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +18: [2023-05-11 00:16:33,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 4: [2023-05-11 00:16:33,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +18: [2023-05-11 00:16:33,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +18: [2023-05-11 00:16:33,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +18: [2023-05-11 00:16:33,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +18: [2023-05-11 00:16:33,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +18: [2023-05-11 00:16:33,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +18: [2023-05-11 00:16:33,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 3: [2023-05-11 00:16:33,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +19: [2023-05-11 00:16:33,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 3: [2023-05-11 00:16:33,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +18: [2023-05-11 00:16:33,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 6: [2023-05-11 00:16:33,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +18: [2023-05-11 00:16:33,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +18: [2023-05-11 00:16:33,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +18: [2023-05-11 00:16:33,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +18: [2023-05-11 00:16:33,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 7: [2023-05-11 00:16:33,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +11: [2023-05-11 00:16:33,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +21: [2023-05-11 00:16:33,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +21: [2023-05-11 00:16:33,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +31: [2023-05-11 00:16:33,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 6: [2023-05-11 00:16:33,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +27: [2023-05-11 00:16:33,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +11: [2023-05-11 00:16:33,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +18: [2023-05-11 00:16:33,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +31: [2023-05-11 00:16:33,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +27: [2023-05-11 00:16:33,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +27: [2023-05-11 00:16:33,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 7: [2023-05-11 00:16:33,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +27: [2023-05-11 00:16:33,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +27: [2023-05-11 00:16:33,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 3: [2023-05-11 00:16:33,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 3: [2023-05-11 00:16:33,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 0: [2023-05-11 00:16:33,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +19: [2023-05-11 00:16:33,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +31: [2023-05-11 00:16:33,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... +15: [2023-05-11 00:16:33,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +19: [2023-05-11 00:16:33,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +19: [2023-05-11 00:16:33,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +19: [2023-05-11 00:16:33,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 7: [2023-05-11 00:16:33,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +21: [2023-05-11 00:16:33,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 5: [2023-05-11 00:16:33,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 5: [2023-05-11 00:16:33,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 5: [2023-05-11 00:16:33,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 9: [2023-05-11 00:16:33,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +12: [2023-05-11 00:16:33,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +12: [2023-05-11 00:16:33,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 9: [2023-05-11 00:16:33,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +21: [2023-05-11 00:16:33,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +12: [2023-05-11 00:16:33,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 1: [2023-05-11 00:16:33,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +30: [2023-05-11 00:16:33,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +30: [2023-05-11 00:16:33,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +30: [2023-05-11 00:16:33,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +30: [2023-05-11 00:16:33,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +30: [2023-05-11 00:16:33,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +30: [2023-05-11 00:16:33,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +30: [2023-05-11 00:16:33,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +15: [2023-05-11 00:16:33,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 1: [2023-05-11 00:16:33,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +30: [2023-05-11 00:16:33,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +21: [2023-05-11 00:16:33,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 9: [2023-05-11 00:16:33,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 9: [2023-05-11 00:16:33,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 3: [2023-05-11 00:16:33,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 7: [2023-05-11 00:16:33,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +21: [2023-05-11 00:16:33,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +11: [2023-05-11 00:16:33,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +11: [2023-05-11 00:16:33,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +12: [2023-05-11 00:16:33,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +11: [2023-05-11 00:16:33,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +11: [2023-05-11 00:16:33,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 1: [2023-05-11 00:16:33,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +11: [2023-05-11 00:16:33,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +30: [2023-05-11 00:16:33,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +30: [2023-05-11 00:16:33,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +11: [2023-05-11 00:16:33,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +18: [2023-05-11 00:16:33,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +31: [2023-05-11 00:16:33,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +15: [2023-05-11 00:16:33,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +18: [2023-05-11 00:16:33,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +18: [2023-05-11 00:16:33,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +27: [2023-05-11 00:16:33,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +31: [2023-05-11 00:16:33,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +30: [2023-05-11 00:16:33,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 5: [2023-05-11 00:16:33,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +19: [2023-05-11 00:16:33,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +19: [2023-05-11 00:16:33,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 5: [2023-05-11 00:16:33,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +30: [2023-05-11 00:16:33,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +27: [2023-05-11 00:16:33,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +12: [2023-05-11 00:16:33,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +12: [2023-05-11 00:16:33,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +15: [2023-05-11 00:16:33,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 4: [2023-05-11 00:16:33,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +12: [2023-05-11 00:16:33,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +11: [2023-05-11 00:16:33,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +11: [2023-05-11 00:16:33,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 5: [2023-05-11 00:16:33,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +19: [2023-05-11 00:16:33,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +19: [2023-05-11 00:16:33,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 1: [2023-05-11 00:16:33,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +31: [2023-05-11 00:16:33,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +31: [2023-05-11 00:16:33,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 9: [2023-05-11 00:16:33,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +19: [2023-05-11 00:16:33,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +19: [2023-05-11 00:16:33,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 4: [2023-05-11 00:16:33,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 4: [2023-05-11 00:16:33,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 0: [2023-05-11 00:16:33,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 0: [2023-05-11 00:16:33,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 0: [2023-05-11 00:16:33,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 0: [2023-05-11 00:16:33,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +12: [2023-05-11 00:16:33,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 1: [2023-05-11 00:16:33,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +18: [2023-05-11 00:16:33,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +31: [2023-05-11 00:16:33,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +31: [2023-05-11 00:16:33,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 4: [2023-05-11 00:16:33,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +30: [2023-05-11 00:16:33,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +18: [2023-05-11 00:16:33,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 4: [2023-05-11 00:16:33,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 0: [2023-05-11 00:16:33,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt. + 7: [2023-05-11 00:16:33,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +30: [2023-05-11 00:16:33,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +30: [2023-05-11 00:16:33,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +18: [2023-05-11 00:16:33,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 9: [2023-05-11 00:16:33,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +30: [2023-05-11 00:16:33,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +30: [2023-05-11 00:16:33,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +12: [2023-05-11 00:16:33,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +18: [2023-05-11 00:16:33,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +12: [2023-05-11 00:16:33,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +30: [2023-05-11 00:16:33,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +31: [2023-05-11 00:16:33,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +10: [2023-05-11 00:16:33,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +31: [2023-05-11 00:16:33,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +10: [2023-05-11 00:16:33,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +10: [2023-05-11 00:16:33,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 1: [2023-05-11 00:16:33,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +10: [2023-05-11 00:16:33,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 1: [2023-05-11 00:16:33,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 0: [2023-05-11 00:16:33,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +30: [2023-05-11 00:16:33,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +30: [2023-05-11 00:16:33,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +30: [2023-05-11 00:16:33,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 5: [2023-05-11 00:16:33,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 5: [2023-05-11 00:16:33,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 0: [2023-05-11 00:16:33,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +12: [2023-05-11 00:16:33,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +12: [2023-05-11 00:16:33,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +12: [2023-05-11 00:16:33,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +12: [2023-05-11 00:16:33,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +30: [2023-05-11 00:16:33,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt... + 9: [2023-05-11 00:16:33,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +24: [2023-05-11 00:16:33,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +24: [2023-05-11 00:16:33,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +24: [2023-05-11 00:16:33,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +24: [2023-05-11 00:16:33,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +18: [2023-05-11 00:16:33,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 7: [2023-05-11 00:16:33,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +12: [2023-05-11 00:16:33,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +12: [2023-05-11 00:16:33,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 1: [2023-05-11 00:16:33,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 1: [2023-05-11 00:16:33,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +12: [2023-05-11 00:16:33,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +12: [2023-05-11 00:16:33,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +12: [2023-05-11 00:16:33,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +12: [2023-05-11 00:16:33,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +10: [2023-05-11 00:16:33,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +10: [2023-05-11 00:16:33,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +10: [2023-05-11 00:16:33,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +10: [2023-05-11 00:16:33,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +12: [2023-05-11 00:16:33,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +12: [2023-05-11 00:16:33,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +12: [2023-05-11 00:16:33,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +30: [2023-05-11 00:16:33,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 7: [2023-05-11 00:16:33,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +24: [2023-05-11 00:16:33,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +30: [2023-05-11 00:16:33,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +12: [2023-05-11 00:16:33,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +24: [2023-05-11 00:16:33,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +24: [2023-05-11 00:16:33,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +24: [2023-05-11 00:16:33,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +30: [2023-05-11 00:16:33,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +30: [2023-05-11 00:16:33,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 0: [2023-05-11 00:16:33,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 0: [2023-05-11 00:16:33,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... + 5: [2023-05-11 00:16:33,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +15: [2023-05-11 00:16:33,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +15: [2023-05-11 00:16:33,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +15: [2023-05-11 00:16:33,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +15: [2023-05-11 00:16:33,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 1: [2023-05-11 00:16:33,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +30: [2023-05-11 00:16:33,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +30: [2023-05-11 00:16:33,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +30: [2023-05-11 00:16:33,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +30: [2023-05-11 00:16:33,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 1: [2023-05-11 00:16:33,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 0: [2023-05-11 00:16:33,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +30: [2023-05-11 00:16:33,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +30: [2023-05-11 00:16:33,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +12: [2023-05-11 00:16:33,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 5: [2023-05-11 00:16:33,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. + 0: [2023-05-11 00:16:33,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +15: [2023-05-11 00:16:33,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 5: [2023-05-11 00:16:33,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +12: [2023-05-11 00:16:33,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +12: [2023-05-11 00:16:33,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +15: [2023-05-11 00:16:33,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt. +15: [2023-05-11 00:16:33,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +15: [2023-05-11 00:16:33,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 0: [2023-05-11 00:16:33,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt... +12: [2023-05-11 00:16:33,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +12: [2023-05-11 00:16:33,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +16: [2023-05-11 00:16:33,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +16: [2023-05-11 00:16:33,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +16: [2023-05-11 00:16:33,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +16: [2023-05-11 00:16:33,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +29: [2023-05-11 00:16:33,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +29: [2023-05-11 00:16:33,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +29: [2023-05-11 00:16:33,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +29: [2023-05-11 00:16:33,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +12: [2023-05-11 00:16:33,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +15: [2023-05-11 00:16:33,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +15: [2023-05-11 00:16:33,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +15: [2023-05-11 00:16:33,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +15: [2023-05-11 00:16:33,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +12: [2023-05-11 00:16:33,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +29: [2023-05-11 00:16:33,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +29: [2023-05-11 00:16:33,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +29: [2023-05-11 00:16:33,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +29: [2023-05-11 00:16:33,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +16: [2023-05-11 00:16:33,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +16: [2023-05-11 00:16:33,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +16: [2023-05-11 00:16:33,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +16: [2023-05-11 00:16:33,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +22: [2023-05-11 00:16:33,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +22: [2023-05-11 00:16:33,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +22: [2023-05-11 00:16:33,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +22: [2023-05-11 00:16:33,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +12: [2023-05-11 00:16:33,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +22: [2023-05-11 00:16:33,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +22: [2023-05-11 00:16:33,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +22: [2023-05-11 00:16:33,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +22: [2023-05-11 00:16:33,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +15: [2023-05-11 00:16:33,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +15: [2023-05-11 00:16:33,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +24: [2023-05-11 00:16:33,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +24: [2023-05-11 00:16:33,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +24: [2023-05-11 00:16:33,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +24: [2023-05-11 00:16:33,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +24: [2023-05-11 00:16:33,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +24: [2023-05-11 00:16:33,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +24: [2023-05-11 00:16:33,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +24: [2023-05-11 00:16:33,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +24: [2023-05-11 00:16:33,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +24: [2023-05-11 00:16:33,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +24: [2023-05-11 00:16:33,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +24: [2023-05-11 00:16:33,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +15: [2023-05-11 00:16:33,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +24: [2023-05-11 00:16:33,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +24: [2023-05-11 00:16:33,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +24: [2023-05-11 00:16:33,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +15: [2023-05-11 00:16:33,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +24: [2023-05-11 00:16:33,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +22: [2023-05-11 00:16:33,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +22: [2023-05-11 00:16:33,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +22: [2023-05-11 00:16:33,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +24: [2023-05-11 00:16:33,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +25: [2023-05-11 00:16:33,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +25: [2023-05-11 00:16:33,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +25: [2023-05-11 00:16:33,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +25: [2023-05-11 00:16:33,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +25: [2023-05-11 00:16:33,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +25: [2023-05-11 00:16:33,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +25: [2023-05-11 00:16:33,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +25: [2023-05-11 00:16:33,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +22: [2023-05-11 00:16:33,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +24: [2023-05-11 00:16:33,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +22: [2023-05-11 00:16:33,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +22: [2023-05-11 00:16:33,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +24: [2023-05-11 00:16:33,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +25: [2023-05-11 00:16:33,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +25: [2023-05-11 00:16:33,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +22: [2023-05-11 00:16:33,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +25: [2023-05-11 00:16:33,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +24: [2023-05-11 00:16:33,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 8: [2023-05-11 00:16:33,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +10: [2023-05-11 00:16:33,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +10: [2023-05-11 00:16:33,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +10: [2023-05-11 00:16:33,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +10: [2023-05-11 00:16:33,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +10: [2023-05-11 00:16:33,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +10: [2023-05-11 00:16:33,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +10: [2023-05-11 00:16:33,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +10: [2023-05-11 00:16:33,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +10: [2023-05-11 00:16:33,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +10: [2023-05-11 00:16:33,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +10: [2023-05-11 00:16:33,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +10: [2023-05-11 00:16:33,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +10: [2023-05-11 00:16:33,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +10: [2023-05-11 00:16:33,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +10: [2023-05-11 00:16:33,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 8: [2023-05-11 00:16:33,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +10: [2023-05-11 00:16:33,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +22: [2023-05-11 00:16:33,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +25: [2023-05-11 00:16:33,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +14: [2023-05-11 00:16:33,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +14: [2023-05-11 00:16:33,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +14: [2023-05-11 00:16:33,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +24: [2023-05-11 00:16:33,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +24: [2023-05-11 00:16:33,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +14: [2023-05-11 00:16:33,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +25: [2023-05-11 00:16:33,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +25: [2023-05-11 00:16:33,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +29: [2023-05-11 00:16:33,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +29: [2023-05-11 00:16:33,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +29: [2023-05-11 00:16:33,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +29: [2023-05-11 00:16:33,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +29: [2023-05-11 00:16:33,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +29: [2023-05-11 00:16:33,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +29: [2023-05-11 00:16:33,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +29: [2023-05-11 00:16:33,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +29: [2023-05-11 00:16:33,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +29: [2023-05-11 00:16:33,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +29: [2023-05-11 00:16:33,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +29: [2023-05-11 00:16:33,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +29: [2023-05-11 00:16:33,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +29: [2023-05-11 00:16:33,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +29: [2023-05-11 00:16:33,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +10: [2023-05-11 00:16:33,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +29: [2023-05-11 00:16:33,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +25: [2023-05-11 00:16:33,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 8: [2023-05-11 00:16:33,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +10: [2023-05-11 00:16:34,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +24: [2023-05-11 00:16:34,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +10: [2023-05-11 00:16:34,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +14: [2023-05-11 00:16:34,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +24: [2023-05-11 00:16:34,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +14: [2023-05-11 00:16:34,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +14: [2023-05-11 00:16:34,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +14: [2023-05-11 00:16:34,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +19: [2023-05-11 00:16:34,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +19: [2023-05-11 00:16:34,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +19: [2023-05-11 00:16:34,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +21: [2023-05-11 00:16:34,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +21: [2023-05-11 00:16:34,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +19: [2023-05-11 00:16:34,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +21: [2023-05-11 00:16:34,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +21: [2023-05-11 00:16:34,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +19: [2023-05-11 00:16:34,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +19: [2023-05-11 00:16:34,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +19: [2023-05-11 00:16:34,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +19: [2023-05-11 00:16:34,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +21: [2023-05-11 00:16:34,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +21: [2023-05-11 00:16:34,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +21: [2023-05-11 00:16:34,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 2: [2023-05-11 00:16:34,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 2: [2023-05-11 00:16:34,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 2: [2023-05-11 00:16:34,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +21: [2023-05-11 00:16:34,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +10: [2023-05-11 00:16:34,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +25: [2023-05-11 00:16:34,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +29: [2023-05-11 00:16:34,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +10: [2023-05-11 00:16:34,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +10: [2023-05-11 00:16:34,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +16: [2023-05-11 00:16:34,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +16: [2023-05-11 00:16:34,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +16: [2023-05-11 00:16:34,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +16: [2023-05-11 00:16:34,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +16: [2023-05-11 00:16:34,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +16: [2023-05-11 00:16:34,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +16: [2023-05-11 00:16:34,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +16: [2023-05-11 00:16:34,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +16: [2023-05-11 00:16:34,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +16: [2023-05-11 00:16:34,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +16: [2023-05-11 00:16:34,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +16: [2023-05-11 00:16:34,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +16: [2023-05-11 00:16:34,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +16: [2023-05-11 00:16:34,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +14: [2023-05-11 00:16:34,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +14: [2023-05-11 00:16:34,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +14: [2023-05-11 00:16:34,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +14: [2023-05-11 00:16:34,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +14: [2023-05-11 00:16:34,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +14: [2023-05-11 00:16:34,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +28: [2023-05-11 00:16:34,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +28: [2023-05-11 00:16:34,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +28: [2023-05-11 00:16:34,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +28: [2023-05-11 00:16:34,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +16: [2023-05-11 00:16:34,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +16: [2023-05-11 00:16:34,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +14: [2023-05-11 00:16:34,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +14: [2023-05-11 00:16:34,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +14: [2023-05-11 00:16:34,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +14: [2023-05-11 00:16:34,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +28: [2023-05-11 00:16:34,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +28: [2023-05-11 00:16:34,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +14: [2023-05-11 00:16:34,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +28: [2023-05-11 00:16:34,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +19: [2023-05-11 00:16:34,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +19: [2023-05-11 00:16:34,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +29: [2023-05-11 00:16:34,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +29: [2023-05-11 00:16:34,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +29: [2023-05-11 00:16:34,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +29: [2023-05-11 00:16:34,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +28: [2023-05-11 00:16:34,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +19: [2023-05-11 00:16:34,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +10: [2023-05-11 00:16:34,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +19: [2023-05-11 00:16:34,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +14: [2023-05-11 00:16:34,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +21: [2023-05-11 00:16:34,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +21: [2023-05-11 00:16:34,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +21: [2023-05-11 00:16:34,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +11: [2023-05-11 00:16:34,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +11: [2023-05-11 00:16:34,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +11: [2023-05-11 00:16:34,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +11: [2023-05-11 00:16:34,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +11: [2023-05-11 00:16:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +17: [2023-05-11 00:16:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +17: [2023-05-11 00:16:34,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +11: [2023-05-11 00:16:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +11: [2023-05-11 00:16:34,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +11: [2023-05-11 00:16:34,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +17: [2023-05-11 00:16:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +14: [2023-05-11 00:16:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +16: [2023-05-11 00:16:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +17: [2023-05-11 00:16:34,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +17: [2023-05-11 00:16:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +26: [2023-05-11 00:16:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +26: [2023-05-11 00:16:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +26: [2023-05-11 00:16:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +26: [2023-05-11 00:16:34,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +26: [2023-05-11 00:16:34,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +17: [2023-05-11 00:16:34,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +26: [2023-05-11 00:16:34,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +26: [2023-05-11 00:16:34,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +27: [2023-05-11 00:16:34,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +27: [2023-05-11 00:16:34,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +26: [2023-05-11 00:16:34,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +17: [2023-05-11 00:16:34,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +27: [2023-05-11 00:16:34,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +27: [2023-05-11 00:16:34,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +19: [2023-05-11 00:16:34,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +10: [2023-05-11 00:16:34,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +17: [2023-05-11 00:16:34,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +29: [2023-05-11 00:16:34,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +29: [2023-05-11 00:16:34,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +27: [2023-05-11 00:16:34,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +27: [2023-05-11 00:16:34,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +27: [2023-05-11 00:16:34,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +28: [2023-05-11 00:16:34,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +28: [2023-05-11 00:16:34,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +28: [2023-05-11 00:16:34,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +16: [2023-05-11 00:16:34,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +27: [2023-05-11 00:16:34,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +29: [2023-05-11 00:16:34,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +19: [2023-05-11 00:16:34,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +16: [2023-05-11 00:16:34,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +19: [2023-05-11 00:16:34,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +14: [2023-05-11 00:16:34,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +14: [2023-05-11 00:16:34,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +19: [2023-05-11 00:16:34,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +30: [2023-05-11 00:16:34,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +30: [2023-05-11 00:16:34,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +21: [2023-05-11 00:16:34,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +21: [2023-05-11 00:16:34,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +21: [2023-05-11 00:16:34,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +30: [2023-05-11 00:16:34,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +30: [2023-05-11 00:16:34,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +16: [2023-05-11 00:16:34,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +14: [2023-05-11 00:16:34,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +21: [2023-05-11 00:16:34,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +11: [2023-05-11 00:16:34,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +14: [2023-05-11 00:16:34,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +14: [2023-05-11 00:16:34,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +28: [2023-05-11 00:16:34,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +11: [2023-05-11 00:16:34,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +28: [2023-05-11 00:16:34,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +14: [2023-05-11 00:16:34,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +19: [2023-05-11 00:16:34,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +19: [2023-05-11 00:16:34,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +19: [2023-05-11 00:16:34,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +19: [2023-05-11 00:16:34,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +30: [2023-05-11 00:16:34,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +30: [2023-05-11 00:16:34,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +30: [2023-05-11 00:16:34,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +30: [2023-05-11 00:16:34,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +27: [2023-05-11 00:16:34,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +14: [2023-05-11 00:16:34,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +16: [2023-05-11 00:16:34,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +28: [2023-05-11 00:16:34,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +16: [2023-05-11 00:16:34,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +13: [2023-05-11 00:16:34,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +13: [2023-05-11 00:16:34,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +13: [2023-05-11 00:16:34,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +13: [2023-05-11 00:16:34,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +17: [2023-05-11 00:16:34,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +15: [2023-05-11 00:16:34,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +15: [2023-05-11 00:16:34,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +17: [2023-05-11 00:16:34,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +17: [2023-05-11 00:16:34,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +11: [2023-05-11 00:16:34,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +15: [2023-05-11 00:16:34,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +27: [2023-05-11 00:16:34,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +15: [2023-05-11 00:16:34,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +11: [2023-05-11 00:16:34,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +21: [2023-05-11 00:16:34,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +15: [2023-05-11 00:16:34,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +15: [2023-05-11 00:16:34,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +15: [2023-05-11 00:16:34,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +15: [2023-05-11 00:16:34,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +13: [2023-05-11 00:16:34,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +13: [2023-05-11 00:16:34,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +13: [2023-05-11 00:16:34,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +27: [2023-05-11 00:16:34,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +11: [2023-05-11 00:16:34,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +26: [2023-05-11 00:16:34,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +26: [2023-05-11 00:16:34,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +26: [2023-05-11 00:16:34,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +13: [2023-05-11 00:16:34,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +28: [2023-05-11 00:16:34,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +16: [2023-05-11 00:16:34,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +30: [2023-05-11 00:16:34,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +14: [2023-05-11 00:16:34,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +19: [2023-05-11 00:16:34,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +11: [2023-05-11 00:16:34,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +20: [2023-05-11 00:16:34,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +20: [2023-05-11 00:16:34,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +20: [2023-05-11 00:16:34,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +14: [2023-05-11 00:16:34,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +20: [2023-05-11 00:16:34,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +20: [2023-05-11 00:16:34,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +20: [2023-05-11 00:16:34,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +19: [2023-05-11 00:16:34,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +20: [2023-05-11 00:16:34,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +17: [2023-05-11 00:16:34,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +17: [2023-05-11 00:16:34,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +30: [2023-05-11 00:16:34,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +30: [2023-05-11 00:16:34,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +19: [2023-05-11 00:16:34,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +11: [2023-05-11 00:16:34,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +11: [2023-05-11 00:16:34,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +30: [2023-05-11 00:16:34,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +31: [2023-05-11 00:16:34,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +31: [2023-05-11 00:16:34,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +19: [2023-05-11 00:16:34,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +31: [2023-05-11 00:16:34,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +31: [2023-05-11 00:16:34,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +31: [2023-05-11 00:16:34,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +31: [2023-05-11 00:16:34,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +20: [2023-05-11 00:16:34,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +27: [2023-05-11 00:16:34,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +27: [2023-05-11 00:16:34,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +18: [2023-05-11 00:16:34,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +18: [2023-05-11 00:16:34,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +18: [2023-05-11 00:16:34,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +18: [2023-05-11 00:16:34,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +31: [2023-05-11 00:16:34,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +18: [2023-05-11 00:16:34,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +31: [2023-05-11 00:16:34,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +18: [2023-05-11 00:16:34,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +15: [2023-05-11 00:16:34,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +15: [2023-05-11 00:16:34,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +18: [2023-05-11 00:16:34,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +27: [2023-05-11 00:16:34,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +28: [2023-05-11 00:16:34,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +15: [2023-05-11 00:16:34,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +26: [2023-05-11 00:16:34,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 6: [2023-05-11 00:16:34,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +16: [2023-05-11 00:16:34,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +26: [2023-05-11 00:16:34,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 6: [2023-05-11 00:16:34,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +17: [2023-05-11 00:16:34,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +18: [2023-05-11 00:16:34,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +13: [2023-05-11 00:16:34,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +13: [2023-05-11 00:16:34,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +26: [2023-05-11 00:16:34,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +26: [2023-05-11 00:16:34,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +26: [2023-05-11 00:16:34,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 7: [2023-05-11 00:16:34,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +17: [2023-05-11 00:16:34,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +25: [2023-05-11 00:16:34,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +25: [2023-05-11 00:16:34,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +26: [2023-05-11 00:16:34,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +28: [2023-05-11 00:16:34,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 8: [2023-05-11 00:16:34,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 8: [2023-05-11 00:16:34,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 8: [2023-05-11 00:16:34,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 9: [2023-05-11 00:16:34,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +28: [2023-05-11 00:16:34,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +28: [2023-05-11 00:16:34,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +26: [2023-05-11 00:16:34,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +15: [2023-05-11 00:16:34,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +30: [2023-05-11 00:16:34,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +30: [2023-05-11 00:16:34,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +27: [2023-05-11 00:16:34,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +27: [2023-05-11 00:16:34,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +26: [2023-05-11 00:16:34,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +30: [2023-05-11 00:16:34,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +27: [2023-05-11 00:16:34,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 5: [2023-05-11 00:16:34,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +30: [2023-05-11 00:16:34,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +13: [2023-05-11 00:16:34,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +27: [2023-05-11 00:16:34,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 2: [2023-05-11 00:16:34,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +25: [2023-05-11 00:16:34,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +28: [2023-05-11 00:16:34,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +14: [2023-05-11 00:16:34,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +20: [2023-05-11 00:16:34,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +20: [2023-05-11 00:16:34,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +20: [2023-05-11 00:16:34,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +25: [2023-05-11 00:16:34,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +31: [2023-05-11 00:16:34,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +15: [2023-05-11 00:16:34,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +15: [2023-05-11 00:16:34,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +13: [2023-05-11 00:16:34,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +26: [2023-05-11 00:16:34,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +15: [2023-05-11 00:16:34,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +18: [2023-05-11 00:16:34,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +17: [2023-05-11 00:16:34,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +31: [2023-05-11 00:16:34,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +31: [2023-05-11 00:16:34,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +18: [2023-05-11 00:16:34,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +13: [2023-05-11 00:16:34,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +26: [2023-05-11 00:16:34,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 8: [2023-05-11 00:16:34,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +25: [2023-05-11 00:16:34,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +28: [2023-05-11 00:16:34,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +26: [2023-05-11 00:16:34,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +26: [2023-05-11 00:16:34,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +27: [2023-05-11 00:16:34,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +28: [2023-05-11 00:16:34,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +20: [2023-05-11 00:16:34,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +20: [2023-05-11 00:16:34,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +20: [2023-05-11 00:16:34,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +25: [2023-05-11 00:16:34,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +13: [2023-05-11 00:16:34,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +14: [2023-05-11 00:16:34,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +28: [2023-05-11 00:16:34,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +19: [2023-05-11 00:16:34,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +15: [2023-05-11 00:16:34,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +26: [2023-05-11 00:16:34,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +28: [2023-05-11 00:16:34,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +20: [2023-05-11 00:16:34,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +25: [2023-05-11 00:16:34,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +27: [2023-05-11 00:16:34,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +31: [2023-05-11 00:16:34,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +20: [2023-05-11 00:16:34,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +13: [2023-05-11 00:16:34,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +19: [2023-05-11 00:16:34,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +19: [2023-05-11 00:16:34,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 1: [2023-05-11 00:16:34,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +25: [2023-05-11 00:16:34,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +18: [2023-05-11 00:16:34,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +19: [2023-05-11 00:16:34,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 1: [2023-05-11 00:16:34,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +27: [2023-05-11 00:16:34,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +27: [2023-05-11 00:16:34,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +31: [2023-05-11 00:16:34,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +20: [2023-05-11 00:16:34,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +20: [2023-05-11 00:16:34,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +27: [2023-05-11 00:16:34,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +21: [2023-05-11 00:16:34,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +21: [2023-05-11 00:16:34,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 1: [2023-05-11 00:16:34,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 9: [2023-05-11 00:16:34,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +27: [2023-05-11 00:16:34,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +31: [2023-05-11 00:16:34,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +19: [2023-05-11 00:16:34,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +19: [2023-05-11 00:16:34,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +20: [2023-05-11 00:16:34,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +20: [2023-05-11 00:16:34,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 1: [2023-05-11 00:16:34,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +18: [2023-05-11 00:16:34,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +21: [2023-05-11 00:16:34,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +19: [2023-05-11 00:16:34,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +17: [2023-05-11 00:16:34,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +17: [2023-05-11 00:16:34,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +21: [2023-05-11 00:16:34,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +19: [2023-05-11 00:16:34,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +26: [2023-05-11 00:16:34,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +26: [2023-05-11 00:16:34,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +26: [2023-05-11 00:16:34,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +26: [2023-05-11 00:16:34,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +20: [2023-05-11 00:16:34,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +18: [2023-05-11 00:16:34,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +23: [2023-05-11 00:16:34,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +23: [2023-05-11 00:16:34,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +23: [2023-05-11 00:16:34,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +23: [2023-05-11 00:16:34,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +23: [2023-05-11 00:16:34,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +23: [2023-05-11 00:16:34,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +23: [2023-05-11 00:16:34,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +23: [2023-05-11 00:16:34,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +13: [2023-05-11 00:16:34,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +28: [2023-05-11 00:16:34,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +28: [2023-05-11 00:16:34,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +26: [2023-05-11 00:16:34,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +26: [2023-05-11 00:16:34,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 9: [2023-05-11 00:16:34,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 9: [2023-05-11 00:16:34,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 9: [2023-05-11 00:16:34,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 8: [2023-05-11 00:16:34,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +25: [2023-05-11 00:16:34,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +17: [2023-05-11 00:16:34,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +25: [2023-05-11 00:16:34,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +21: [2023-05-11 00:16:34,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +18: [2023-05-11 00:16:34,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +17: [2023-05-11 00:16:34,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +20: [2023-05-11 00:16:34,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +20: [2023-05-11 00:16:34,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 5: [2023-05-11 00:16:34,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +31: [2023-05-11 00:16:34,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 5: [2023-05-11 00:16:34,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +21: [2023-05-11 00:16:34,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +25: [2023-05-11 00:16:34,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +25: [2023-05-11 00:16:34,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 8: [2023-05-11 00:16:34,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +28: [2023-05-11 00:16:34,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +28: [2023-05-11 00:16:34,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 1: [2023-05-11 00:16:34,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +15: [2023-05-11 00:16:34,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +15: [2023-05-11 00:16:34,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +15: [2023-05-11 00:16:34,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +15: [2023-05-11 00:16:34,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +17: [2023-05-11 00:16:34,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +17: [2023-05-11 00:16:34,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +21: [2023-05-11 00:16:34,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +20: [2023-05-11 00:16:34,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +27: [2023-05-11 00:16:34,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +27: [2023-05-11 00:16:34,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +21: [2023-05-11 00:16:34,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +18: [2023-05-11 00:16:34,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +28: [2023-05-11 00:16:34,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +18: [2023-05-11 00:16:34,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +27: [2023-05-11 00:16:34,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +13: [2023-05-11 00:16:34,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +13: [2023-05-11 00:16:34,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +13: [2023-05-11 00:16:34,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +13: [2023-05-11 00:16:34,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +27: [2023-05-11 00:16:34,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 9: [2023-05-11 00:16:34,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +24: [2023-05-11 00:16:34,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +24: [2023-05-11 00:16:34,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +24: [2023-05-11 00:16:34,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +24: [2023-05-11 00:16:34,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 7: [2023-05-11 00:16:34,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 7: [2023-05-11 00:16:34,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +28: [2023-05-11 00:16:34,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +17: [2023-05-11 00:16:34,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +17: [2023-05-11 00:16:34,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +27: [2023-05-11 00:16:34,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +27: [2023-05-11 00:16:34,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +25: [2023-05-11 00:16:34,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +31: [2023-05-11 00:16:34,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +25: [2023-05-11 00:16:34,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 7: [2023-05-11 00:16:34,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 0: [2023-05-11 00:16:34,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +24: [2023-05-11 00:16:34,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +24: [2023-05-11 00:16:34,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +24: [2023-05-11 00:16:34,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +23: [2023-05-11 00:16:34,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +15: [2023-05-11 00:16:34,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +15: [2023-05-11 00:16:34,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +24: [2023-05-11 00:16:34,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +23: [2023-05-11 00:16:34,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +23: [2023-05-11 00:16:34,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +15: [2023-05-11 00:16:34,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +20: [2023-05-11 00:16:34,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +20: [2023-05-11 00:16:34,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +26: [2023-05-11 00:16:34,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +15: [2023-05-11 00:16:34,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +20: [2023-05-11 00:16:34,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +25: [2023-05-11 00:16:34,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +28: [2023-05-11 00:16:34,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +23: [2023-05-11 00:16:34,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +26: [2023-05-11 00:16:34,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +25: [2023-05-11 00:16:34,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 0: [2023-05-11 00:16:34,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +28: [2023-05-11 00:16:34,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +13: [2023-05-11 00:16:34,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +17: [2023-05-11 00:16:34,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +17: [2023-05-11 00:16:34,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 1: [2023-05-11 00:16:34,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +13: [2023-05-11 00:16:34,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +13: [2023-05-11 00:16:34,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +21: [2023-05-11 00:16:34,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +21: [2023-05-11 00:16:34,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +24: [2023-05-11 00:16:34,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +17: [2023-05-11 00:16:34,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +17: [2023-05-11 00:16:34,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +13: [2023-05-11 00:16:34,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +27: [2023-05-11 00:16:34,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +17: [2023-05-11 00:16:34,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +17: [2023-05-11 00:16:34,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +23: [2023-05-11 00:16:34,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +23: [2023-05-11 00:16:34,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +21: [2023-05-11 00:16:34,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +21: [2023-05-11 00:16:34,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +27: [2023-05-11 00:16:34,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +17: [2023-05-11 00:16:34,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +17: [2023-05-11 00:16:34,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +23: [2023-05-11 00:16:34,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +23: [2023-05-11 00:16:34,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +21: [2023-05-11 00:16:34,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +21: [2023-05-11 00:16:34,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +20: [2023-05-11 00:16:34,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +15: [2023-05-11 00:16:34,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +15: [2023-05-11 00:16:34,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +15: [2023-05-11 00:16:34,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 5: [2023-05-11 00:16:34,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +15: [2023-05-11 00:16:34,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 5: [2023-05-11 00:16:34,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 5: [2023-05-11 00:16:34,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +12: [2023-05-11 00:16:34,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +12: [2023-05-11 00:16:34,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +15: [2023-05-11 00:16:34,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +15: [2023-05-11 00:16:34,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +20: [2023-05-11 00:16:34,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +20: [2023-05-11 00:16:34,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 9: [2023-05-11 00:16:34,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 9: [2023-05-11 00:16:34,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 9: [2023-05-11 00:16:34,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +24: [2023-05-11 00:16:34,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +12: [2023-05-11 00:16:34,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +24: [2023-05-11 00:16:34,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +24: [2023-05-11 00:16:34,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +12: [2023-05-11 00:16:34,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +13: [2023-05-11 00:16:34,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 6: [2023-05-11 00:16:34,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 6: [2023-05-11 00:16:34,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +13: [2023-05-11 00:16:34,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 0: [2023-05-11 00:16:34,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +21: [2023-05-11 00:16:34,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +20: [2023-05-11 00:16:34,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +15: [2023-05-11 00:16:34,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +15: [2023-05-11 00:16:34,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +21: [2023-05-11 00:16:34,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +25: [2023-05-11 00:16:34,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +25: [2023-05-11 00:16:34,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +25: [2023-05-11 00:16:34,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +25: [2023-05-11 00:16:34,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +20: [2023-05-11 00:16:34,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 9: [2023-05-11 00:16:34,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +13: [2023-05-11 00:16:34,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +13: [2023-05-11 00:16:34,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 7: [2023-05-11 00:16:34,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 6: [2023-05-11 00:16:34,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +12: [2023-05-11 00:16:34,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +25: [2023-05-11 00:16:34,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +25: [2023-05-11 00:16:34,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +13: [2023-05-11 00:16:34,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +13: [2023-05-11 00:16:34,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 7: [2023-05-11 00:16:34,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 7: [2023-05-11 00:16:34,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +25: [2023-05-11 00:16:34,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +25: [2023-05-11 00:16:34,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +12: [2023-05-11 00:16:34,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +24: [2023-05-11 00:16:34,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +12: [2023-05-11 00:16:34,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +12: [2023-05-11 00:16:34,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +30: [2023-05-11 00:16:34,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +30: [2023-05-11 00:16:34,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +30: [2023-05-11 00:16:34,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +30: [2023-05-11 00:16:34,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +24: [2023-05-11 00:16:34,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 0: [2023-05-11 00:16:34,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +24: [2023-05-11 00:16:34,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +11: [2023-05-11 00:16:34,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 5: [2023-05-11 00:16:34,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +11: [2023-05-11 00:16:34,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +11: [2023-05-11 00:16:34,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +11: [2023-05-11 00:16:34,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +31: [2023-05-11 00:16:34,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +31: [2023-05-11 00:16:34,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +13: [2023-05-11 00:16:34,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +31: [2023-05-11 00:16:34,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +31: [2023-05-11 00:16:34,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 6: [2023-05-11 00:16:34,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +13: [2023-05-11 00:16:34,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +18: [2023-05-11 00:16:34,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +18: [2023-05-11 00:16:34,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +18: [2023-05-11 00:16:34,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +30: [2023-05-11 00:16:34,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +10: [2023-05-11 00:16:34,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +10: [2023-05-11 00:16:34,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +10: [2023-05-11 00:16:34,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +10: [2023-05-11 00:16:34,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +18: [2023-05-11 00:16:34,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +25: [2023-05-11 00:16:34,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +25: [2023-05-11 00:16:34,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +24: [2023-05-11 00:16:34,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 4: [2023-05-11 00:16:34,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +12: [2023-05-11 00:16:34,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +10: [2023-05-11 00:16:34,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +10: [2023-05-11 00:16:34,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +10: [2023-05-11 00:16:34,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +12: [2023-05-11 00:16:34,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 8: [2023-05-11 00:16:34,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +10: [2023-05-11 00:16:34,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +30: [2023-05-11 00:16:34,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +30: [2023-05-11 00:16:34,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +11: [2023-05-11 00:16:34,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +11: [2023-05-11 00:16:34,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +30: [2023-05-11 00:16:34,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +31: [2023-05-11 00:16:34,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +31: [2023-05-11 00:16:34,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +31: [2023-05-11 00:16:34,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 8: [2023-05-11 00:16:34,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 0: [2023-05-11 00:16:34,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 8: [2023-05-11 00:16:34,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +11: [2023-05-11 00:16:34,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +12: [2023-05-11 00:16:34,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +12: [2023-05-11 00:16:34,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 8: [2023-05-11 00:16:34,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +31: [2023-05-11 00:16:34,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +12: [2023-05-11 00:16:34,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +12: [2023-05-11 00:16:34,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +11: [2023-05-11 00:16:34,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +18: [2023-05-11 00:16:34,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +18: [2023-05-11 00:16:34,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +10: [2023-05-11 00:16:34,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 6: [2023-05-11 00:16:34,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 6: [2023-05-11 00:16:34,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +10: [2023-05-11 00:16:34,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +18: [2023-05-11 00:16:34,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +18: [2023-05-11 00:16:34,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +25: [2023-05-11 00:16:34,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +25: [2023-05-11 00:16:34,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +25: [2023-05-11 00:16:34,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +25: [2023-05-11 00:16:34,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +10: [2023-05-11 00:16:34,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +12: [2023-05-11 00:16:34,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +12: [2023-05-11 00:16:34,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +12: [2023-05-11 00:16:34,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +12: [2023-05-11 00:16:34,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +12: [2023-05-11 00:16:34,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +12: [2023-05-11 00:16:34,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +12: [2023-05-11 00:16:34,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +12: [2023-05-11 00:16:34,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +31: [2023-05-11 00:16:34,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +31: [2023-05-11 00:16:34,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 8: [2023-05-11 00:16:34,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +12: [2023-05-11 00:16:34,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +12: [2023-05-11 00:16:34,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +11: [2023-05-11 00:16:34,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +11: [2023-05-11 00:16:34,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +18: [2023-05-11 00:16:34,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +18: [2023-05-11 00:16:34,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +11: [2023-05-11 00:16:34,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +11: [2023-05-11 00:16:34,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +18: [2023-05-11 00:16:34,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +18: [2023-05-11 00:16:34,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +10: [2023-05-11 00:16:34,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +25: [2023-05-11 00:16:34,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +30: [2023-05-11 00:16:34,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +25: [2023-05-11 00:16:34,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +23: [2023-05-11 00:16:34,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +23: [2023-05-11 00:16:34,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +23: [2023-05-11 00:16:34,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +30: [2023-05-11 00:16:34,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +23: [2023-05-11 00:16:34,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +10: [2023-05-11 00:16:34,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +10: [2023-05-11 00:16:34,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +31: [2023-05-11 00:16:34,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +31: [2023-05-11 00:16:34,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +31: [2023-05-11 00:16:34,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +31: [2023-05-11 00:16:34,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +31: [2023-05-11 00:16:34,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +31: [2023-05-11 00:16:34,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 8: [2023-05-11 00:16:34,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +12: [2023-05-11 00:16:34,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +10: [2023-05-11 00:16:34,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +11: [2023-05-11 00:16:34,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +11: [2023-05-11 00:16:34,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +11: [2023-05-11 00:16:34,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +11: [2023-05-11 00:16:34,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +22: [2023-05-11 00:16:34,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +22: [2023-05-11 00:16:34,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +22: [2023-05-11 00:16:34,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +30: [2023-05-11 00:16:34,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +30: [2023-05-11 00:16:34,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +12: [2023-05-11 00:16:34,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. + 3: [2023-05-11 00:16:34,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +22: [2023-05-11 00:16:34,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +23: [2023-05-11 00:16:34,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +30: [2023-05-11 00:16:34,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 3: [2023-05-11 00:16:34,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt. +30: [2023-05-11 00:16:34,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +12: [2023-05-11 00:16:34,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +30: [2023-05-11 00:16:34,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +10: [2023-05-11 00:16:34,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +23: [2023-05-11 00:16:34,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +23: [2023-05-11 00:16:34,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +23: [2023-05-11 00:16:34,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +12: [2023-05-11 00:16:34,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +30: [2023-05-11 00:16:34,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +18: [2023-05-11 00:16:34,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +18: [2023-05-11 00:16:34,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 4: [2023-05-11 00:16:34,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +22: [2023-05-11 00:16:34,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +22: [2023-05-11 00:16:34,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +22: [2023-05-11 00:16:34,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +22: [2023-05-11 00:16:34,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +18: [2023-05-11 00:16:34,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +18: [2023-05-11 00:16:34,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +22: [2023-05-11 00:16:34,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +22: [2023-05-11 00:16:34,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +22: [2023-05-11 00:16:34,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +22: [2023-05-11 00:16:34,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +22: [2023-05-11 00:16:34,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 8: [2023-05-11 00:16:34,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +22: [2023-05-11 00:16:34,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +24: [2023-05-11 00:16:34,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +24: [2023-05-11 00:16:34,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +12: [2023-05-11 00:16:34,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +12: [2023-05-11 00:16:34,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +29: [2023-05-11 00:16:34,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +29: [2023-05-11 00:16:34,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +29: [2023-05-11 00:16:34,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +29: [2023-05-11 00:16:34,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +29: [2023-05-11 00:16:34,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +22: [2023-05-11 00:16:34,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... +29: [2023-05-11 00:16:34,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +12: [2023-05-11 00:16:34,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +24: [2023-05-11 00:16:34,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +22: [2023-05-11 00:16:34,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +29: [2023-05-11 00:16:34,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +29: [2023-05-11 00:16:34,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +23: [2023-05-11 00:16:34,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +23: [2023-05-11 00:16:34,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 4: [2023-05-11 00:16:34,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +24: [2023-05-11 00:16:34,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 4: [2023-05-11 00:16:34,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +23: [2023-05-11 00:16:34,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +23: [2023-05-11 00:16:34,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 4: [2023-05-11 00:16:34,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... + 4: [2023-05-11 00:16:34,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +24: [2023-05-11 00:16:34,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +12: [2023-05-11 00:16:34,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +24: [2023-05-11 00:16:34,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +24: [2023-05-11 00:16:34,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +16: [2023-05-11 00:16:34,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +16: [2023-05-11 00:16:34,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +16: [2023-05-11 00:16:34,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +16: [2023-05-11 00:16:34,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +22: [2023-05-11 00:16:34,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +29: [2023-05-11 00:16:34,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +24: [2023-05-11 00:16:34,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +22: [2023-05-11 00:16:34,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +16: [2023-05-11 00:16:34,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +16: [2023-05-11 00:16:34,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +16: [2023-05-11 00:16:34,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +16: [2023-05-11 00:16:34,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +23: [2023-05-11 00:16:34,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +23: [2023-05-11 00:16:34,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +22: [2023-05-11 00:16:34,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +22: [2023-05-11 00:16:34,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +23: [2023-05-11 00:16:34,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +29: [2023-05-11 00:16:34,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +23: [2023-05-11 00:16:34,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +29: [2023-05-11 00:16:34,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +22: [2023-05-11 00:16:34,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +22: [2023-05-11 00:16:34,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +22: [2023-05-11 00:16:34,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +22: [2023-05-11 00:16:34,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +10: [2023-05-11 00:16:34,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +10: [2023-05-11 00:16:34,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +10: [2023-05-11 00:16:34,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +10: [2023-05-11 00:16:34,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 3: [2023-05-11 00:16:34,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +22: [2023-05-11 00:16:34,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +24: [2023-05-11 00:16:34,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +24: [2023-05-11 00:16:34,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 4: [2023-05-11 00:16:34,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +22: [2023-05-11 00:16:34,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +22: [2023-05-11 00:16:34,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +24: [2023-05-11 00:16:34,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +24: [2023-05-11 00:16:34,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +16: [2023-05-11 00:16:34,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +24: [2023-05-11 00:16:34,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +22: [2023-05-11 00:16:34,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +16: [2023-05-11 00:16:34,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +16: [2023-05-11 00:16:34,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +22: [2023-05-11 00:16:34,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +22: [2023-05-11 00:16:34,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +24: [2023-05-11 00:16:34,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +24: [2023-05-11 00:16:34,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 3: [2023-05-11 00:16:34,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +24: [2023-05-11 00:16:34,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +21: [2023-05-11 00:16:34,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +21: [2023-05-11 00:16:34,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +21: [2023-05-11 00:16:34,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +21: [2023-05-11 00:16:34,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +21: [2023-05-11 00:16:34,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +21: [2023-05-11 00:16:34,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +21: [2023-05-11 00:16:34,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +14: [2023-05-11 00:16:34,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +14: [2023-05-11 00:16:34,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +14: [2023-05-11 00:16:34,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +14: [2023-05-11 00:16:34,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +29: [2023-05-11 00:16:34,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +27: [2023-05-11 00:16:34,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +27: [2023-05-11 00:16:34,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +27: [2023-05-11 00:16:34,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +27: [2023-05-11 00:16:34,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +26: [2023-05-11 00:16:34,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +26: [2023-05-11 00:16:34,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +26: [2023-05-11 00:16:34,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +26: [2023-05-11 00:16:34,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +26: [2023-05-11 00:16:34,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +26: [2023-05-11 00:16:34,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +21: [2023-05-11 00:16:34,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +26: [2023-05-11 00:16:34,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +14: [2023-05-11 00:16:34,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +27: [2023-05-11 00:16:34,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +27: [2023-05-11 00:16:34,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +27: [2023-05-11 00:16:34,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +22: [2023-05-11 00:16:34,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt. +14: [2023-05-11 00:16:34,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +10: [2023-05-11 00:16:34,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +10: [2023-05-11 00:16:34,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +26: [2023-05-11 00:16:34,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +14: [2023-05-11 00:16:34,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +29: [2023-05-11 00:16:34,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +29: [2023-05-11 00:16:34,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +10: [2023-05-11 00:16:34,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +22: [2023-05-11 00:16:34,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt... +10: [2023-05-11 00:16:34,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +27: [2023-05-11 00:16:34,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +16: [2023-05-11 00:16:34,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +14: [2023-05-11 00:16:34,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +16: [2023-05-11 00:16:34,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +16: [2023-05-11 00:16:34,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +13: [2023-05-11 00:16:34,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +13: [2023-05-11 00:16:34,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +20: [2023-05-11 00:16:34,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +20: [2023-05-11 00:16:34,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +20: [2023-05-11 00:16:34,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +20: [2023-05-11 00:16:34,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +20: [2023-05-11 00:16:34,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +20: [2023-05-11 00:16:34,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +20: [2023-05-11 00:16:34,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +13: [2023-05-11 00:16:34,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +13: [2023-05-11 00:16:34,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +21: [2023-05-11 00:16:34,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +29: [2023-05-11 00:16:34,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +29: [2023-05-11 00:16:34,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +27: [2023-05-11 00:16:34,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +27: [2023-05-11 00:16:34,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +13: [2023-05-11 00:16:34,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +20: [2023-05-11 00:16:34,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +13: [2023-05-11 00:16:34,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +13: [2023-05-11 00:16:34,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +13: [2023-05-11 00:16:34,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +27: [2023-05-11 00:16:34,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +16: [2023-05-11 00:16:34,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +14: [2023-05-11 00:16:34,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +14: [2023-05-11 00:16:34,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +10: [2023-05-11 00:16:34,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +10: [2023-05-11 00:16:34,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +21: [2023-05-11 00:16:34,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +21: [2023-05-11 00:16:34,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +13: [2023-05-11 00:16:34,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +26: [2023-05-11 00:16:34,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +26: [2023-05-11 00:16:34,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +26: [2023-05-11 00:16:34,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +21: [2023-05-11 00:16:34,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +11: [2023-05-11 00:16:34,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +11: [2023-05-11 00:16:34,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +11: [2023-05-11 00:16:34,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +11: [2023-05-11 00:16:34,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +11: [2023-05-11 00:16:34,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +11: [2023-05-11 00:16:34,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +20: [2023-05-11 00:16:34,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +27: [2023-05-11 00:16:34,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +27: [2023-05-11 00:16:34,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +10: [2023-05-11 00:16:34,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +10: [2023-05-11 00:16:34,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +11: [2023-05-11 00:16:34,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +10: [2023-05-11 00:16:34,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +10: [2023-05-11 00:16:34,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +11: [2023-05-11 00:16:34,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +20: [2023-05-11 00:16:34,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +16: [2023-05-11 00:16:34,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +14: [2023-05-11 00:16:34,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +27: [2023-05-11 00:16:34,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +13: [2023-05-11 00:16:34,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +14: [2023-05-11 00:16:34,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +14: [2023-05-11 00:16:34,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +21: [2023-05-11 00:16:34,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +23: [2023-05-11 00:16:34,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +23: [2023-05-11 00:16:34,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +21: [2023-05-11 00:16:34,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +17: [2023-05-11 00:16:34,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +17: [2023-05-11 00:16:34,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +17: [2023-05-11 00:16:34,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +17: [2023-05-11 00:16:34,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +23: [2023-05-11 00:16:34,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +23: [2023-05-11 00:16:34,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +13: [2023-05-11 00:16:34,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +18: [2023-05-11 00:16:34,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +18: [2023-05-11 00:16:34,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +18: [2023-05-11 00:16:34,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +18: [2023-05-11 00:16:34,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +13: [2023-05-11 00:16:34,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +17: [2023-05-11 00:16:34,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +17: [2023-05-11 00:16:34,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +18: [2023-05-11 00:16:34,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +18: [2023-05-11 00:16:34,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +31: [2023-05-11 00:16:34,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +18: [2023-05-11 00:16:34,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +31: [2023-05-11 00:16:34,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +28: [2023-05-11 00:16:34,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +31: [2023-05-11 00:16:34,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +28: [2023-05-11 00:16:34,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +10: [2023-05-11 00:16:34,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +10: [2023-05-11 00:16:34,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 2: [2023-05-11 00:16:34,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +31: [2023-05-11 00:16:34,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +31: [2023-05-11 00:16:34,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +31: [2023-05-11 00:16:34,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +31: [2023-05-11 00:16:34,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +28: [2023-05-11 00:16:34,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +20: [2023-05-11 00:16:34,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +26: [2023-05-11 00:16:34,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +23: [2023-05-11 00:16:34,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +23: [2023-05-11 00:16:34,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +23: [2023-05-11 00:16:34,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +17: [2023-05-11 00:16:34,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +17: [2023-05-11 00:16:34,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +28: [2023-05-11 00:16:34,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +31: [2023-05-11 00:16:34,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +13: [2023-05-11 00:16:34,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +28: [2023-05-11 00:16:34,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +28: [2023-05-11 00:16:34,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +28: [2023-05-11 00:16:34,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +28: [2023-05-11 00:16:34,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +18: [2023-05-11 00:16:34,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +23: [2023-05-11 00:16:34,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +26: [2023-05-11 00:16:34,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +26: [2023-05-11 00:16:34,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +20: [2023-05-11 00:16:34,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +11: [2023-05-11 00:16:34,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +20: [2023-05-11 00:16:34,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +27: [2023-05-11 00:16:34,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +14: [2023-05-11 00:16:34,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +11: [2023-05-11 00:16:34,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +13: [2023-05-11 00:16:34,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +21: [2023-05-11 00:16:34,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +13: [2023-05-11 00:16:34,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +20: [2023-05-11 00:16:34,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +13: [2023-05-11 00:16:34,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +23: [2023-05-11 00:16:34,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +17: [2023-05-11 00:16:34,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +17: [2023-05-11 00:16:34,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +11: [2023-05-11 00:16:34,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +14: [2023-05-11 00:16:34,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +31: [2023-05-11 00:16:34,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +14: [2023-05-11 00:16:34,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +11: [2023-05-11 00:16:34,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +23: [2023-05-11 00:16:34,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +18: [2023-05-11 00:16:34,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +18: [2023-05-11 00:16:34,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +18: [2023-05-11 00:16:34,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +31: [2023-05-11 00:16:34,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +26: [2023-05-11 00:16:34,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +14: [2023-05-11 00:16:34,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +19: [2023-05-11 00:16:34,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +19: [2023-05-11 00:16:34,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +27: [2023-05-11 00:16:34,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +28: [2023-05-11 00:16:34,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +14: [2023-05-11 00:16:34,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +20: [2023-05-11 00:16:34,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +28: [2023-05-11 00:16:34,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +19: [2023-05-11 00:16:34,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +31: [2023-05-11 00:16:34,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +23: [2023-05-11 00:16:34,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +14: [2023-05-11 00:16:34,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 0: [2023-05-11 00:16:34,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +28: [2023-05-11 00:16:34,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +21: [2023-05-11 00:16:34,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +11: [2023-05-11 00:16:34,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +19: [2023-05-11 00:16:34,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +19: [2023-05-11 00:16:34,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +19: [2023-05-11 00:16:34,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +19: [2023-05-11 00:16:34,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +17: [2023-05-11 00:16:34,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +19: [2023-05-11 00:16:34,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +30: [2023-05-11 00:16:34,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +30: [2023-05-11 00:16:34,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +23: [2023-05-11 00:16:34,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +30: [2023-05-11 00:16:34,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +30: [2023-05-11 00:16:34,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +23: [2023-05-11 00:16:34,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +17: [2023-05-11 00:16:34,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +14: [2023-05-11 00:16:34,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +26: [2023-05-11 00:16:34,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +30: [2023-05-11 00:16:34,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +30: [2023-05-11 00:16:34,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +30: [2023-05-11 00:16:34,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +15: [2023-05-11 00:16:34,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +15: [2023-05-11 00:16:34,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +15: [2023-05-11 00:16:34,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +15: [2023-05-11 00:16:34,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +31: [2023-05-11 00:16:34,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +14: [2023-05-11 00:16:34,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +11: [2023-05-11 00:16:34,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +30: [2023-05-11 00:16:34,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +11: [2023-05-11 00:16:34,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +18: [2023-05-11 00:16:34,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +15: [2023-05-11 00:16:34,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +15: [2023-05-11 00:16:34,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +20: [2023-05-11 00:16:34,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +31: [2023-05-11 00:16:34,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +15: [2023-05-11 00:16:34,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +17: [2023-05-11 00:16:34,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +28: [2023-05-11 00:16:34,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +23: [2023-05-11 00:16:34,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +14: [2023-05-11 00:16:34,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +28: [2023-05-11 00:16:34,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +18: [2023-05-11 00:16:34,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +14: [2023-05-11 00:16:34,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +18: [2023-05-11 00:16:34,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +28: [2023-05-11 00:16:34,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +23: [2023-05-11 00:16:34,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +18: [2023-05-11 00:16:34,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +15: [2023-05-11 00:16:34,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +17: [2023-05-11 00:16:34,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +14: [2023-05-11 00:16:34,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +31: [2023-05-11 00:16:34,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +19: [2023-05-11 00:16:34,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +28: [2023-05-11 00:16:34,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +11: [2023-05-11 00:16:34,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +30: [2023-05-11 00:16:34,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +31: [2023-05-11 00:16:34,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +17: [2023-05-11 00:16:34,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +14: [2023-05-11 00:16:34,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +15: [2023-05-11 00:16:34,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +14: [2023-05-11 00:16:34,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +23: [2023-05-11 00:16:34,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +30: [2023-05-11 00:16:34,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +15: [2023-05-11 00:16:34,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +24: [2023-05-11 00:16:34,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +24: [2023-05-11 00:16:34,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +24: [2023-05-11 00:16:34,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +24: [2023-05-11 00:16:34,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +24: [2023-05-11 00:16:34,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +24: [2023-05-11 00:16:34,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +24: [2023-05-11 00:16:34,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +18: [2023-05-11 00:16:34,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +17: [2023-05-11 00:16:34,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +14: [2023-05-11 00:16:34,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +14: [2023-05-11 00:16:34,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 0: [2023-05-11 00:16:34,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +24: [2023-05-11 00:16:34,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 2: [2023-05-11 00:16:34,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 5: [2023-05-11 00:16:34,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 9: [2023-05-11 00:16:34,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +28: [2023-05-11 00:16:34,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +15: [2023-05-11 00:16:34,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +19: [2023-05-11 00:16:34,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +30: [2023-05-11 00:16:34,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +31: [2023-05-11 00:16:34,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +16: [2023-05-11 00:16:34,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +16: [2023-05-11 00:16:34,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +16: [2023-05-11 00:16:34,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 1: [2023-05-11 00:16:34,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +30: [2023-05-11 00:16:34,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +14: [2023-05-11 00:16:34,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +14: [2023-05-11 00:16:34,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 1: [2023-05-11 00:16:34,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +15: [2023-05-11 00:16:34,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +19: [2023-05-11 00:16:34,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +15: [2023-05-11 00:16:34,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +16: [2023-05-11 00:16:34,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 9: [2023-05-11 00:16:34,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +30: [2023-05-11 00:16:34,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +30: [2023-05-11 00:16:34,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +15: [2023-05-11 00:16:34,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +24: [2023-05-11 00:16:34,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +19: [2023-05-11 00:16:34,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +30: [2023-05-11 00:16:34,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +29: [2023-05-11 00:16:34,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +29: [2023-05-11 00:16:34,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +16: [2023-05-11 00:16:34,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +19: [2023-05-11 00:16:34,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +29: [2023-05-11 00:16:34,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +29: [2023-05-11 00:16:34,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 5: [2023-05-11 00:16:34,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 5: [2023-05-11 00:16:34,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +14: [2023-05-11 00:16:34,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +14: [2023-05-11 00:16:34,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 5: [2023-05-11 00:16:34,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 8: [2023-05-11 00:16:34,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 8: [2023-05-11 00:16:34,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 8: [2023-05-11 00:16:34,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 8: [2023-05-11 00:16:34,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 5: [2023-05-11 00:16:34,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +16: [2023-05-11 00:16:34,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +24: [2023-05-11 00:16:34,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +24: [2023-05-11 00:16:34,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +19: [2023-05-11 00:16:34,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +16: [2023-05-11 00:16:34,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +16: [2023-05-11 00:16:34,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +30: [2023-05-11 00:16:34,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +24: [2023-05-11 00:16:34,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +19: [2023-05-11 00:16:34,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +19: [2023-05-11 00:16:34,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +15: [2023-05-11 00:16:34,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +29: [2023-05-11 00:16:34,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +29: [2023-05-11 00:16:34,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +29: [2023-05-11 00:16:34,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +29: [2023-05-11 00:16:34,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +24: [2023-05-11 00:16:34,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +24: [2023-05-11 00:16:34,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +16: [2023-05-11 00:16:34,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +16: [2023-05-11 00:16:34,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 9: [2023-05-11 00:16:34,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +15: [2023-05-11 00:16:34,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +16: [2023-05-11 00:16:34,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +16: [2023-05-11 00:16:34,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +25: [2023-05-11 00:16:34,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +25: [2023-05-11 00:16:34,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +25: [2023-05-11 00:16:34,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +25: [2023-05-11 00:16:34,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +25: [2023-05-11 00:16:34,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +25: [2023-05-11 00:16:34,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +25: [2023-05-11 00:16:34,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 2: [2023-05-11 00:16:34,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 2: [2023-05-11 00:16:34,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +24: [2023-05-11 00:16:34,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +25: [2023-05-11 00:16:34,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +29: [2023-05-11 00:16:34,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +29: [2023-05-11 00:16:34,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +29: [2023-05-11 00:16:34,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +29: [2023-05-11 00:16:34,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +29: [2023-05-11 00:16:34,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +29: [2023-05-11 00:16:34,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 5: [2023-05-11 00:16:34,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +29: [2023-05-11 00:16:34,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +29: [2023-05-11 00:16:34,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 5: [2023-05-11 00:16:34,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 5: [2023-05-11 00:16:34,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 8: [2023-05-11 00:16:34,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 8: [2023-05-11 00:16:34,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 8: [2023-05-11 00:16:34,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 2: [2023-05-11 00:16:34,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 5: [2023-05-11 00:16:34,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +16: [2023-05-11 00:16:34,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +16: [2023-05-11 00:16:34,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +16: [2023-05-11 00:16:34,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +25: [2023-05-11 00:16:34,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +25: [2023-05-11 00:16:34,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +25: [2023-05-11 00:16:34,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +16: [2023-05-11 00:16:34,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +24: [2023-05-11 00:16:34,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +26: [2023-05-11 00:16:34,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +26: [2023-05-11 00:16:34,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +26: [2023-05-11 00:16:34,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 5: [2023-05-11 00:16:34,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +26: [2023-05-11 00:16:34,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +13: [2023-05-11 00:16:34,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +13: [2023-05-11 00:16:34,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +13: [2023-05-11 00:16:34,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 9: [2023-05-11 00:16:34,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 9: [2023-05-11 00:16:34,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 9: [2023-05-11 00:16:34,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +13: [2023-05-11 00:16:34,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 9: [2023-05-11 00:16:34,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +29: [2023-05-11 00:16:34,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +29: [2023-05-11 00:16:34,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +29: [2023-05-11 00:16:34,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +29: [2023-05-11 00:16:34,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +29: [2023-05-11 00:16:34,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +29: [2023-05-11 00:16:34,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +29: [2023-05-11 00:16:34,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +29: [2023-05-11 00:16:34,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +25: [2023-05-11 00:16:34,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +25: [2023-05-11 00:16:34,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +26: [2023-05-11 00:16:34,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +25: [2023-05-11 00:16:34,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +26: [2023-05-11 00:16:34,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +30: [2023-05-11 00:16:34,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +30: [2023-05-11 00:16:34,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +30: [2023-05-11 00:16:34,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +30: [2023-05-11 00:16:34,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +13: [2023-05-11 00:16:34,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +15: [2023-05-11 00:16:34,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +15: [2023-05-11 00:16:34,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +15: [2023-05-11 00:16:34,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +15: [2023-05-11 00:16:34,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +26: [2023-05-11 00:16:34,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +13: [2023-05-11 00:16:34,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +13: [2023-05-11 00:16:34,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +26: [2023-05-11 00:16:34,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +13: [2023-05-11 00:16:34,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +25: [2023-05-11 00:16:34,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +19: [2023-05-11 00:16:34,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +19: [2023-05-11 00:16:34,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +19: [2023-05-11 00:16:34,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +19: [2023-05-11 00:16:34,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +29: [2023-05-11 00:16:34,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +26: [2023-05-11 00:16:34,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +30: [2023-05-11 00:16:34,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +15: [2023-05-11 00:16:34,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +26: [2023-05-11 00:16:34,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 8: [2023-05-11 00:16:34,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +30: [2023-05-11 00:16:34,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +15: [2023-05-11 00:16:34,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +30: [2023-05-11 00:16:34,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +30: [2023-05-11 00:16:34,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +15: [2023-05-11 00:16:34,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +13: [2023-05-11 00:16:34,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +26: [2023-05-11 00:16:34,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +26: [2023-05-11 00:16:34,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +13: [2023-05-11 00:16:34,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 4: [2023-05-11 00:16:34,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +15: [2023-05-11 00:16:34,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +28: [2023-05-11 00:16:34,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +28: [2023-05-11 00:16:34,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 4: [2023-05-11 00:16:34,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +29: [2023-05-11 00:16:34,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +29: [2023-05-11 00:16:34,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +26: [2023-05-11 00:16:34,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +26: [2023-05-11 00:16:34,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +28: [2023-05-11 00:16:34,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +29: [2023-05-11 00:16:34,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +28: [2023-05-11 00:16:34,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +29: [2023-05-11 00:16:34,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +25: [2023-05-11 00:16:34,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +13: [2023-05-11 00:16:34,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +13: [2023-05-11 00:16:34,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +13: [2023-05-11 00:16:34,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +13: [2023-05-11 00:16:34,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +13: [2023-05-11 00:16:34,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +19: [2023-05-11 00:16:34,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +19: [2023-05-11 00:16:34,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +13: [2023-05-11 00:16:34,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +19: [2023-05-11 00:16:34,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 7: [2023-05-11 00:16:34,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +21: [2023-05-11 00:16:34,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +21: [2023-05-11 00:16:34,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +21: [2023-05-11 00:16:34,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +21: [2023-05-11 00:16:34,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 9: [2023-05-11 00:16:34,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +19: [2023-05-11 00:16:34,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 8: [2023-05-11 00:16:34,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +11: [2023-05-11 00:16:34,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +11: [2023-05-11 00:16:34,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 8: [2023-05-11 00:16:34,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +20: [2023-05-11 00:16:34,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +20: [2023-05-11 00:16:34,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +20: [2023-05-11 00:16:34,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +20: [2023-05-11 00:16:34,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +15: [2023-05-11 00:16:34,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +15: [2023-05-11 00:16:34,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 8: [2023-05-11 00:16:34,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 0: [2023-05-11 00:16:34,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 0: [2023-05-11 00:16:34,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 4: [2023-05-11 00:16:34,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 8: [2023-05-11 00:16:34,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +28: [2023-05-11 00:16:34,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +15: [2023-05-11 00:16:34,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +28: [2023-05-11 00:16:34,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +15: [2023-05-11 00:16:34,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +28: [2023-05-11 00:16:34,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +11: [2023-05-11 00:16:34,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +11: [2023-05-11 00:16:34,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +31: [2023-05-11 00:16:34,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +28: [2023-05-11 00:16:34,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +31: [2023-05-11 00:16:34,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +31: [2023-05-11 00:16:34,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +31: [2023-05-11 00:16:34,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 4: [2023-05-11 00:16:34,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 7: [2023-05-11 00:16:34,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +29: [2023-05-11 00:16:34,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +29: [2023-05-11 00:16:34,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +29: [2023-05-11 00:16:34,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +26: [2023-05-11 00:16:34,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +11: [2023-05-11 00:16:34,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +21: [2023-05-11 00:16:34,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +21: [2023-05-11 00:16:34,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +20: [2023-05-11 00:16:34,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +21: [2023-05-11 00:16:34,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +21: [2023-05-11 00:16:34,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +26: [2023-05-11 00:16:34,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +20: [2023-05-11 00:16:34,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +19: [2023-05-11 00:16:34,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +19: [2023-05-11 00:16:34,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +11: [2023-05-11 00:16:34,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +30: [2023-05-11 00:16:34,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +30: [2023-05-11 00:16:34,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +30: [2023-05-11 00:16:34,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +30: [2023-05-11 00:16:34,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +20: [2023-05-11 00:16:34,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +30: [2023-05-11 00:16:34,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +30: [2023-05-11 00:16:34,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +11: [2023-05-11 00:16:34,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 1: [2023-05-11 00:16:34,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 1: [2023-05-11 00:16:34,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 1: [2023-05-11 00:16:34,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 0: [2023-05-11 00:16:34,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +31: [2023-05-11 00:16:34,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +31: [2023-05-11 00:16:34,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +15: [2023-05-11 00:16:34,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +20: [2023-05-11 00:16:34,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +28: [2023-05-11 00:16:34,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +11: [2023-05-11 00:16:34,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +15: [2023-05-11 00:16:34,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +28: [2023-05-11 00:16:34,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 9: [2023-05-11 00:16:34,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +31: [2023-05-11 00:16:34,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +19: [2023-05-11 00:16:34,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +19: [2023-05-11 00:16:34,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +19: [2023-05-11 00:16:34,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +31: [2023-05-11 00:16:34,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +19: [2023-05-11 00:16:34,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +19: [2023-05-11 00:16:34,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +19: [2023-05-11 00:16:34,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +30: [2023-05-11 00:16:34,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +30: [2023-05-11 00:16:34,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +12: [2023-05-11 00:16:34,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +12: [2023-05-11 00:16:34,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +12: [2023-05-11 00:16:34,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 9: [2023-05-11 00:16:34,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +12: [2023-05-11 00:16:34,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +28: [2023-05-11 00:16:34,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +28: [2023-05-11 00:16:34,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +28: [2023-05-11 00:16:34,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +28: [2023-05-11 00:16:34,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 4: [2023-05-11 00:16:34,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 7: [2023-05-11 00:16:34,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +15: [2023-05-11 00:16:34,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +21: [2023-05-11 00:16:34,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +21: [2023-05-11 00:16:34,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +28: [2023-05-11 00:16:34,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +28: [2023-05-11 00:16:34,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +15: [2023-05-11 00:16:34,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 1: [2023-05-11 00:16:34,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +11: [2023-05-11 00:16:34,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +20: [2023-05-11 00:16:34,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +20: [2023-05-11 00:16:34,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 0: [2023-05-11 00:16:34,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +21: [2023-05-11 00:16:34,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +21: [2023-05-11 00:16:34,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 0: [2023-05-11 00:16:34,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +11: [2023-05-11 00:16:34,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 1: [2023-05-11 00:16:34,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +11: [2023-05-11 00:16:34,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +11: [2023-05-11 00:16:34,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 7: [2023-05-11 00:16:34,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +21: [2023-05-11 00:16:34,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +21: [2023-05-11 00:16:34,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +21: [2023-05-11 00:16:34,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +21: [2023-05-11 00:16:34,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 7: [2023-05-11 00:16:34,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 6: [2023-05-11 00:16:34,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 6: [2023-05-11 00:16:34,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 7: [2023-05-11 00:16:34,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 1: [2023-05-11 00:16:34,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +25: [2023-05-11 00:16:34,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +25: [2023-05-11 00:16:34,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +25: [2023-05-11 00:16:34,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +25: [2023-05-11 00:16:34,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +11: [2023-05-11 00:16:34,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +11: [2023-05-11 00:16:34,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +27: [2023-05-11 00:16:34,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +27: [2023-05-11 00:16:34,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +27: [2023-05-11 00:16:34,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +27: [2023-05-11 00:16:34,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 6: [2023-05-11 00:16:34,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +18: [2023-05-11 00:16:34,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +18: [2023-05-11 00:16:34,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +11: [2023-05-11 00:16:34,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +12: [2023-05-11 00:16:34,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +12: [2023-05-11 00:16:34,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +12: [2023-05-11 00:16:34,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +20: [2023-05-11 00:16:34,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +20: [2023-05-11 00:16:34,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 4: [2023-05-11 00:16:34,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 4: [2023-05-11 00:16:34,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +12: [2023-05-11 00:16:34,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +20: [2023-05-11 00:16:34,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +11: [2023-05-11 00:16:34,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +20: [2023-05-11 00:16:34,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +26: [2023-05-11 00:16:34,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +26: [2023-05-11 00:16:34,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +26: [2023-05-11 00:16:34,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +26: [2023-05-11 00:16:34,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +26: [2023-05-11 00:16:34,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +26: [2023-05-11 00:16:34,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +31: [2023-05-11 00:16:34,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +17: [2023-05-11 00:16:34,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +17: [2023-05-11 00:16:34,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +17: [2023-05-11 00:16:34,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +17: [2023-05-11 00:16:34,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +31: [2023-05-11 00:16:34,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +26: [2023-05-11 00:16:34,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +26: [2023-05-11 00:16:34,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +18: [2023-05-11 00:16:34,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 0: [2023-05-11 00:16:34,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +18: [2023-05-11 00:16:34,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 6: [2023-05-11 00:16:34,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +10: [2023-05-11 00:16:34,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +10: [2023-05-11 00:16:34,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +10: [2023-05-11 00:16:34,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +10: [2023-05-11 00:16:34,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +25: [2023-05-11 00:16:34,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +31: [2023-05-11 00:16:34,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +25: [2023-05-11 00:16:34,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 6: [2023-05-11 00:16:34,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +31: [2023-05-11 00:16:34,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +27: [2023-05-11 00:16:34,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +18: [2023-05-11 00:16:34,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +25: [2023-05-11 00:16:34,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +20: [2023-05-11 00:16:34,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +20: [2023-05-11 00:16:34,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +20: [2023-05-11 00:16:34,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +20: [2023-05-11 00:16:34,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +20: [2023-05-11 00:16:34,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +20: [2023-05-11 00:16:34,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +27: [2023-05-11 00:16:34,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +20: [2023-05-11 00:16:34,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +18: [2023-05-11 00:16:34,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +31: [2023-05-11 00:16:34,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +27: [2023-05-11 00:16:34,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +27: [2023-05-11 00:16:34,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +31: [2023-05-11 00:16:34,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +10: [2023-05-11 00:16:34,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +10: [2023-05-11 00:16:34,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +10: [2023-05-11 00:16:34,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +10: [2023-05-11 00:16:34,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +20: [2023-05-11 00:16:34,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +10: [2023-05-11 00:16:34,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +17: [2023-05-11 00:16:34,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +17: [2023-05-11 00:16:34,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +25: [2023-05-11 00:16:34,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +17: [2023-05-11 00:16:34,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +10: [2023-05-11 00:16:34,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +13: [2023-05-11 00:16:34,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +13: [2023-05-11 00:16:34,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +13: [2023-05-11 00:16:34,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +13: [2023-05-11 00:16:34,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +13: [2023-05-11 00:16:34,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +10: [2023-05-11 00:16:34,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +17: [2023-05-11 00:16:34,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +13: [2023-05-11 00:16:34,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +13: [2023-05-11 00:16:34,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +13: [2023-05-11 00:16:34,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +18: [2023-05-11 00:16:34,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +31: [2023-05-11 00:16:34,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +18: [2023-05-11 00:16:34,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +10: [2023-05-11 00:16:34,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 0: [2023-05-11 00:16:34,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +10: [2023-05-11 00:16:34,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +24: [2023-05-11 00:16:34,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 1: [2023-05-11 00:16:34,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +16: [2023-05-11 00:16:34,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +16: [2023-05-11 00:16:34,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +16: [2023-05-11 00:16:34,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +16: [2023-05-11 00:16:34,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +31: [2023-05-11 00:16:34,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +24: [2023-05-11 00:16:34,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +10: [2023-05-11 00:16:34,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +16: [2023-05-11 00:16:34,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +16: [2023-05-11 00:16:34,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +16: [2023-05-11 00:16:34,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +24: [2023-05-11 00:16:34,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +10: [2023-05-11 00:16:34,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +24: [2023-05-11 00:16:34,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +25: [2023-05-11 00:16:34,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +12: [2023-05-11 00:16:34,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +25: [2023-05-11 00:16:34,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +16: [2023-05-11 00:16:34,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +12: [2023-05-11 00:16:34,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +12: [2023-05-11 00:16:34,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +12: [2023-05-11 00:16:34,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +10: [2023-05-11 00:16:34,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 6: [2023-05-11 00:16:34,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +20: [2023-05-11 00:16:34,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +14: [2023-05-11 00:16:34,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +14: [2023-05-11 00:16:34,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +14: [2023-05-11 00:16:34,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +14: [2023-05-11 00:16:34,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +14: [2023-05-11 00:16:34,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +14: [2023-05-11 00:16:34,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +14: [2023-05-11 00:16:34,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +12: [2023-05-11 00:16:34,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +12: [2023-05-11 00:16:34,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +12: [2023-05-11 00:16:34,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +12: [2023-05-11 00:16:34,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +12: [2023-05-11 00:16:34,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +12: [2023-05-11 00:16:34,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +12: [2023-05-11 00:16:34,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +24: [2023-05-11 00:16:34,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +12: [2023-05-11 00:16:34,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +17: [2023-05-11 00:16:34,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +12: [2023-05-11 00:16:34,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +17: [2023-05-11 00:16:34,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +10: [2023-05-11 00:16:34,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +17: [2023-05-11 00:16:34,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +17: [2023-05-11 00:16:34,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +12: [2023-05-11 00:16:34,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +17: [2023-05-11 00:16:34,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +26: [2023-05-11 00:16:34,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +26: [2023-05-11 00:16:34,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +17: [2023-05-11 00:16:34,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +14: [2023-05-11 00:16:34,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +20: [2023-05-11 00:16:34,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +20: [2023-05-11 00:16:34,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +20: [2023-05-11 00:16:34,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +26: [2023-05-11 00:16:34,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +27: [2023-05-11 00:16:34,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +27: [2023-05-11 00:16:34,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +13: [2023-05-11 00:16:34,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +24: [2023-05-11 00:16:34,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +20: [2023-05-11 00:16:34,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +24: [2023-05-11 00:16:34,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +27: [2023-05-11 00:16:34,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +27: [2023-05-11 00:16:34,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +27: [2023-05-11 00:16:34,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +27: [2023-05-11 00:16:34,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +27: [2023-05-11 00:16:34,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +27: [2023-05-11 00:16:34,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +24: [2023-05-11 00:16:34,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +27: [2023-05-11 00:16:34,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +13: [2023-05-11 00:16:34,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +16: [2023-05-11 00:16:34,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +27: [2023-05-11 00:16:34,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 6: [2023-05-11 00:16:34,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +18: [2023-05-11 00:16:34,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +25: [2023-05-11 00:16:34,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +25: [2023-05-11 00:16:34,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +26: [2023-05-11 00:16:34,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +12: [2023-05-11 00:16:34,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +27: [2023-05-11 00:16:34,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +25: [2023-05-11 00:16:34,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +27: [2023-05-11 00:16:34,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +10: [2023-05-11 00:16:34,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +18: [2023-05-11 00:16:34,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +16: [2023-05-11 00:16:34,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +18: [2023-05-11 00:16:34,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +18: [2023-05-11 00:16:34,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +18: [2023-05-11 00:16:34,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +18: [2023-05-11 00:16:34,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +13: [2023-05-11 00:16:34,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +13: [2023-05-11 00:16:34,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +12: [2023-05-11 00:16:34,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +25: [2023-05-11 00:16:34,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +16: [2023-05-11 00:16:34,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +10: [2023-05-11 00:16:34,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +10: [2023-05-11 00:16:34,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 6: [2023-05-11 00:16:34,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +18: [2023-05-11 00:16:34,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +18: [2023-05-11 00:16:34,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +14: [2023-05-11 00:16:34,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +10: [2023-05-11 00:16:34,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +10: [2023-05-11 00:16:34,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +20: [2023-05-11 00:16:34,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +20: [2023-05-11 00:16:34,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +13: [2023-05-11 00:16:34,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +13: [2023-05-11 00:16:34,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +12: [2023-05-11 00:16:34,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +26: [2023-05-11 00:16:34,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +26: [2023-05-11 00:16:34,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +26: [2023-05-11 00:16:34,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +17: [2023-05-11 00:16:34,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +16: [2023-05-11 00:16:34,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +25: [2023-05-11 00:16:34,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +17: [2023-05-11 00:16:34,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +26: [2023-05-11 00:16:34,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +20: [2023-05-11 00:16:34,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +13: [2023-05-11 00:16:34,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +10: [2023-05-11 00:16:34,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +13: [2023-05-11 00:16:34,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +24: [2023-05-11 00:16:34,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +14: [2023-05-11 00:16:34,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +24: [2023-05-11 00:16:34,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +25: [2023-05-11 00:16:34,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +24: [2023-05-11 00:16:34,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +24: [2023-05-11 00:16:34,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +20: [2023-05-11 00:16:34,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +27: [2023-05-11 00:16:34,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +27: [2023-05-11 00:16:34,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +16: [2023-05-11 00:16:34,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +16: [2023-05-11 00:16:34,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +10: [2023-05-11 00:16:34,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +10: [2023-05-11 00:16:34,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +12: [2023-05-11 00:16:34,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +24: [2023-05-11 00:16:34,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +24: [2023-05-11 00:16:34,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +12: [2023-05-11 00:16:34,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +10: [2023-05-11 00:16:34,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +10: [2023-05-11 00:16:34,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +27: [2023-05-11 00:16:34,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +27: [2023-05-11 00:16:34,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +14: [2023-05-11 00:16:34,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +10: [2023-05-11 00:16:34,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +14: [2023-05-11 00:16:34,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +12: [2023-05-11 00:16:34,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +27: [2023-05-11 00:16:34,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +24: [2023-05-11 00:16:34,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +24: [2023-05-11 00:16:34,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +22: [2023-05-11 00:16:34,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +22: [2023-05-11 00:16:34,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +22: [2023-05-11 00:16:34,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +22: [2023-05-11 00:16:34,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. + 6: [2023-05-11 00:16:34,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +20: [2023-05-11 00:16:34,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +14: [2023-05-11 00:16:34,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +27: [2023-05-11 00:16:34,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +27: [2023-05-11 00:16:34,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +16: [2023-05-11 00:16:34,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +27: [2023-05-11 00:16:34,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +27: [2023-05-11 00:16:34,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +23: [2023-05-11 00:16:34,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +23: [2023-05-11 00:16:34,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +14: [2023-05-11 00:16:34,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +23: [2023-05-11 00:16:34,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +12: [2023-05-11 00:16:34,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +12: [2023-05-11 00:16:34,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +23: [2023-05-11 00:16:34,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt. +27: [2023-05-11 00:16:34,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +27: [2023-05-11 00:16:34,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +22: [2023-05-11 00:16:34,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +12: [2023-05-11 00:16:34,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +22: [2023-05-11 00:16:34,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +10: [2023-05-11 00:16:34,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +22: [2023-05-11 00:16:34,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +22: [2023-05-11 00:16:34,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +16: [2023-05-11 00:16:34,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +14: [2023-05-11 00:16:34,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +27: [2023-05-11 00:16:34,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +23: [2023-05-11 00:16:34,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +23: [2023-05-11 00:16:34,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +23: [2023-05-11 00:16:34,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +12: [2023-05-11 00:16:34,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +23: [2023-05-11 00:16:34,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt... +18: [2023-05-11 00:16:34,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +11: [2023-05-11 00:16:34,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +11: [2023-05-11 00:16:34,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +11: [2023-05-11 00:16:34,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +11: [2023-05-11 00:16:34,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +18: [2023-05-11 00:16:34,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +10: [2023-05-11 00:16:34,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +11: [2023-05-11 00:16:34,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +18: [2023-05-11 00:16:34,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +18: [2023-05-11 00:16:34,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +11: [2023-05-11 00:16:34,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +11: [2023-05-11 00:16:34,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +10: [2023-05-11 00:16:34,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +11: [2023-05-11 00:16:34,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +22: [2023-05-11 00:16:34,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +22: [2023-05-11 00:16:34,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +22: [2023-05-11 00:16:34,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +22: [2023-05-11 00:16:34,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +10: [2023-05-11 00:16:34,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +22: [2023-05-11 00:16:34,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +22: [2023-05-11 00:16:34,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +22: [2023-05-11 00:16:34,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +22: [2023-05-11 00:16:34,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +18: [2023-05-11 00:16:34,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +18: [2023-05-11 00:16:34,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +18: [2023-05-11 00:16:34,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +14: [2023-05-11 00:16:34,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +22: [2023-05-11 00:16:34,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +18: [2023-05-11 00:16:34,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 3: [2023-05-11 00:16:34,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +22: [2023-05-11 00:16:34,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +22: [2023-05-11 00:16:34,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +22: [2023-05-11 00:16:34,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +22: [2023-05-11 00:16:34,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +22: [2023-05-11 00:16:34,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 3: [2023-05-11 00:16:34,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +22: [2023-05-11 00:16:34,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +22: [2023-05-11 00:16:34,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +11: [2023-05-11 00:16:34,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +17: [2023-05-11 00:16:34,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +17: [2023-05-11 00:16:34,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +17: [2023-05-11 00:16:34,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +17: [2023-05-11 00:16:34,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +17: [2023-05-11 00:16:34,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +17: [2023-05-11 00:16:34,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +17: [2023-05-11 00:16:34,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +23: [2023-05-11 00:16:34,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +23: [2023-05-11 00:16:34,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +17: [2023-05-11 00:16:34,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +23: [2023-05-11 00:16:34,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +23: [2023-05-11 00:16:34,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +11: [2023-05-11 00:16:34,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +23: [2023-05-11 00:16:34,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +23: [2023-05-11 00:16:34,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +23: [2023-05-11 00:16:34,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +23: [2023-05-11 00:16:34,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +23: [2023-05-11 00:16:34,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +23: [2023-05-11 00:16:34,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +18: [2023-05-11 00:16:34,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +18: [2023-05-11 00:16:34,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +23: [2023-05-11 00:16:34,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +23: [2023-05-11 00:16:34,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +11: [2023-05-11 00:16:34,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +22: [2023-05-11 00:16:34,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +22: [2023-05-11 00:16:34,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +18: [2023-05-11 00:16:34,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +11: [2023-05-11 00:16:34,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +19: [2023-05-11 00:16:34,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +19: [2023-05-11 00:16:34,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +31: [2023-05-11 00:16:34,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +31: [2023-05-11 00:16:34,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +31: [2023-05-11 00:16:34,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +31: [2023-05-11 00:16:34,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +19: [2023-05-11 00:16:34,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +19: [2023-05-11 00:16:34,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +28: [2023-05-11 00:16:34,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +28: [2023-05-11 00:16:34,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +28: [2023-05-11 00:16:34,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +28: [2023-05-11 00:16:34,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +31: [2023-05-11 00:16:34,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +31: [2023-05-11 00:16:34,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +31: [2023-05-11 00:16:34,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +28: [2023-05-11 00:16:34,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +19: [2023-05-11 00:16:34,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +28: [2023-05-11 00:16:34,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +19: [2023-05-11 00:16:34,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +24: [2023-05-11 00:16:34,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +24: [2023-05-11 00:16:34,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +24: [2023-05-11 00:16:34,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +24: [2023-05-11 00:16:34,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +24: [2023-05-11 00:16:34,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +24: [2023-05-11 00:16:34,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +24: [2023-05-11 00:16:34,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +24: [2023-05-11 00:16:34,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +19: [2023-05-11 00:16:34,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +19: [2023-05-11 00:16:34,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +17: [2023-05-11 00:16:34,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +11: [2023-05-11 00:16:34,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +31: [2023-05-11 00:16:34,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +17: [2023-05-11 00:16:34,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +17: [2023-05-11 00:16:34,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +18: [2023-05-11 00:16:34,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +28: [2023-05-11 00:16:34,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 2: [2023-05-11 00:16:34,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +21: [2023-05-11 00:16:34,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +21: [2023-05-11 00:16:34,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +21: [2023-05-11 00:16:34,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +21: [2023-05-11 00:16:34,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +21: [2023-05-11 00:16:34,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +21: [2023-05-11 00:16:34,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +21: [2023-05-11 00:16:34,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +28: [2023-05-11 00:16:34,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +11: [2023-05-11 00:16:34,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +18: [2023-05-11 00:16:34,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +22: [2023-05-11 00:16:34,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +22: [2023-05-11 00:16:34,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +21: [2023-05-11 00:16:34,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +11: [2023-05-11 00:16:34,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +23: [2023-05-11 00:16:34,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +23: [2023-05-11 00:16:34,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +23: [2023-05-11 00:16:34,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +29: [2023-05-11 00:16:34,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +29: [2023-05-11 00:16:34,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +29: [2023-05-11 00:16:34,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +29: [2023-05-11 00:16:34,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +18: [2023-05-11 00:16:34,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +23: [2023-05-11 00:16:34,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +23: [2023-05-11 00:16:34,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... + 3: [2023-05-11 00:16:34,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +22: [2023-05-11 00:16:34,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +23: [2023-05-11 00:16:34,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt. +31: [2023-05-11 00:16:34,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +17: [2023-05-11 00:16:34,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +22: [2023-05-11 00:16:34,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +23: [2023-05-11 00:16:34,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +24: [2023-05-11 00:16:34,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +18: [2023-05-11 00:16:34,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +23: [2023-05-11 00:16:34,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt... +19: [2023-05-11 00:16:34,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +19: [2023-05-11 00:16:34,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +31: [2023-05-11 00:16:34,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +31: [2023-05-11 00:16:34,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +11: [2023-05-11 00:16:34,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +30: [2023-05-11 00:16:34,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +30: [2023-05-11 00:16:34,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +19: [2023-05-11 00:16:34,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +17: [2023-05-11 00:16:34,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +17: [2023-05-11 00:16:34,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +17: [2023-05-11 00:16:34,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +28: [2023-05-11 00:16:34,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +28: [2023-05-11 00:16:34,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +28: [2023-05-11 00:16:34,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +30: [2023-05-11 00:16:34,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +30: [2023-05-11 00:16:34,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +30: [2023-05-11 00:16:34,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +19: [2023-05-11 00:16:34,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +30: [2023-05-11 00:16:34,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +29: [2023-05-11 00:16:34,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +23: [2023-05-11 00:16:34,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +23: [2023-05-11 00:16:34,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +29: [2023-05-11 00:16:34,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +22: [2023-05-11 00:16:34,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +30: [2023-05-11 00:16:34,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +30: [2023-05-11 00:16:34,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +31: [2023-05-11 00:16:34,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +24: [2023-05-11 00:16:34,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +24: [2023-05-11 00:16:34,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +22: [2023-05-11 00:16:34,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +24: [2023-05-11 00:16:34,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +29: [2023-05-11 00:16:34,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +29: [2023-05-11 00:16:34,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +23: [2023-05-11 00:16:34,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +18: [2023-05-11 00:16:34,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +23: [2023-05-11 00:16:34,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +31: [2023-05-11 00:16:34,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 8: [2023-05-11 00:16:34,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 8: [2023-05-11 00:16:34,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 8: [2023-05-11 00:16:34,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +28: [2023-05-11 00:16:34,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +24: [2023-05-11 00:16:34,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +17: [2023-05-11 00:16:34,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +19: [2023-05-11 00:16:34,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +21: [2023-05-11 00:16:34,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +21: [2023-05-11 00:16:34,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +28: [2023-05-11 00:16:34,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +19: [2023-05-11 00:16:34,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +28: [2023-05-11 00:16:34,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +21: [2023-05-11 00:16:34,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +28: [2023-05-11 00:16:34,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +19: [2023-05-11 00:16:34,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +31: [2023-05-11 00:16:34,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +24: [2023-05-11 00:16:34,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +24: [2023-05-11 00:16:34,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +30: [2023-05-11 00:16:34,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +30: [2023-05-11 00:16:34,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +24: [2023-05-11 00:16:34,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +19: [2023-05-11 00:16:34,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +29: [2023-05-11 00:16:34,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +29: [2023-05-11 00:16:34,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +31: [2023-05-11 00:16:34,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +21: [2023-05-11 00:16:34,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +28: [2023-05-11 00:16:34,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +21: [2023-05-11 00:16:34,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +21: [2023-05-11 00:16:34,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +21: [2023-05-11 00:16:34,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +30: [2023-05-11 00:16:34,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +29: [2023-05-11 00:16:34,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +29: [2023-05-11 00:16:34,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 8: [2023-05-11 00:16:34,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +30: [2023-05-11 00:16:34,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +30: [2023-05-11 00:16:34,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +21: [2023-05-11 00:16:34,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +31: [2023-05-11 00:16:34,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +29: [2023-05-11 00:16:34,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +30: [2023-05-11 00:16:34,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +29: [2023-05-11 00:16:34,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +30: [2023-05-11 00:16:34,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +29: [2023-05-11 00:16:34,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 0: [2023-05-11 00:16:34,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +29: [2023-05-11 00:16:34,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 7: [2023-05-11 00:16:34,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 8: [2023-05-11 00:16:34,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 0: [2023-05-11 00:16:34,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 4: [2023-05-11 00:16:34,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 4: [2023-05-11 00:16:34,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 4: [2023-05-11 00:16:34,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +21: [2023-05-11 00:16:34,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +21: [2023-05-11 00:16:34,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +21: [2023-05-11 00:16:34,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +21: [2023-05-11 00:16:34,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +30: [2023-05-11 00:16:34,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 4: [2023-05-11 00:16:34,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +21: [2023-05-11 00:16:34,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +21: [2023-05-11 00:16:34,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +21: [2023-05-11 00:16:34,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +21: [2023-05-11 00:16:34,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 5: [2023-05-11 00:16:34,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 5: [2023-05-11 00:16:34,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 0: [2023-05-11 00:16:34,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +15: [2023-05-11 00:16:34,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +15: [2023-05-11 00:16:34,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +15: [2023-05-11 00:16:34,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +15: [2023-05-11 00:16:34,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +15: [2023-05-11 00:16:34,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +15: [2023-05-11 00:16:34,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +15: [2023-05-11 00:16:34,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +15: [2023-05-11 00:16:34,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +21: [2023-05-11 00:16:34,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +21: [2023-05-11 00:16:34,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 5: [2023-05-11 00:16:34,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +21: [2023-05-11 00:16:34,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +21: [2023-05-11 00:16:34,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 5: [2023-05-11 00:16:34,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +15: [2023-05-11 00:16:34,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 4: [2023-05-11 00:16:34,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 4: [2023-05-11 00:16:34,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 5: [2023-05-11 00:16:34,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +15: [2023-05-11 00:16:34,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +19: [2023-05-11 00:16:34,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +19: [2023-05-11 00:16:34,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +19: [2023-05-11 00:16:34,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +19: [2023-05-11 00:16:34,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 0: [2023-05-11 00:16:34,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +15: [2023-05-11 00:16:34,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +15: [2023-05-11 00:16:34,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +21: [2023-05-11 00:16:34,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +21: [2023-05-11 00:16:34,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +21: [2023-05-11 00:16:34,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 0: [2023-05-11 00:16:34,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +10: [2023-05-11 00:16:34,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +10: [2023-05-11 00:16:34,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +10: [2023-05-11 00:16:34,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +21: [2023-05-11 00:16:34,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 9: [2023-05-11 00:16:34,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +15: [2023-05-11 00:16:34,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 5: [2023-05-11 00:16:34,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +10: [2023-05-11 00:16:34,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +15: [2023-05-11 00:16:34,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +15: [2023-05-11 00:16:34,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +19: [2023-05-11 00:16:34,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 9: [2023-05-11 00:16:34,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 5: [2023-05-11 00:16:34,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 5: [2023-05-11 00:16:34,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 1: [2023-05-11 00:16:34,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 1: [2023-05-11 00:16:34,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 1: [2023-05-11 00:16:34,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +19: [2023-05-11 00:16:34,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +10: [2023-05-11 00:16:34,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +15: [2023-05-11 00:16:34,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +19: [2023-05-11 00:16:34,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +19: [2023-05-11 00:16:34,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +13: [2023-05-11 00:16:34,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +13: [2023-05-11 00:16:34,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +13: [2023-05-11 00:16:34,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +13: [2023-05-11 00:16:34,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +10: [2023-05-11 00:16:34,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +10: [2023-05-11 00:16:34,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +12: [2023-05-11 00:16:34,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +12: [2023-05-11 00:16:34,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +12: [2023-05-11 00:16:34,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +30: [2023-05-11 00:16:34,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +30: [2023-05-11 00:16:34,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +30: [2023-05-11 00:16:34,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +30: [2023-05-11 00:16:34,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 1: [2023-05-11 00:16:34,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +10: [2023-05-11 00:16:34,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +12: [2023-05-11 00:16:34,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +15: [2023-05-11 00:16:34,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +15: [2023-05-11 00:16:34,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +29: [2023-05-11 00:16:34,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +29: [2023-05-11 00:16:34,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +29: [2023-05-11 00:16:34,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +15: [2023-05-11 00:16:34,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +29: [2023-05-11 00:16:34,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +15: [2023-05-11 00:16:34,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 1: [2023-05-11 00:16:34,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +29: [2023-05-11 00:16:34,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 1: [2023-05-11 00:16:34,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +29: [2023-05-11 00:16:34,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +29: [2023-05-11 00:16:34,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +29: [2023-05-11 00:16:34,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +13: [2023-05-11 00:16:34,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +13: [2023-05-11 00:16:34,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +13: [2023-05-11 00:16:34,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +12: [2023-05-11 00:16:34,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +13: [2023-05-11 00:16:34,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +15: [2023-05-11 00:16:34,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +12: [2023-05-11 00:16:34,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +12: [2023-05-11 00:16:34,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +12: [2023-05-11 00:16:34,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +15: [2023-05-11 00:16:34,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +30: [2023-05-11 00:16:34,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 5: [2023-05-11 00:16:34,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +30: [2023-05-11 00:16:34,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +15: [2023-05-11 00:16:34,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +15: [2023-05-11 00:16:34,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 9: [2023-05-11 00:16:34,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +30: [2023-05-11 00:16:34,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +30: [2023-05-11 00:16:34,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +19: [2023-05-11 00:16:34,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +14: [2023-05-11 00:16:34,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +14: [2023-05-11 00:16:34,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +14: [2023-05-11 00:16:34,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +14: [2023-05-11 00:16:34,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +19: [2023-05-11 00:16:34,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 9: [2023-05-11 00:16:34,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +13: [2023-05-11 00:16:34,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +13: [2023-05-11 00:16:34,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +14: [2023-05-11 00:16:34,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +14: [2023-05-11 00:16:34,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +19: [2023-05-11 00:16:34,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +29: [2023-05-11 00:16:34,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +14: [2023-05-11 00:16:34,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 1: [2023-05-11 00:16:34,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 1: [2023-05-11 00:16:34,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 9: [2023-05-11 00:16:34,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +19: [2023-05-11 00:16:34,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +19: [2023-05-11 00:16:34,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 9: [2023-05-11 00:16:34,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +14: [2023-05-11 00:16:34,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +19: [2023-05-11 00:16:34,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +13: [2023-05-11 00:16:34,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +15: [2023-05-11 00:16:34,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +12: [2023-05-11 00:16:34,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +12: [2023-05-11 00:16:34,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +19: [2023-05-11 00:16:34,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +13: [2023-05-11 00:16:34,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +15: [2023-05-11 00:16:34,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +15: [2023-05-11 00:16:34,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +15: [2023-05-11 00:16:34,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +19: [2023-05-11 00:16:34,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +13: [2023-05-11 00:16:34,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +13: [2023-05-11 00:16:34,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +29: [2023-05-11 00:16:34,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +29: [2023-05-11 00:16:34,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +15: [2023-05-11 00:16:34,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +15: [2023-05-11 00:16:34,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 1: [2023-05-11 00:16:34,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +12: [2023-05-11 00:16:34,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +12: [2023-05-11 00:16:34,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +12: [2023-05-11 00:16:34,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +12: [2023-05-11 00:16:34,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 1: [2023-05-11 00:16:34,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +14: [2023-05-11 00:16:34,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +20: [2023-05-11 00:16:34,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +20: [2023-05-11 00:16:34,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +20: [2023-05-11 00:16:34,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +20: [2023-05-11 00:16:34,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +13: [2023-05-11 00:16:34,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +13: [2023-05-11 00:16:34,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 9: [2023-05-11 00:16:34,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +15: [2023-05-11 00:16:34,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +15: [2023-05-11 00:16:34,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 9: [2023-05-11 00:16:34,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 9: [2023-05-11 00:16:34,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +29: [2023-05-11 00:16:34,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +29: [2023-05-11 00:16:34,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +25: [2023-05-11 00:16:34,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +25: [2023-05-11 00:16:34,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +25: [2023-05-11 00:16:34,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +25: [2023-05-11 00:16:34,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +25: [2023-05-11 00:16:34,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +25: [2023-05-11 00:16:34,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +14: [2023-05-11 00:16:34,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +25: [2023-05-11 00:16:34,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +25: [2023-05-11 00:16:34,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +30: [2023-05-11 00:16:34,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +12: [2023-05-11 00:16:34,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +20: [2023-05-11 00:16:34,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +30: [2023-05-11 00:16:34,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +20: [2023-05-11 00:16:34,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +20: [2023-05-11 00:16:34,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +16: [2023-05-11 00:16:34,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +16: [2023-05-11 00:16:34,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +16: [2023-05-11 00:16:34,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +16: [2023-05-11 00:16:34,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +16: [2023-05-11 00:16:34,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +16: [2023-05-11 00:16:34,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +30: [2023-05-11 00:16:34,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +16: [2023-05-11 00:16:34,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +12: [2023-05-11 00:16:34,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 9: [2023-05-11 00:16:34,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +14: [2023-05-11 00:16:34,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +14: [2023-05-11 00:16:34,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +29: [2023-05-11 00:16:34,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +20: [2023-05-11 00:16:34,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +16: [2023-05-11 00:16:34,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +30: [2023-05-11 00:16:34,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +30: [2023-05-11 00:16:34,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +30: [2023-05-11 00:16:34,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +29: [2023-05-11 00:16:34,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +30: [2023-05-11 00:16:34,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +14: [2023-05-11 00:16:34,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +30: [2023-05-11 00:16:34,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +25: [2023-05-11 00:16:34,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +20: [2023-05-11 00:16:34,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +25: [2023-05-11 00:16:34,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +20: [2023-05-11 00:16:34,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +25: [2023-05-11 00:16:34,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +29: [2023-05-11 00:16:34,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +25: [2023-05-11 00:16:34,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +16: [2023-05-11 00:16:34,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +14: [2023-05-11 00:16:34,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +27: [2023-05-11 00:16:34,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +27: [2023-05-11 00:16:34,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +27: [2023-05-11 00:16:34,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +27: [2023-05-11 00:16:34,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +27: [2023-05-11 00:16:34,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +27: [2023-05-11 00:16:34,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +27: [2023-05-11 00:16:34,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +14: [2023-05-11 00:16:34,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +27: [2023-05-11 00:16:34,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +16: [2023-05-11 00:16:34,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +16: [2023-05-11 00:16:34,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +20: [2023-05-11 00:16:34,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +20: [2023-05-11 00:16:34,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +25: [2023-05-11 00:16:34,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +20: [2023-05-11 00:16:34,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +20: [2023-05-11 00:16:34,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +25: [2023-05-11 00:16:34,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +28: [2023-05-11 00:16:34,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +28: [2023-05-11 00:16:34,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +28: [2023-05-11 00:16:34,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +28: [2023-05-11 00:16:34,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +25: [2023-05-11 00:16:34,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +16: [2023-05-11 00:16:34,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +14: [2023-05-11 00:16:34,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +25: [2023-05-11 00:16:34,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +26: [2023-05-11 00:16:34,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +26: [2023-05-11 00:16:34,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +26: [2023-05-11 00:16:34,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +18: [2023-05-11 00:16:34,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +18: [2023-05-11 00:16:34,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +26: [2023-05-11 00:16:34,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +18: [2023-05-11 00:16:34,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +14: [2023-05-11 00:16:34,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +14: [2023-05-11 00:16:34,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +14: [2023-05-11 00:16:34,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +14: [2023-05-11 00:16:34,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +18: [2023-05-11 00:16:34,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 6: [2023-05-11 00:16:34,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +16: [2023-05-11 00:16:34,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +27: [2023-05-11 00:16:34,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 6: [2023-05-11 00:16:34,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 6: [2023-05-11 00:16:34,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +17: [2023-05-11 00:16:34,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +17: [2023-05-11 00:16:34,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +17: [2023-05-11 00:16:34,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +16: [2023-05-11 00:16:34,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +27: [2023-05-11 00:16:34,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +27: [2023-05-11 00:16:34,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +17: [2023-05-11 00:16:34,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 7: [2023-05-11 00:16:34,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 7: [2023-05-11 00:16:34,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 7: [2023-05-11 00:16:34,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 7: [2023-05-11 00:16:34,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +20: [2023-05-11 00:16:34,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +28: [2023-05-11 00:16:34,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +28: [2023-05-11 00:16:34,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +16: [2023-05-11 00:16:34,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +28: [2023-05-11 00:16:34,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +20: [2023-05-11 00:16:34,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +28: [2023-05-11 00:16:34,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +26: [2023-05-11 00:16:34,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +26: [2023-05-11 00:16:34,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +14: [2023-05-11 00:16:34,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +27: [2023-05-11 00:16:34,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +29: [2023-05-11 00:16:34,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +29: [2023-05-11 00:16:34,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 6: [2023-05-11 00:16:34,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +16: [2023-05-11 00:16:34,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +16: [2023-05-11 00:16:34,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +18: [2023-05-11 00:16:34,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +27: [2023-05-11 00:16:34,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +18: [2023-05-11 00:16:34,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +18: [2023-05-11 00:16:34,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +26: [2023-05-11 00:16:34,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +26: [2023-05-11 00:16:34,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +26: [2023-05-11 00:16:34,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +26: [2023-05-11 00:16:34,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +26: [2023-05-11 00:16:34,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +26: [2023-05-11 00:16:34,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +16: [2023-05-11 00:16:34,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +14: [2023-05-11 00:16:34,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +17: [2023-05-11 00:16:34,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +26: [2023-05-11 00:16:34,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +26: [2023-05-11 00:16:34,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +14: [2023-05-11 00:16:34,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +16: [2023-05-11 00:16:34,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +14: [2023-05-11 00:16:34,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +27: [2023-05-11 00:16:34,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 2: [2023-05-11 00:16:34,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 2: [2023-05-11 00:16:34,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +17: [2023-05-11 00:16:34,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +17: [2023-05-11 00:16:34,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +23: [2023-05-11 00:16:34,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +23: [2023-05-11 00:16:34,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +17: [2023-05-11 00:16:34,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +22: [2023-05-11 00:16:34,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 7: [2023-05-11 00:16:34,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +22: [2023-05-11 00:16:34,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +26: [2023-05-11 00:16:34,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +22: [2023-05-11 00:16:34,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 7: [2023-05-11 00:16:34,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +29: [2023-05-11 00:16:34,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +23: [2023-05-11 00:16:34,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +23: [2023-05-11 00:16:34,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +29: [2023-05-11 00:16:34,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +16: [2023-05-11 00:16:34,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +18: [2023-05-11 00:16:34,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 7: [2023-05-11 00:16:34,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +26: [2023-05-11 00:16:34,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +22: [2023-05-11 00:16:34,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +27: [2023-05-11 00:16:34,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +29: [2023-05-11 00:16:34,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +16: [2023-05-11 00:16:34,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +16: [2023-05-11 00:16:34,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +29: [2023-05-11 00:16:34,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +26: [2023-05-11 00:16:34,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +26: [2023-05-11 00:16:34,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +16: [2023-05-11 00:16:34,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +26: [2023-05-11 00:16:34,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +26: [2023-05-11 00:16:34,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +22: [2023-05-11 00:16:34,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +23: [2023-05-11 00:16:34,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +29: [2023-05-11 00:16:34,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +29: [2023-05-11 00:16:34,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +28: [2023-05-11 00:16:34,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +22: [2023-05-11 00:16:34,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +14: [2023-05-11 00:16:34,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +14: [2023-05-11 00:16:34,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +28: [2023-05-11 00:16:34,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +22: [2023-05-11 00:16:34,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +16: [2023-05-11 00:16:34,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +23: [2023-05-11 00:16:34,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +14: [2023-05-11 00:16:34,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +23: [2023-05-11 00:16:34,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +14: [2023-05-11 00:16:34,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +17: [2023-05-11 00:16:34,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +23: [2023-05-11 00:16:34,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +17: [2023-05-11 00:16:34,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +17: [2023-05-11 00:16:34,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +17: [2023-05-11 00:16:34,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +18: [2023-05-11 00:16:34,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +18: [2023-05-11 00:16:34,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 2: [2023-05-11 00:16:34,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +27: [2023-05-11 00:16:34,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +12: [2023-05-11 00:16:34,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +17: [2023-05-11 00:16:34,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +12: [2023-05-11 00:16:34,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +17: [2023-05-11 00:16:34,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 2: [2023-05-11 00:16:34,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +12: [2023-05-11 00:16:34,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +12: [2023-05-11 00:16:34,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +22: [2023-05-11 00:16:34,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +17: [2023-05-11 00:16:34,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +17: [2023-05-11 00:16:34,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +27: [2023-05-11 00:16:34,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 7: [2023-05-11 00:16:34,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +12: [2023-05-11 00:16:34,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +12: [2023-05-11 00:16:34,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +28: [2023-05-11 00:16:34,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +25: [2023-05-11 00:16:34,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +25: [2023-05-11 00:16:34,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +31: [2023-05-11 00:16:34,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +31: [2023-05-11 00:16:34,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +28: [2023-05-11 00:16:34,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +27: [2023-05-11 00:16:34,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +25: [2023-05-11 00:16:34,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +27: [2023-05-11 00:16:34,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +12: [2023-05-11 00:16:34,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +12: [2023-05-11 00:16:34,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +25: [2023-05-11 00:16:34,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +27: [2023-05-11 00:16:34,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +28: [2023-05-11 00:16:34,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 6: [2023-05-11 00:16:34,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +14: [2023-05-11 00:16:34,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +14: [2023-05-11 00:16:34,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +28: [2023-05-11 00:16:34,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +28: [2023-05-11 00:16:34,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +18: [2023-05-11 00:16:34,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +18: [2023-05-11 00:16:34,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +31: [2023-05-11 00:16:34,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 6: [2023-05-11 00:16:34,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 6: [2023-05-11 00:16:34,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 4: [2023-05-11 00:16:34,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +31: [2023-05-11 00:16:34,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +28: [2023-05-11 00:16:34,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 7: [2023-05-11 00:16:34,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 4: [2023-05-11 00:16:34,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +29: [2023-05-11 00:16:34,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +29: [2023-05-11 00:16:34,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +29: [2023-05-11 00:16:34,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +29: [2023-05-11 00:16:34,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 0: [2023-05-11 00:16:34,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 0: [2023-05-11 00:16:34,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +22: [2023-05-11 00:16:34,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +22: [2023-05-11 00:16:34,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 0: [2023-05-11 00:16:34,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +27: [2023-05-11 00:16:34,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +22: [2023-05-11 00:16:34,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +22: [2023-05-11 00:16:34,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +16: [2023-05-11 00:16:34,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +18: [2023-05-11 00:16:34,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +16: [2023-05-11 00:16:34,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +16: [2023-05-11 00:16:34,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +16: [2023-05-11 00:16:34,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +25: [2023-05-11 00:16:34,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +29: [2023-05-11 00:16:34,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +29: [2023-05-11 00:16:34,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +24: [2023-05-11 00:16:34,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +24: [2023-05-11 00:16:34,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +22: [2023-05-11 00:16:34,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +22: [2023-05-11 00:16:34,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +18: [2023-05-11 00:16:34,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +25: [2023-05-11 00:16:34,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +24: [2023-05-11 00:16:34,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +31: [2023-05-11 00:16:34,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +31: [2023-05-11 00:16:34,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +12: [2023-05-11 00:16:34,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +24: [2023-05-11 00:16:34,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +23: [2023-05-11 00:16:34,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +27: [2023-05-11 00:16:34,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +31: [2023-05-11 00:16:34,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +26: [2023-05-11 00:16:34,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +21: [2023-05-11 00:16:34,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +21: [2023-05-11 00:16:34,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +23: [2023-05-11 00:16:34,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +26: [2023-05-11 00:16:34,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +25: [2023-05-11 00:16:34,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +27: [2023-05-11 00:16:34,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +25: [2023-05-11 00:16:34,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +27: [2023-05-11 00:16:34,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +18: [2023-05-11 00:16:34,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +18: [2023-05-11 00:16:34,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +18: [2023-05-11 00:16:34,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +18: [2023-05-11 00:16:34,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +18: [2023-05-11 00:16:34,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +18: [2023-05-11 00:16:34,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +18: [2023-05-11 00:16:34,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +18: [2023-05-11 00:16:34,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +16: [2023-05-11 00:16:34,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +16: [2023-05-11 00:16:34,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +14: [2023-05-11 00:16:34,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +21: [2023-05-11 00:16:34,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 7: [2023-05-11 00:16:34,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +21: [2023-05-11 00:16:34,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +31: [2023-05-11 00:16:34,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +12: [2023-05-11 00:16:34,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +26: [2023-05-11 00:16:34,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +18: [2023-05-11 00:16:34,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +23: [2023-05-11 00:16:34,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +23: [2023-05-11 00:16:34,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 7: [2023-05-11 00:16:34,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 4: [2023-05-11 00:16:34,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 7: [2023-05-11 00:16:34,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +14: [2023-05-11 00:16:34,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +29: [2023-05-11 00:16:34,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +26: [2023-05-11 00:16:34,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 4: [2023-05-11 00:16:34,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +24: [2023-05-11 00:16:34,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +18: [2023-05-11 00:16:34,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +26: [2023-05-11 00:16:34,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +26: [2023-05-11 00:16:34,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +29: [2023-05-11 00:16:34,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 2: [2023-05-11 00:16:34,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 2: [2023-05-11 00:16:34,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 0: [2023-05-11 00:16:34,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +21: [2023-05-11 00:16:34,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +24: [2023-05-11 00:16:34,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +26: [2023-05-11 00:16:34,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +23: [2023-05-11 00:16:34,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +22: [2023-05-11 00:16:34,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +26: [2023-05-11 00:16:34,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +25: [2023-05-11 00:16:34,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +16: [2023-05-11 00:16:34,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +25: [2023-05-11 00:16:34,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +24: [2023-05-11 00:16:34,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +22: [2023-05-11 00:16:34,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +12: [2023-05-11 00:16:34,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +23: [2023-05-11 00:16:34,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +21: [2023-05-11 00:16:34,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +24: [2023-05-11 00:16:34,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +16: [2023-05-11 00:16:34,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +31: [2023-05-11 00:16:34,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 8: [2023-05-11 00:16:34,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 8: [2023-05-11 00:16:34,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 8: [2023-05-11 00:16:34,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 3: [2023-05-11 00:16:34,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +23: [2023-05-11 00:16:34,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +27: [2023-05-11 00:16:34,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +12: [2023-05-11 00:16:34,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +21: [2023-05-11 00:16:34,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +27: [2023-05-11 00:16:34,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 3: [2023-05-11 00:16:34,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +21: [2023-05-11 00:16:34,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +23: [2023-05-11 00:16:34,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +26: [2023-05-11 00:16:34,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +25: [2023-05-11 00:16:34,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +25: [2023-05-11 00:16:34,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +25: [2023-05-11 00:16:34,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +25: [2023-05-11 00:16:34,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +31: [2023-05-11 00:16:34,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +18: [2023-05-11 00:16:34,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +18: [2023-05-11 00:16:34,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +18: [2023-05-11 00:16:34,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +18: [2023-05-11 00:16:34,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +25: [2023-05-11 00:16:34,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +25: [2023-05-11 00:16:34,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 4: [2023-05-11 00:16:34,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +27: [2023-05-11 00:16:34,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +27: [2023-05-11 00:16:34,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +27: [2023-05-11 00:16:34,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +27: [2023-05-11 00:16:34,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +27: [2023-05-11 00:16:34,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +27: [2023-05-11 00:16:34,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +31: [2023-05-11 00:16:34,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +31: [2023-05-11 00:16:34,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +12: [2023-05-11 00:16:34,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +26: [2023-05-11 00:16:34,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +12: [2023-05-11 00:16:34,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +20: [2023-05-11 00:16:34,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +20: [2023-05-11 00:16:34,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +21: [2023-05-11 00:16:34,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +20: [2023-05-11 00:16:34,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +20: [2023-05-11 00:16:34,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +31: [2023-05-11 00:16:34,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +31: [2023-05-11 00:16:34,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 8: [2023-05-11 00:16:34,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +26: [2023-05-11 00:16:34,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +24: [2023-05-11 00:16:34,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +21: [2023-05-11 00:16:34,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +24: [2023-05-11 00:16:34,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +24: [2023-05-11 00:16:34,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +24: [2023-05-11 00:16:34,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 2: [2023-05-11 00:16:34,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 8: [2023-05-11 00:16:34,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +24: [2023-05-11 00:16:34,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +24: [2023-05-11 00:16:34,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 6: [2023-05-11 00:16:34,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +20: [2023-05-11 00:16:34,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +20: [2023-05-11 00:16:34,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +20: [2023-05-11 00:16:34,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +21: [2023-05-11 00:16:34,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +21: [2023-05-11 00:16:34,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 4: [2023-05-11 00:16:34,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +13: [2023-05-11 00:16:34,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +13: [2023-05-11 00:16:34,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +13: [2023-05-11 00:16:34,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +13: [2023-05-11 00:16:34,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +13: [2023-05-11 00:16:34,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +18: [2023-05-11 00:16:34,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +18: [2023-05-11 00:16:34,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +18: [2023-05-11 00:16:34,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +18: [2023-05-11 00:16:34,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +21: [2023-05-11 00:16:34,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +21: [2023-05-11 00:16:34,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +21: [2023-05-11 00:16:34,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +21: [2023-05-11 00:16:34,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 6: [2023-05-11 00:16:34,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +26: [2023-05-11 00:16:34,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +20: [2023-05-11 00:16:34,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +31: [2023-05-11 00:16:34,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +12: [2023-05-11 00:16:34,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +31: [2023-05-11 00:16:34,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +12: [2023-05-11 00:16:34,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +28: [2023-05-11 00:16:34,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +28: [2023-05-11 00:16:34,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +28: [2023-05-11 00:16:34,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +28: [2023-05-11 00:16:34,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +10: [2023-05-11 00:16:34,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +10: [2023-05-11 00:16:34,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +28: [2023-05-11 00:16:34,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +28: [2023-05-11 00:16:34,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +28: [2023-05-11 00:16:34,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +11: [2023-05-11 00:16:34,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +11: [2023-05-11 00:16:34,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +11: [2023-05-11 00:16:34,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +11: [2023-05-11 00:16:34,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +11: [2023-05-11 00:16:34,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +11: [2023-05-11 00:16:34,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +10: [2023-05-11 00:16:34,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +11: [2023-05-11 00:16:34,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 4: [2023-05-11 00:16:34,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 4: [2023-05-11 00:16:34,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +24: [2023-05-11 00:16:34,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +10: [2023-05-11 00:16:34,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +28: [2023-05-11 00:16:34,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 8: [2023-05-11 00:16:34,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +11: [2023-05-11 00:16:34,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +10: [2023-05-11 00:16:34,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +24: [2023-05-11 00:16:34,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +10: [2023-05-11 00:16:34,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +10: [2023-05-11 00:16:34,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +10: [2023-05-11 00:16:34,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +10: [2023-05-11 00:16:34,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +10: [2023-05-11 00:16:34,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +10: [2023-05-11 00:16:34,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +10: [2023-05-11 00:16:34,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +10: [2023-05-11 00:16:34,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +10: [2023-05-11 00:16:34,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +10: [2023-05-11 00:16:34,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 3: [2023-05-11 00:16:34,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +10: [2023-05-11 00:16:34,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +20: [2023-05-11 00:16:34,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +11: [2023-05-11 00:16:34,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +11: [2023-05-11 00:16:34,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 3: [2023-05-11 00:16:34,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 8: [2023-05-11 00:16:34,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +20: [2023-05-11 00:16:34,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +10: [2023-05-11 00:16:34,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 8: [2023-05-11 00:16:34,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 8: [2023-05-11 00:16:34,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 6: [2023-05-11 00:16:34,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 6: [2023-05-11 00:16:34,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +20: [2023-05-11 00:16:34,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +11: [2023-05-11 00:16:34,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 4: [2023-05-11 00:16:34,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +11: [2023-05-11 00:16:34,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 3: [2023-05-11 00:16:34,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. +28: [2023-05-11 00:16:34,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt. + 2: [2023-05-11 00:16:34,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +11: [2023-05-11 00:16:34,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +20: [2023-05-11 00:16:34,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +11: [2023-05-11 00:16:34,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 2: [2023-05-11 00:16:34,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 2: [2023-05-11 00:16:34,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +10: [2023-05-11 00:16:34,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +28: [2023-05-11 00:16:34,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +10: [2023-05-11 00:16:34,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +20: [2023-05-11 00:16:34,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +28: [2023-05-11 00:16:34,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +20: [2023-05-11 00:16:34,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +11: [2023-05-11 00:16:34,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +10: [2023-05-11 00:16:34,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +11: [2023-05-11 00:16:34,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +28: [2023-05-11 00:16:34,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +11: [2023-05-11 00:16:34,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 6: [2023-05-11 00:16:34,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +11: [2023-05-11 00:16:34,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +11: [2023-05-11 00:16:34,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +11: [2023-05-11 00:16:34,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt... +28: [2023-05-11 00:16:34,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +10: [2023-05-11 00:16:34,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +17: [2023-05-11 00:16:34,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +17: [2023-05-11 00:16:34,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +17: [2023-05-11 00:16:34,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +11: [2023-05-11 00:16:34,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +17: [2023-05-11 00:16:34,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +17: [2023-05-11 00:16:34,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +17: [2023-05-11 00:16:34,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +13: [2023-05-11 00:16:34,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +13: [2023-05-11 00:16:34,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +28: [2023-05-11 00:16:34,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +13: [2023-05-11 00:16:34,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +13: [2023-05-11 00:16:34,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +13: [2023-05-11 00:16:34,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +13: [2023-05-11 00:16:34,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +13: [2023-05-11 00:16:34,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +13: [2023-05-11 00:16:34,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +13: [2023-05-11 00:16:34,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +13: [2023-05-11 00:16:34,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +13: [2023-05-11 00:16:34,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +13: [2023-05-11 00:16:34,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +13: [2023-05-11 00:16:34,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +17: [2023-05-11 00:16:34,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +20: [2023-05-11 00:16:34,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +10: [2023-05-11 00:16:34,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +17: [2023-05-11 00:16:34,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +10: [2023-05-11 00:16:34,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 4: [2023-05-11 00:16:34,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 4: [2023-05-11 00:16:34,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +28: [2023-05-11 00:16:34,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +11: [2023-05-11 00:16:34,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 4: [2023-05-11 00:16:34,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 4: [2023-05-11 00:16:34,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 8: [2023-05-11 00:16:34,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +20: [2023-05-11 00:16:34,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +10: [2023-05-11 00:16:34,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 8: [2023-05-11 00:16:34,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +17: [2023-05-11 00:16:34,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 1: [2023-05-11 00:16:34,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +28: [2023-05-11 00:16:34,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 1: [2023-05-11 00:16:34,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +11: [2023-05-11 00:16:34,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +11: [2023-05-11 00:16:34,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +11: [2023-05-11 00:16:34,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +11: [2023-05-11 00:16:34,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +11: [2023-05-11 00:16:34,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 8: [2023-05-11 00:16:34,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +23: [2023-05-11 00:16:34,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +23: [2023-05-11 00:16:34,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +31: [2023-05-11 00:16:34,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +31: [2023-05-11 00:16:34,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +31: [2023-05-11 00:16:34,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +31: [2023-05-11 00:16:34,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +23: [2023-05-11 00:16:34,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +23: [2023-05-11 00:16:34,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +31: [2023-05-11 00:16:34,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +31: [2023-05-11 00:16:34,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +31: [2023-05-11 00:16:34,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +23: [2023-05-11 00:16:34,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 8: [2023-05-11 00:16:34,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +23: [2023-05-11 00:16:34,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +23: [2023-05-11 00:16:34,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +31: [2023-05-11 00:16:34,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +17: [2023-05-11 00:16:34,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +23: [2023-05-11 00:16:34,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +20: [2023-05-11 00:16:34,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +20: [2023-05-11 00:16:34,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +20: [2023-05-11 00:16:34,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +20: [2023-05-11 00:16:34,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 4: [2023-05-11 00:16:34,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 3: [2023-05-11 00:16:34,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 3: [2023-05-11 00:16:34,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +11: [2023-05-11 00:16:34,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 3: [2023-05-11 00:16:34,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 4: [2023-05-11 00:16:34,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +11: [2023-05-11 00:16:34,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +11: [2023-05-11 00:16:34,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt. +17: [2023-05-11 00:16:34,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +11: [2023-05-11 00:16:34,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... +11: [2023-05-11 00:16:34,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt... + 0: [2023-05-11 00:16:34,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +17: [2023-05-11 00:16:34,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +17: [2023-05-11 00:16:34,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +23: [2023-05-11 00:16:34,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +20: [2023-05-11 00:16:34,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +20: [2023-05-11 00:16:34,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +20: [2023-05-11 00:16:34,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +20: [2023-05-11 00:16:34,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +31: [2023-05-11 00:16:34,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +31: [2023-05-11 00:16:34,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +17: [2023-05-11 00:16:34,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +23: [2023-05-11 00:16:34,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +23: [2023-05-11 00:16:34,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +31: [2023-05-11 00:16:34,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +24: [2023-05-11 00:16:34,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +24: [2023-05-11 00:16:34,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +24: [2023-05-11 00:16:34,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +24: [2023-05-11 00:16:34,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +24: [2023-05-11 00:16:34,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +24: [2023-05-11 00:16:34,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +23: [2023-05-11 00:16:34,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +17: [2023-05-11 00:16:34,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +24: [2023-05-11 00:16:34,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +17: [2023-05-11 00:16:34,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +24: [2023-05-11 00:16:34,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +13: [2023-05-11 00:16:34,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +13: [2023-05-11 00:16:34,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +13: [2023-05-11 00:16:34,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +13: [2023-05-11 00:16:34,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +13: [2023-05-11 00:16:34,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +13: [2023-05-11 00:16:34,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +13: [2023-05-11 00:16:34,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +13: [2023-05-11 00:16:34,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +13: [2023-05-11 00:16:34,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +13: [2023-05-11 00:16:34,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +13: [2023-05-11 00:16:34,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +13: [2023-05-11 00:16:34,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +13: [2023-05-11 00:16:34,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +13: [2023-05-11 00:16:34,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +31: [2023-05-11 00:16:34,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +23: [2023-05-11 00:16:34,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +31: [2023-05-11 00:16:34,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +23: [2023-05-11 00:16:34,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +31: [2023-05-11 00:16:34,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +20: [2023-05-11 00:16:34,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +20: [2023-05-11 00:16:34,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 0: [2023-05-11 00:16:34,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +20: [2023-05-11 00:16:34,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +20: [2023-05-11 00:16:34,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 0: [2023-05-11 00:16:34,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +23: [2023-05-11 00:16:34,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +31: [2023-05-11 00:16:34,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +24: [2023-05-11 00:16:34,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +24: [2023-05-11 00:16:34,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +20: [2023-05-11 00:16:34,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +20: [2023-05-11 00:16:34,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +23: [2023-05-11 00:16:34,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +28: [2023-05-11 00:16:34,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +28: [2023-05-11 00:16:34,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +28: [2023-05-11 00:16:34,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +28: [2023-05-11 00:16:34,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +14: [2023-05-11 00:16:34,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +14: [2023-05-11 00:16:34,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +31: [2023-05-11 00:16:34,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +14: [2023-05-11 00:16:34,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +14: [2023-05-11 00:16:34,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 0: [2023-05-11 00:16:34,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 0: [2023-05-11 00:16:34,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +24: [2023-05-11 00:16:34,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 0: [2023-05-11 00:16:34,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +24: [2023-05-11 00:16:34,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +20: [2023-05-11 00:16:34,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +20: [2023-05-11 00:16:34,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +24: [2023-05-11 00:16:34,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +28: [2023-05-11 00:16:34,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +28: [2023-05-11 00:16:34,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +28: [2023-05-11 00:16:34,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +28: [2023-05-11 00:16:34,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +14: [2023-05-11 00:16:34,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +14: [2023-05-11 00:16:34,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +14: [2023-05-11 00:16:34,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +24: [2023-05-11 00:16:34,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +14: [2023-05-11 00:16:34,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +24: [2023-05-11 00:16:34,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +25: [2023-05-11 00:16:34,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +25: [2023-05-11 00:16:34,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +25: [2023-05-11 00:16:34,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +25: [2023-05-11 00:16:34,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +28: [2023-05-11 00:16:34,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +28: [2023-05-11 00:16:34,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 9: [2023-05-11 00:16:34,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 9: [2023-05-11 00:16:34,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 9: [2023-05-11 00:16:34,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 9: [2023-05-11 00:16:34,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +28: [2023-05-11 00:16:34,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +28: [2023-05-11 00:16:34,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +18: [2023-05-11 00:16:34,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +18: [2023-05-11 00:16:34,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +28: [2023-05-11 00:16:34,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +28: [2023-05-11 00:16:34,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +24: [2023-05-11 00:16:34,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +14: [2023-05-11 00:16:34,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +14: [2023-05-11 00:16:34,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +18: [2023-05-11 00:16:34,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +18: [2023-05-11 00:16:34,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +25: [2023-05-11 00:16:34,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +25: [2023-05-11 00:16:34,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +28: [2023-05-11 00:16:35,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +28: [2023-05-11 00:16:35,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +25: [2023-05-11 00:16:35,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +16: [2023-05-11 00:16:35,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +16: [2023-05-11 00:16:35,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +16: [2023-05-11 00:16:35,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +14: [2023-05-11 00:16:35,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +14: [2023-05-11 00:16:35,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 9: [2023-05-11 00:16:35,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +16: [2023-05-11 00:16:35,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +14: [2023-05-11 00:16:35,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +14: [2023-05-11 00:16:35,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +18: [2023-05-11 00:16:35,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +14: [2023-05-11 00:16:35,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +14: [2023-05-11 00:16:35,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 9: [2023-05-11 00:16:35,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +25: [2023-05-11 00:16:35,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +18: [2023-05-11 00:16:35,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 7: [2023-05-11 00:16:35,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +22: [2023-05-11 00:16:35,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +22: [2023-05-11 00:16:35,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +22: [2023-05-11 00:16:35,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +22: [2023-05-11 00:16:35,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +22: [2023-05-11 00:16:35,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +18: [2023-05-11 00:16:35,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +22: [2023-05-11 00:16:35,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +18: [2023-05-11 00:16:35,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +22: [2023-05-11 00:16:35,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +22: [2023-05-11 00:16:35,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +23: [2023-05-11 00:16:35,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +23: [2023-05-11 00:16:35,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +16: [2023-05-11 00:16:35,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +16: [2023-05-11 00:16:35,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 7: [2023-05-11 00:16:35,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 7: [2023-05-11 00:16:35,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +16: [2023-05-11 00:16:35,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +25: [2023-05-11 00:16:35,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +25: [2023-05-11 00:16:35,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 2: [2023-05-11 00:16:35,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +23: [2023-05-11 00:16:35,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 7: [2023-05-11 00:16:35,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +23: [2023-05-11 00:16:35,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 0: [2023-05-11 00:16:35,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +16: [2023-05-11 00:16:35,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 0: [2023-05-11 00:16:35,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +25: [2023-05-11 00:16:35,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +25: [2023-05-11 00:16:35,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 0: [2023-05-11 00:16:35,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 6: [2023-05-11 00:16:35,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 6: [2023-05-11 00:16:35,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 6: [2023-05-11 00:16:35,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 7: [2023-05-11 00:16:35,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +27: [2023-05-11 00:16:35,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +27: [2023-05-11 00:16:35,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 6: [2023-05-11 00:16:35,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +27: [2023-05-11 00:16:35,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +27: [2023-05-11 00:16:35,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +23: [2023-05-11 00:16:35,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +26: [2023-05-11 00:16:35,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +26: [2023-05-11 00:16:35,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +26: [2023-05-11 00:16:35,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +22: [2023-05-11 00:16:35,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +18: [2023-05-11 00:16:35,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +18: [2023-05-11 00:16:35,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +25: [2023-05-11 00:16:35,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 7: [2023-05-11 00:16:35,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +26: [2023-05-11 00:16:35,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +22: [2023-05-11 00:16:35,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +19: [2023-05-11 00:16:35,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +18: [2023-05-11 00:16:35,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +19: [2023-05-11 00:16:35,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +18: [2023-05-11 00:16:35,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +23: [2023-05-11 00:16:35,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +23: [2023-05-11 00:16:35,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +25: [2023-05-11 00:16:35,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +15: [2023-05-11 00:16:35,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +15: [2023-05-11 00:16:35,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +15: [2023-05-11 00:16:35,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +15: [2023-05-11 00:16:35,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +15: [2023-05-11 00:16:35,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +15: [2023-05-11 00:16:35,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +23: [2023-05-11 00:16:35,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +19: [2023-05-11 00:16:35,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +15: [2023-05-11 00:16:35,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +22: [2023-05-11 00:16:35,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 0: [2023-05-11 00:16:35,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +15: [2023-05-11 00:16:35,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +19: [2023-05-11 00:16:35,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +19: [2023-05-11 00:16:35,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +19: [2023-05-11 00:16:35,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +15: [2023-05-11 00:16:35,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +27: [2023-05-11 00:16:35,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +15: [2023-05-11 00:16:35,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +16: [2023-05-11 00:16:35,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +16: [2023-05-11 00:16:35,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +26: [2023-05-11 00:16:35,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +16: [2023-05-11 00:16:35,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +19: [2023-05-11 00:16:35,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +15: [2023-05-11 00:16:35,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +22: [2023-05-11 00:16:35,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +27: [2023-05-11 00:16:35,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +26: [2023-05-11 00:16:35,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +27: [2023-05-11 00:16:35,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +27: [2023-05-11 00:16:35,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +15: [2023-05-11 00:16:35,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +22: [2023-05-11 00:16:35,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +22: [2023-05-11 00:16:35,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +18: [2023-05-11 00:16:35,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +18: [2023-05-11 00:16:35,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 6: [2023-05-11 00:16:35,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +19: [2023-05-11 00:16:35,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +22: [2023-05-11 00:16:35,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 2: [2023-05-11 00:16:35,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +16: [2023-05-11 00:16:35,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +16: [2023-05-11 00:16:35,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 2: [2023-05-11 00:16:35,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +16: [2023-05-11 00:16:35,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +17: [2023-05-11 00:16:35,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +17: [2023-05-11 00:16:35,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +17: [2023-05-11 00:16:35,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +17: [2023-05-11 00:16:35,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +26: [2023-05-11 00:16:35,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +22: [2023-05-11 00:16:35,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 3: [2023-05-11 00:16:35,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 3: [2023-05-11 00:16:35,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +26: [2023-05-11 00:16:35,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +23: [2023-05-11 00:16:35,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +18: [2023-05-11 00:16:35,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +22: [2023-05-11 00:16:35,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +25: [2023-05-11 00:16:35,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +23: [2023-05-11 00:16:35,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +15: [2023-05-11 00:16:35,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +19: [2023-05-11 00:16:35,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +15: [2023-05-11 00:16:35,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +18: [2023-05-11 00:16:35,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +23: [2023-05-11 00:16:35,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +23: [2023-05-11 00:16:35,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +25: [2023-05-11 00:16:35,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 3: [2023-05-11 00:16:35,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +15: [2023-05-11 00:16:35,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +15: [2023-05-11 00:16:35,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 7: [2023-05-11 00:16:35,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +22: [2023-05-11 00:16:35,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +22: [2023-05-11 00:16:35,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +22: [2023-05-11 00:16:35,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +22: [2023-05-11 00:16:35,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +22: [2023-05-11 00:16:35,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +19: [2023-05-11 00:16:35,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +26: [2023-05-11 00:16:35,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +17: [2023-05-11 00:16:35,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +16: [2023-05-11 00:16:35,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +22: [2023-05-11 00:16:35,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +26: [2023-05-11 00:16:35,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 5: [2023-05-11 00:16:35,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +17: [2023-05-11 00:16:35,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +22: [2023-05-11 00:16:35,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +26: [2023-05-11 00:16:35,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +26: [2023-05-11 00:16:35,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +16: [2023-05-11 00:16:35,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 5: [2023-05-11 00:16:35,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +17: [2023-05-11 00:16:35,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +27: [2023-05-11 00:16:35,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +23: [2023-05-11 00:16:35,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +23: [2023-05-11 00:16:35,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 6: [2023-05-11 00:16:35,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 6: [2023-05-11 00:16:35,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 6: [2023-05-11 00:16:35,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +27: [2023-05-11 00:16:35,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 0: [2023-05-11 00:16:35,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +17: [2023-05-11 00:16:35,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +19: [2023-05-11 00:16:35,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +26: [2023-05-11 00:16:35,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +26: [2023-05-11 00:16:35,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +21: [2023-05-11 00:16:35,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +21: [2023-05-11 00:16:35,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +21: [2023-05-11 00:16:35,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +21: [2023-05-11 00:16:35,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +21: [2023-05-11 00:16:35,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +21: [2023-05-11 00:16:35,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +21: [2023-05-11 00:16:35,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +21: [2023-05-11 00:16:35,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +19: [2023-05-11 00:16:35,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +12: [2023-05-11 00:16:35,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +12: [2023-05-11 00:16:35,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +12: [2023-05-11 00:16:35,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +23: [2023-05-11 00:16:35,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +15: [2023-05-11 00:16:35,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +27: [2023-05-11 00:16:35,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 2: [2023-05-11 00:16:35,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +27: [2023-05-11 00:16:35,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +27: [2023-05-11 00:16:35,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 5: [2023-05-11 00:16:35,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +23: [2023-05-11 00:16:35,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +15: [2023-05-11 00:16:35,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 1: [2023-05-11 00:16:35,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 1: [2023-05-11 00:16:35,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 5: [2023-05-11 00:16:35,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +15: [2023-05-11 00:16:35,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +27: [2023-05-11 00:16:35,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 1: [2023-05-11 00:16:35,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +12: [2023-05-11 00:16:35,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 1: [2023-05-11 00:16:35,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +15: [2023-05-11 00:16:35,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 5: [2023-05-11 00:16:35,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +22: [2023-05-11 00:16:35,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +17: [2023-05-11 00:16:35,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +17: [2023-05-11 00:16:35,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +22: [2023-05-11 00:16:35,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 5: [2023-05-11 00:16:35,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 7: [2023-05-11 00:16:35,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +15: [2023-05-11 00:16:35,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +19: [2023-05-11 00:16:35,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +22: [2023-05-11 00:16:35,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +22: [2023-05-11 00:16:35,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +17: [2023-05-11 00:16:35,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +27: [2023-05-11 00:16:35,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +21: [2023-05-11 00:16:35,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 3: [2023-05-11 00:16:35,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +15: [2023-05-11 00:16:35,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +15: [2023-05-11 00:16:35,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 6: [2023-05-11 00:16:35,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +12: [2023-05-11 00:16:35,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +12: [2023-05-11 00:16:35,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +27: [2023-05-11 00:16:35,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +22: [2023-05-11 00:16:35,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +22: [2023-05-11 00:16:35,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 0: [2023-05-11 00:16:35,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +17: [2023-05-11 00:16:35,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +15: [2023-05-11 00:16:35,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +15: [2023-05-11 00:16:35,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +15: [2023-05-11 00:16:35,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +15: [2023-05-11 00:16:35,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +15: [2023-05-11 00:16:35,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 2: [2023-05-11 00:16:35,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 6: [2023-05-11 00:16:35,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +26: [2023-05-11 00:16:35,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +15: [2023-05-11 00:16:35,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 3: [2023-05-11 00:16:35,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +12: [2023-05-11 00:16:35,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +19: [2023-05-11 00:16:35,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +12: [2023-05-11 00:16:35,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +26: [2023-05-11 00:16:35,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +21: [2023-05-11 00:16:35,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +21: [2023-05-11 00:16:35,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +15: [2023-05-11 00:16:35,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +17: [2023-05-11 00:16:35,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +22: [2023-05-11 00:16:35,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +19: [2023-05-11 00:16:35,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +17: [2023-05-11 00:16:35,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 2: [2023-05-11 00:16:35,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 3: [2023-05-11 00:16:35,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +22: [2023-05-11 00:16:35,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 5: [2023-05-11 00:16:35,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 5: [2023-05-11 00:16:35,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +21: [2023-05-11 00:16:35,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +31: [2023-05-11 00:16:35,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 5: [2023-05-11 00:16:35,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +31: [2023-05-11 00:16:35,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +31: [2023-05-11 00:16:35,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +15: [2023-05-11 00:16:35,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +31: [2023-05-11 00:16:35,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 5: [2023-05-11 00:16:35,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +12: [2023-05-11 00:16:35,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +12: [2023-05-11 00:16:35,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +19: [2023-05-11 00:16:35,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +21: [2023-05-11 00:16:35,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +21: [2023-05-11 00:16:35,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +21: [2023-05-11 00:16:35,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +17: [2023-05-11 00:16:35,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +21: [2023-05-11 00:16:35,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +21: [2023-05-11 00:16:35,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +21: [2023-05-11 00:16:35,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +17: [2023-05-11 00:16:35,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... + 4: [2023-05-11 00:16:35,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +12: [2023-05-11 00:16:35,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +12: [2023-05-11 00:16:35,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +12: [2023-05-11 00:16:35,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +12: [2023-05-11 00:16:35,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +15: [2023-05-11 00:16:35,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 1: [2023-05-11 00:16:35,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 1: [2023-05-11 00:16:35,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +12: [2023-05-11 00:16:35,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +12: [2023-05-11 00:16:35,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 5: [2023-05-11 00:16:35,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +31: [2023-05-11 00:16:35,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +31: [2023-05-11 00:16:35,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +12: [2023-05-11 00:16:35,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +12: [2023-05-11 00:16:35,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +30: [2023-05-11 00:16:35,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +30: [2023-05-11 00:16:35,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +30: [2023-05-11 00:16:35,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +30: [2023-05-11 00:16:35,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +12: [2023-05-11 00:16:35,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +12: [2023-05-11 00:16:35,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +31: [2023-05-11 00:16:35,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +31: [2023-05-11 00:16:35,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +12: [2023-05-11 00:16:35,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +12: [2023-05-11 00:16:35,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +30: [2023-05-11 00:16:35,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +30: [2023-05-11 00:16:35,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +30: [2023-05-11 00:16:35,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +21: [2023-05-11 00:16:35,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +21: [2023-05-11 00:16:35,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +30: [2023-05-11 00:16:35,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +21: [2023-05-11 00:16:35,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +21: [2023-05-11 00:16:35,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +21: [2023-05-11 00:16:35,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +11: [2023-05-11 00:16:35,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +11: [2023-05-11 00:16:35,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 4: [2023-05-11 00:16:35,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 5: [2023-05-11 00:16:35,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +12: [2023-05-11 00:16:35,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +11: [2023-05-11 00:16:35,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. + 1: [2023-05-11 00:16:35,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +11: [2023-05-11 00:16:35,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt. +12: [2023-05-11 00:16:35,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +12: [2023-05-11 00:16:35,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 9: [2023-05-11 00:16:35,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +12: [2023-05-11 00:16:35,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 5: [2023-05-11 00:16:35,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 9: [2023-05-11 00:16:35,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 9: [2023-05-11 00:16:35,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +21: [2023-05-11 00:16:35,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +30: [2023-05-11 00:16:35,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +30: [2023-05-11 00:16:35,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 9: [2023-05-11 00:16:35,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +19: [2023-05-11 00:16:35,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +19: [2023-05-11 00:16:35,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +19: [2023-05-11 00:16:35,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +19: [2023-05-11 00:16:35,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 4: [2023-05-11 00:16:35,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +21: [2023-05-11 00:16:35,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +11: [2023-05-11 00:16:35,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +11: [2023-05-11 00:16:35,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +21: [2023-05-11 00:16:35,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +31: [2023-05-11 00:16:35,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +31: [2023-05-11 00:16:35,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +31: [2023-05-11 00:16:35,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +31: [2023-05-11 00:16:35,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +30: [2023-05-11 00:16:35,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +29: [2023-05-11 00:16:35,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +29: [2023-05-11 00:16:35,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +29: [2023-05-11 00:16:35,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +29: [2023-05-11 00:16:35,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +10: [2023-05-11 00:16:35,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +11: [2023-05-11 00:16:35,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +10: [2023-05-11 00:16:35,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +10: [2023-05-11 00:16:35,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +10: [2023-05-11 00:16:35,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +11: [2023-05-11 00:16:35,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt... +31: [2023-05-11 00:16:35,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +30: [2023-05-11 00:16:35,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +14: [2023-05-11 00:16:35,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +14: [2023-05-11 00:16:35,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +14: [2023-05-11 00:16:35,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +14: [2023-05-11 00:16:35,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +14: [2023-05-11 00:16:35,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +14: [2023-05-11 00:16:35,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +14: [2023-05-11 00:16:35,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +12: [2023-05-11 00:16:35,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +12: [2023-05-11 00:16:35,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +31: [2023-05-11 00:16:35,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +30: [2023-05-11 00:16:35,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +12: [2023-05-11 00:16:35,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +14: [2023-05-11 00:16:35,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +31: [2023-05-11 00:16:35,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +19: [2023-05-11 00:16:35,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +19: [2023-05-11 00:16:35,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +19: [2023-05-11 00:16:35,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +19: [2023-05-11 00:16:35,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +30: [2023-05-11 00:16:35,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +31: [2023-05-11 00:16:35,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +29: [2023-05-11 00:16:35,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +30: [2023-05-11 00:16:35,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +21: [2023-05-11 00:16:35,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +29: [2023-05-11 00:16:35,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +29: [2023-05-11 00:16:35,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +10: [2023-05-11 00:16:35,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +21: [2023-05-11 00:16:35,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +29: [2023-05-11 00:16:35,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +21: [2023-05-11 00:16:35,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +10: [2023-05-11 00:16:35,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +10: [2023-05-11 00:16:35,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +21: [2023-05-11 00:16:35,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 9: [2023-05-11 00:16:35,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +10: [2023-05-11 00:16:35,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +11: [2023-05-11 00:16:35,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +11: [2023-05-11 00:16:35,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +11: [2023-05-11 00:16:35,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +11: [2023-05-11 00:16:35,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +14: [2023-05-11 00:16:35,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 8: [2023-05-11 00:16:35,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 8: [2023-05-11 00:16:35,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +21: [2023-05-11 00:16:35,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +12: [2023-05-11 00:16:35,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +12: [2023-05-11 00:16:35,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +21: [2023-05-11 00:16:35,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 9: [2023-05-11 00:16:35,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +30: [2023-05-11 00:16:35,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +11: [2023-05-11 00:16:35,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +11: [2023-05-11 00:16:35,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +11: [2023-05-11 00:16:35,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt. +19: [2023-05-11 00:16:35,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +11: [2023-05-11 00:16:35,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt... +14: [2023-05-11 00:16:35,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +14: [2023-05-11 00:16:35,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +19: [2023-05-11 00:16:35,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 1: [2023-05-11 00:16:35,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +14: [2023-05-11 00:16:35,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +12: [2023-05-11 00:16:35,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +19: [2023-05-11 00:16:35,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +19: [2023-05-11 00:16:35,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +19: [2023-05-11 00:16:35,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +19: [2023-05-11 00:16:35,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 8: [2023-05-11 00:16:35,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +14: [2023-05-11 00:16:35,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +19: [2023-05-11 00:16:35,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +19: [2023-05-11 00:16:35,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +14: [2023-05-11 00:16:35,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +29: [2023-05-11 00:16:35,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +29: [2023-05-11 00:16:35,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +29: [2023-05-11 00:16:35,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +29: [2023-05-11 00:16:35,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 1: [2023-05-11 00:16:35,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +29: [2023-05-11 00:16:35,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +29: [2023-05-11 00:16:35,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +29: [2023-05-11 00:16:35,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +29: [2023-05-11 00:16:35,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +29: [2023-05-11 00:16:35,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +29: [2023-05-11 00:16:35,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +29: [2023-05-11 00:16:35,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +29: [2023-05-11 00:16:35,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +29: [2023-05-11 00:16:35,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +29: [2023-05-11 00:16:35,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +29: [2023-05-11 00:16:35,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +29: [2023-05-11 00:16:35,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +14: [2023-05-11 00:16:35,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +24: [2023-05-11 00:16:35,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +24: [2023-05-11 00:16:35,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +24: [2023-05-11 00:16:35,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +24: [2023-05-11 00:16:35,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 3: [2023-05-11 00:16:35,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +24: [2023-05-11 00:16:35,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +24: [2023-05-11 00:16:35,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +24: [2023-05-11 00:16:35,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +24: [2023-05-11 00:16:35,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +29: [2023-05-11 00:16:35,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +29: [2023-05-11 00:16:35,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +29: [2023-05-11 00:16:35,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +14: [2023-05-11 00:16:35,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +24: [2023-05-11 00:16:35,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +24: [2023-05-11 00:16:35,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +24: [2023-05-11 00:16:35,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +24: [2023-05-11 00:16:35,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +24: [2023-05-11 00:16:35,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +24: [2023-05-11 00:16:35,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +13: [2023-05-11 00:16:35,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +13: [2023-05-11 00:16:35,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +13: [2023-05-11 00:16:35,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +13: [2023-05-11 00:16:35,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +24: [2023-05-11 00:16:35,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +24: [2023-05-11 00:16:35,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +29: [2023-05-11 00:16:35,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +29: [2023-05-11 00:16:35,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +29: [2023-05-11 00:16:35,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +13: [2023-05-11 00:16:35,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +27: [2023-05-11 00:16:35,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +27: [2023-05-11 00:16:35,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +27: [2023-05-11 00:16:35,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +27: [2023-05-11 00:16:35,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +24: [2023-05-11 00:16:35,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +27: [2023-05-11 00:16:35,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +27: [2023-05-11 00:16:35,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +27: [2023-05-11 00:16:35,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +24: [2023-05-11 00:16:35,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +27: [2023-05-11 00:16:35,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +24: [2023-05-11 00:16:35,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 7: [2023-05-11 00:16:35,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +13: [2023-05-11 00:16:35,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +13: [2023-05-11 00:16:35,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +29: [2023-05-11 00:16:35,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +13: [2023-05-11 00:16:35,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +24: [2023-05-11 00:16:35,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +24: [2023-05-11 00:16:35,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 3: [2023-05-11 00:16:35,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +24: [2023-05-11 00:16:35,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +24: [2023-05-11 00:16:35,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +29: [2023-05-11 00:16:35,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +27: [2023-05-11 00:16:35,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +27: [2023-05-11 00:16:35,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +27: [2023-05-11 00:16:35,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +22: [2023-05-11 00:16:35,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +22: [2023-05-11 00:16:35,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +22: [2023-05-11 00:16:35,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +22: [2023-05-11 00:16:35,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +24: [2023-05-11 00:16:35,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +24: [2023-05-11 00:16:35,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +24: [2023-05-11 00:16:35,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +22: [2023-05-11 00:16:35,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +22: [2023-05-11 00:16:35,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +24: [2023-05-11 00:16:35,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +22: [2023-05-11 00:16:35,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +22: [2023-05-11 00:16:35,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +28: [2023-05-11 00:16:35,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +28: [2023-05-11 00:16:35,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +28: [2023-05-11 00:16:35,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +28: [2023-05-11 00:16:35,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +28: [2023-05-11 00:16:35,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +28: [2023-05-11 00:16:35,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +28: [2023-05-11 00:16:35,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +28: [2023-05-11 00:16:35,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +27: [2023-05-11 00:16:35,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +24: [2023-05-11 00:16:35,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +24: [2023-05-11 00:16:35,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +27: [2023-05-11 00:16:35,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +27: [2023-05-11 00:16:35,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +24: [2023-05-11 00:16:35,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +15: [2023-05-11 00:16:35,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +15: [2023-05-11 00:16:35,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +15: [2023-05-11 00:16:35,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +15: [2023-05-11 00:16:35,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +15: [2023-05-11 00:16:35,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +15: [2023-05-11 00:16:35,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +15: [2023-05-11 00:16:35,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +15: [2023-05-11 00:16:35,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 4: [2023-05-11 00:16:35,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 4: [2023-05-11 00:16:35,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +24: [2023-05-11 00:16:35,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +24: [2023-05-11 00:16:35,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +22: [2023-05-11 00:16:35,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +22: [2023-05-11 00:16:35,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +27: [2023-05-11 00:16:35,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +30: [2023-05-11 00:16:35,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +30: [2023-05-11 00:16:35,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +30: [2023-05-11 00:16:35,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +30: [2023-05-11 00:16:35,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +28: [2023-05-11 00:16:35,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +20: [2023-05-11 00:16:35,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +20: [2023-05-11 00:16:35,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 7: [2023-05-11 00:16:35,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +20: [2023-05-11 00:16:35,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +20: [2023-05-11 00:16:35,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +22: [2023-05-11 00:16:35,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +27: [2023-05-11 00:16:35,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +22: [2023-05-11 00:16:35,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +15: [2023-05-11 00:16:35,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +15: [2023-05-11 00:16:35,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +28: [2023-05-11 00:16:35,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +28: [2023-05-11 00:16:35,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +22: [2023-05-11 00:16:35,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +28: [2023-05-11 00:16:35,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +15: [2023-05-11 00:16:35,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +28: [2023-05-11 00:16:35,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +15: [2023-05-11 00:16:35,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +30: [2023-05-11 00:16:35,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +20: [2023-05-11 00:16:35,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +20: [2023-05-11 00:16:35,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +30: [2023-05-11 00:16:35,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +30: [2023-05-11 00:16:35,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +30: [2023-05-11 00:16:35,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +20: [2023-05-11 00:16:35,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +22: [2023-05-11 00:16:35,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +20: [2023-05-11 00:16:35,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +15: [2023-05-11 00:16:35,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +28: [2023-05-11 00:16:35,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +28: [2023-05-11 00:16:35,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +22: [2023-05-11 00:16:35,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +15: [2023-05-11 00:16:35,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +22: [2023-05-11 00:16:35,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +15: [2023-05-11 00:16:35,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +28: [2023-05-11 00:16:35,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +15: [2023-05-11 00:16:35,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +10: [2023-05-11 00:16:35,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +10: [2023-05-11 00:16:35,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +10: [2023-05-11 00:16:35,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +10: [2023-05-11 00:16:35,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +10: [2023-05-11 00:16:35,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +10: [2023-05-11 00:16:35,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +10: [2023-05-11 00:16:35,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +10: [2023-05-11 00:16:35,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +10: [2023-05-11 00:16:35,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +10: [2023-05-11 00:16:35,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +26: [2023-05-11 00:16:35,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +26: [2023-05-11 00:16:35,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +26: [2023-05-11 00:16:35,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +26: [2023-05-11 00:16:35,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +26: [2023-05-11 00:16:35,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +26: [2023-05-11 00:16:35,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +26: [2023-05-11 00:16:35,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +26: [2023-05-11 00:16:35,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +10: [2023-05-11 00:16:35,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +10: [2023-05-11 00:16:35,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +10: [2023-05-11 00:16:35,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +10: [2023-05-11 00:16:35,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +16: [2023-05-11 00:16:35,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +16: [2023-05-11 00:16:35,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +10: [2023-05-11 00:16:35,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +10: [2023-05-11 00:16:35,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +16: [2023-05-11 00:16:35,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +16: [2023-05-11 00:16:35,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +16: [2023-05-11 00:16:35,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +16: [2023-05-11 00:16:35,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +18: [2023-05-11 00:16:35,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +18: [2023-05-11 00:16:35,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +18: [2023-05-11 00:16:35,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +18: [2023-05-11 00:16:35,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +28: [2023-05-11 00:16:35,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +28: [2023-05-11 00:16:35,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +28: [2023-05-11 00:16:35,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 0: [2023-05-11 00:16:35,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +28: [2023-05-11 00:16:35,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +16: [2023-05-11 00:16:35,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +25: [2023-05-11 00:16:35,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +25: [2023-05-11 00:16:35,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +25: [2023-05-11 00:16:35,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +25: [2023-05-11 00:16:35,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +30: [2023-05-11 00:16:35,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +30: [2023-05-11 00:16:35,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +16: [2023-05-11 00:16:35,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +30: [2023-05-11 00:16:35,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +25: [2023-05-11 00:16:35,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +18: [2023-05-11 00:16:35,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +18: [2023-05-11 00:16:35,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +18: [2023-05-11 00:16:35,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +25: [2023-05-11 00:16:35,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +25: [2023-05-11 00:16:35,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 0: [2023-05-11 00:16:35,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +30: [2023-05-11 00:16:35,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +30: [2023-05-11 00:16:35,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +30: [2023-05-11 00:16:35,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 0: [2023-05-11 00:16:35,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +13: [2023-05-11 00:16:35,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +13: [2023-05-11 00:16:35,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +13: [2023-05-11 00:16:35,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +13: [2023-05-11 00:16:35,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +13: [2023-05-11 00:16:35,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +13: [2023-05-11 00:16:35,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +13: [2023-05-11 00:16:35,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +13: [2023-05-11 00:16:35,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +13: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +13: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +13: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +13: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +13: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 6: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +23: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +23: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +23: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +23: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +18: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +23: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +23: [2023-05-11 00:16:35,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +25: [2023-05-11 00:16:35,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +23: [2023-05-11 00:16:35,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +23: [2023-05-11 00:16:35,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +31: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +31: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +31: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +31: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +31: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +31: [2023-05-11 00:16:35,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +13: [2023-05-11 00:16:35,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +31: [2023-05-11 00:16:35,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +10: [2023-05-11 00:16:35,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +13: [2023-05-11 00:16:35,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +13: [2023-05-11 00:16:35,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 2: [2023-05-11 00:16:35,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +26: [2023-05-11 00:16:35,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +26: [2023-05-11 00:16:35,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 0: [2023-05-11 00:16:35,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +30: [2023-05-11 00:16:35,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +30: [2023-05-11 00:16:35,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +10: [2023-05-11 00:16:35,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +31: [2023-05-11 00:16:35,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +11: [2023-05-11 00:16:35,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +11: [2023-05-11 00:16:35,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +11: [2023-05-11 00:16:35,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +11: [2023-05-11 00:16:35,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +26: [2023-05-11 00:16:35,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +20: [2023-05-11 00:16:35,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +20: [2023-05-11 00:16:35,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +20: [2023-05-11 00:16:35,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +20: [2023-05-11 00:16:35,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +20: [2023-05-11 00:16:35,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +20: [2023-05-11 00:16:35,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +20: [2023-05-11 00:16:35,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +20: [2023-05-11 00:16:35,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +20: [2023-05-11 00:16:35,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +20: [2023-05-11 00:16:35,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +20: [2023-05-11 00:16:35,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +20: [2023-05-11 00:16:35,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +11: [2023-05-11 00:16:35,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +11: [2023-05-11 00:16:35,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +20: [2023-05-11 00:16:35,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +20: [2023-05-11 00:16:35,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +20: [2023-05-11 00:16:35,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +11: [2023-05-11 00:16:35,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +20: [2023-05-11 00:16:35,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 6: [2023-05-11 00:16:35,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 5: [2023-05-11 00:16:35,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +28: [2023-05-11 00:16:35,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +11: [2023-05-11 00:16:35,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +26: [2023-05-11 00:16:35,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 6: [2023-05-11 00:16:35,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +23: [2023-05-11 00:16:35,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +25: [2023-05-11 00:16:35,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +23: [2023-05-11 00:16:35,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 4: [2023-05-11 00:16:35,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +28: [2023-05-11 00:16:35,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +28: [2023-05-11 00:16:35,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +26: [2023-05-11 00:16:35,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +25: [2023-05-11 00:16:35,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +16: [2023-05-11 00:16:35,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +10: [2023-05-11 00:16:35,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +28: [2023-05-11 00:16:35,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +25: [2023-05-11 00:16:35,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +13: [2023-05-11 00:16:35,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +16: [2023-05-11 00:16:35,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +31: [2023-05-11 00:16:35,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +18: [2023-05-11 00:16:35,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +10: [2023-05-11 00:16:35,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 0: [2023-05-11 00:16:35,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +18: [2023-05-11 00:16:35,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +18: [2023-05-11 00:16:35,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +31: [2023-05-11 00:16:35,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +31: [2023-05-11 00:16:35,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +23: [2023-05-11 00:16:35,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +13: [2023-05-11 00:16:35,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +13: [2023-05-11 00:16:35,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +26: [2023-05-11 00:16:35,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +26: [2023-05-11 00:16:35,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +17: [2023-05-11 00:16:35,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +17: [2023-05-11 00:16:35,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +17: [2023-05-11 00:16:35,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +17: [2023-05-11 00:16:35,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +17: [2023-05-11 00:16:35,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +17: [2023-05-11 00:16:35,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +11: [2023-05-11 00:16:35,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +23: [2023-05-11 00:16:35,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +23: [2023-05-11 00:16:35,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +23: [2023-05-11 00:16:35,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +17: [2023-05-11 00:16:35,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +17: [2023-05-11 00:16:35,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +10: [2023-05-11 00:16:35,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +25: [2023-05-11 00:16:35,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +11: [2023-05-11 00:16:35,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +25: [2023-05-11 00:16:35,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +26: [2023-05-11 00:16:35,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +26: [2023-05-11 00:16:35,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +11: [2023-05-11 00:16:35,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +16: [2023-05-11 00:16:35,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +26: [2023-05-11 00:16:35,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +23: [2023-05-11 00:16:35,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +16: [2023-05-11 00:16:35,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +23: [2023-05-11 00:16:35,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +16: [2023-05-11 00:16:35,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +23: [2023-05-11 00:16:35,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +19: [2023-05-11 00:16:35,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +19: [2023-05-11 00:16:35,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +20: [2023-05-11 00:16:35,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +20: [2023-05-11 00:16:35,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +18: [2023-05-11 00:16:35,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +25: [2023-05-11 00:16:35,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +20: [2023-05-11 00:16:35,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +19: [2023-05-11 00:16:35,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +19: [2023-05-11 00:16:35,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +23: [2023-05-11 00:16:35,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +19: [2023-05-11 00:16:35,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +19: [2023-05-11 00:16:35,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +19: [2023-05-11 00:16:35,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +19: [2023-05-11 00:16:35,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +28: [2023-05-11 00:16:35,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +31: [2023-05-11 00:16:35,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +28: [2023-05-11 00:16:35,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +18: [2023-05-11 00:16:35,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +26: [2023-05-11 00:16:35,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +13: [2023-05-11 00:16:35,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +31: [2023-05-11 00:16:35,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +13: [2023-05-11 00:16:35,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +26: [2023-05-11 00:16:35,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +11: [2023-05-11 00:16:35,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +13: [2023-05-11 00:16:35,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +26: [2023-05-11 00:16:35,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +23: [2023-05-11 00:16:35,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +18: [2023-05-11 00:16:35,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +10: [2023-05-11 00:16:35,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +26: [2023-05-11 00:16:35,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +23: [2023-05-11 00:16:35,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +31: [2023-05-11 00:16:35,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +21: [2023-05-11 00:16:35,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +21: [2023-05-11 00:16:35,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +21: [2023-05-11 00:16:35,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +18: [2023-05-11 00:16:35,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +21: [2023-05-11 00:16:35,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +21: [2023-05-11 00:16:35,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +13: [2023-05-11 00:16:35,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +21: [2023-05-11 00:16:35,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +10: [2023-05-11 00:16:35,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +25: [2023-05-11 00:16:35,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +21: [2023-05-11 00:16:35,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +21: [2023-05-11 00:16:35,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +11: [2023-05-11 00:16:35,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +23: [2023-05-11 00:16:35,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +20: [2023-05-11 00:16:35,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +23: [2023-05-11 00:16:35,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +11: [2023-05-11 00:16:35,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +16: [2023-05-11 00:16:35,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +23: [2023-05-11 00:16:35,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +23: [2023-05-11 00:16:35,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +16: [2023-05-11 00:16:35,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +20: [2023-05-11 00:16:35,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +20: [2023-05-11 00:16:35,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 0: [2023-05-11 00:16:35,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +17: [2023-05-11 00:16:35,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +17: [2023-05-11 00:16:35,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +17: [2023-05-11 00:16:35,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +26: [2023-05-11 00:16:35,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +31: [2023-05-11 00:16:35,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +26: [2023-05-11 00:16:35,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +28: [2023-05-11 00:16:35,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +19: [2023-05-11 00:16:35,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +19: [2023-05-11 00:16:35,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +10: [2023-05-11 00:16:35,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +20: [2023-05-11 00:16:35,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +28: [2023-05-11 00:16:35,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +26: [2023-05-11 00:16:35,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +11: [2023-05-11 00:16:35,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +18: [2023-05-11 00:16:35,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +13: [2023-05-11 00:16:35,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 2: [2023-05-11 00:16:35,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +26: [2023-05-11 00:16:35,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +26: [2023-05-11 00:16:35,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +19: [2023-05-11 00:16:35,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +28: [2023-05-11 00:16:35,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +28: [2023-05-11 00:16:35,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +25: [2023-05-11 00:16:35,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +23: [2023-05-11 00:16:35,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +28: [2023-05-11 00:16:35,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +28: [2023-05-11 00:16:35,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +20: [2023-05-11 00:16:35,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +23: [2023-05-11 00:16:35,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +23: [2023-05-11 00:16:35,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +23: [2023-05-11 00:16:35,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 6: [2023-05-11 00:16:35,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +23: [2023-05-11 00:16:35,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +23: [2023-05-11 00:16:35,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +16: [2023-05-11 00:16:35,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +26: [2023-05-11 00:16:35,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 2: [2023-05-11 00:16:35,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +17: [2023-05-11 00:16:35,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +31: [2023-05-11 00:16:35,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +17: [2023-05-11 00:16:35,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +26: [2023-05-11 00:16:35,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +26: [2023-05-11 00:16:35,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 6: [2023-05-11 00:16:35,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +26: [2023-05-11 00:16:35,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +19: [2023-05-11 00:16:35,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +17: [2023-05-11 00:16:35,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +19: [2023-05-11 00:16:35,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +11: [2023-05-11 00:16:35,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +17: [2023-05-11 00:16:35,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +26: [2023-05-11 00:16:35,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 0: [2023-05-11 00:16:35,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +21: [2023-05-11 00:16:35,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +21: [2023-05-11 00:16:35,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +18: [2023-05-11 00:16:35,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +18: [2023-05-11 00:16:35,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +18: [2023-05-11 00:16:35,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 0: [2023-05-11 00:16:35,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +14: [2023-05-11 00:16:35,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +14: [2023-05-11 00:16:35,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +14: [2023-05-11 00:16:35,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +14: [2023-05-11 00:16:35,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +14: [2023-05-11 00:16:35,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +14: [2023-05-11 00:16:35,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +19: [2023-05-11 00:16:35,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +23: [2023-05-11 00:16:35,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +23: [2023-05-11 00:16:35,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +19: [2023-05-11 00:16:35,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +14: [2023-05-11 00:16:35,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +18: [2023-05-11 00:16:35,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 6: [2023-05-11 00:16:35,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 0: [2023-05-11 00:16:35,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +30: [2023-05-11 00:16:35,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +30: [2023-05-11 00:16:35,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +30: [2023-05-11 00:16:35,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +30: [2023-05-11 00:16:35,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +30: [2023-05-11 00:16:35,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +30: [2023-05-11 00:16:35,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +30: [2023-05-11 00:16:35,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +14: [2023-05-11 00:16:35,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +30: [2023-05-11 00:16:35,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +17: [2023-05-11 00:16:35,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +21: [2023-05-11 00:16:35,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +21: [2023-05-11 00:16:35,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +19: [2023-05-11 00:16:35,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +18: [2023-05-11 00:16:35,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +21: [2023-05-11 00:16:35,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +21: [2023-05-11 00:16:35,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 9: [2023-05-11 00:16:35,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 9: [2023-05-11 00:16:35,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 9: [2023-05-11 00:16:35,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +18: [2023-05-11 00:16:35,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +14: [2023-05-11 00:16:35,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +18: [2023-05-11 00:16:35,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +18: [2023-05-11 00:16:35,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +30: [2023-05-11 00:16:35,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +21: [2023-05-11 00:16:35,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +30: [2023-05-11 00:16:35,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +14: [2023-05-11 00:16:35,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +21: [2023-05-11 00:16:35,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +14: [2023-05-11 00:16:35,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +30: [2023-05-11 00:16:35,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 5: [2023-05-11 00:16:35,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 5: [2023-05-11 00:16:35,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 5: [2023-05-11 00:16:35,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 9: [2023-05-11 00:16:35,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +14: [2023-05-11 00:16:35,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +18: [2023-05-11 00:16:35,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +30: [2023-05-11 00:16:35,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +18: [2023-05-11 00:16:35,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 9: [2023-05-11 00:16:35,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +30: [2023-05-11 00:16:35,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +11: [2023-05-11 00:16:35,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +11: [2023-05-11 00:16:35,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +11: [2023-05-11 00:16:35,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +11: [2023-05-11 00:16:35,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +30: [2023-05-11 00:16:35,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +29: [2023-05-11 00:16:35,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 1: [2023-05-11 00:16:35,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +29: [2023-05-11 00:16:35,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 1: [2023-05-11 00:16:35,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +14: [2023-05-11 00:16:35,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +30: [2023-05-11 00:16:35,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +29: [2023-05-11 00:16:35,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +29: [2023-05-11 00:16:35,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +12: [2023-05-11 00:16:35,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +12: [2023-05-11 00:16:35,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +12: [2023-05-11 00:16:35,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +12: [2023-05-11 00:16:35,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +12: [2023-05-11 00:16:35,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +12: [2023-05-11 00:16:35,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +12: [2023-05-11 00:16:35,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +18: [2023-05-11 00:16:35,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +18: [2023-05-11 00:16:35,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 9: [2023-05-11 00:16:35,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +14: [2023-05-11 00:16:35,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +12: [2023-05-11 00:16:35,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +19: [2023-05-11 00:16:35,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +19: [2023-05-11 00:16:35,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +19: [2023-05-11 00:16:35,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +19: [2023-05-11 00:16:35,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +11: [2023-05-11 00:16:35,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +17: [2023-05-11 00:16:35,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +16: [2023-05-11 00:16:35,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +16: [2023-05-11 00:16:35,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +16: [2023-05-11 00:16:35,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +29: [2023-05-11 00:16:35,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +29: [2023-05-11 00:16:35,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +29: [2023-05-11 00:16:35,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +29: [2023-05-11 00:16:35,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +29: [2023-05-11 00:16:35,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +29: [2023-05-11 00:16:35,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +29: [2023-05-11 00:16:35,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +29: [2023-05-11 00:16:35,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +18: [2023-05-11 00:16:35,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +30: [2023-05-11 00:16:35,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +14: [2023-05-11 00:16:35,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +14: [2023-05-11 00:16:35,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +14: [2023-05-11 00:16:35,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +14: [2023-05-11 00:16:35,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +11: [2023-05-11 00:16:35,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +27: [2023-05-11 00:16:35,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +29: [2023-05-11 00:16:35,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +18: [2023-05-11 00:16:35,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 9: [2023-05-11 00:16:35,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +29: [2023-05-11 00:16:35,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 9: [2023-05-11 00:16:35,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 1: [2023-05-11 00:16:35,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +17: [2023-05-11 00:16:35,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +29: [2023-05-11 00:16:35,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +11: [2023-05-11 00:16:35,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +27: [2023-05-11 00:16:35,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +11: [2023-05-11 00:16:35,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +29: [2023-05-11 00:16:35,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +16: [2023-05-11 00:16:35,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +18: [2023-05-11 00:16:35,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 2: [2023-05-11 00:16:35,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 2: [2023-05-11 00:16:35,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +17: [2023-05-11 00:16:35,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +12: [2023-05-11 00:16:35,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +17: [2023-05-11 00:16:35,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 5: [2023-05-11 00:16:35,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +19: [2023-05-11 00:16:35,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +18: [2023-05-11 00:16:35,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +14: [2023-05-11 00:16:35,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +27: [2023-05-11 00:16:35,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +22: [2023-05-11 00:16:35,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +22: [2023-05-11 00:16:35,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +22: [2023-05-11 00:16:35,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 1: [2023-05-11 00:16:35,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 5: [2023-05-11 00:16:35,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +27: [2023-05-11 00:16:35,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +16: [2023-05-11 00:16:35,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +24: [2023-05-11 00:16:35,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +24: [2023-05-11 00:16:35,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +19: [2023-05-11 00:16:35,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +16: [2023-05-11 00:16:35,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +24: [2023-05-11 00:16:35,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +22: [2023-05-11 00:16:35,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +22: [2023-05-11 00:16:35,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +24: [2023-05-11 00:16:35,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +22: [2023-05-11 00:16:35,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +22: [2023-05-11 00:16:35,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +22: [2023-05-11 00:16:35,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +22: [2023-05-11 00:16:35,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +22: [2023-05-11 00:16:35,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +22: [2023-05-11 00:16:35,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +19: [2023-05-11 00:16:35,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +19: [2023-05-11 00:16:35,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +27: [2023-05-11 00:16:35,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +22: [2023-05-11 00:16:35,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 1: [2023-05-11 00:16:35,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 7: [2023-05-11 00:16:35,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 7: [2023-05-11 00:16:35,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 7: [2023-05-11 00:16:35,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +14: [2023-05-11 00:16:35,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +17: [2023-05-11 00:16:35,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +17: [2023-05-11 00:16:35,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +12: [2023-05-11 00:16:35,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +12: [2023-05-11 00:16:35,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +14: [2023-05-11 00:16:35,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +14: [2023-05-11 00:16:35,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +16: [2023-05-11 00:16:35,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +14: [2023-05-11 00:16:35,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +16: [2023-05-11 00:16:35,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +27: [2023-05-11 00:16:35,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +25: [2023-05-11 00:16:35,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +25: [2023-05-11 00:16:35,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +17: [2023-05-11 00:16:35,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +11: [2023-05-11 00:16:35,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +17: [2023-05-11 00:16:35,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +25: [2023-05-11 00:16:35,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +11: [2023-05-11 00:16:35,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +25: [2023-05-11 00:16:35,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 3: [2023-05-11 00:16:35,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 3: [2023-05-11 00:16:35,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 2: [2023-05-11 00:16:35,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +31: [2023-05-11 00:16:35,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +31: [2023-05-11 00:16:35,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 2: [2023-05-11 00:16:35,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +14: [2023-05-11 00:16:35,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +27: [2023-05-11 00:16:35,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +22: [2023-05-11 00:16:35,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +24: [2023-05-11 00:16:35,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +24: [2023-05-11 00:16:35,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +12: [2023-05-11 00:16:35,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +24: [2023-05-11 00:16:35,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +29: [2023-05-11 00:16:35,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +27: [2023-05-11 00:16:35,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +29: [2023-05-11 00:16:35,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 1: [2023-05-11 00:16:35,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +11: [2023-05-11 00:16:35,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +31: [2023-05-11 00:16:35,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +11: [2023-05-11 00:16:35,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 2: [2023-05-11 00:16:35,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +31: [2023-05-11 00:16:35,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 1: [2023-05-11 00:16:35,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +22: [2023-05-11 00:16:35,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +12: [2023-05-11 00:16:35,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +24: [2023-05-11 00:16:35,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. + 7: [2023-05-11 00:16:35,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +29: [2023-05-11 00:16:35,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +29: [2023-05-11 00:16:35,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 3: [2023-05-11 00:16:35,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt. +12: [2023-05-11 00:16:35,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +11: [2023-05-11 00:16:35,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +22: [2023-05-11 00:16:35,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +11: [2023-05-11 00:16:35,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +22: [2023-05-11 00:16:35,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +25: [2023-05-11 00:16:35,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +29: [2023-05-11 00:16:35,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +29: [2023-05-11 00:16:35,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +29: [2023-05-11 00:16:35,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +16: [2023-05-11 00:16:35,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +14: [2023-05-11 00:16:35,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +16: [2023-05-11 00:16:35,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 3: [2023-05-11 00:16:35,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +25: [2023-05-11 00:16:35,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +14: [2023-05-11 00:16:35,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 3: [2023-05-11 00:16:35,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +22: [2023-05-11 00:16:35,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +29: [2023-05-11 00:16:35,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +25: [2023-05-11 00:16:35,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +19: [2023-05-11 00:16:35,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +25: [2023-05-11 00:16:35,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +19: [2023-05-11 00:16:35,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +29: [2023-05-11 00:16:35,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +14: [2023-05-11 00:16:35,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +14: [2023-05-11 00:16:35,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +27: [2023-05-11 00:16:35,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +31: [2023-05-11 00:16:35,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +17: [2023-05-11 00:16:35,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +16: [2023-05-11 00:16:35,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +16: [2023-05-11 00:16:35,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +17: [2023-05-11 00:16:35,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +31: [2023-05-11 00:16:35,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +31: [2023-05-11 00:16:35,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +31: [2023-05-11 00:16:35,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +22: [2023-05-11 00:16:35,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +11: [2023-05-11 00:16:35,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +12: [2023-05-11 00:16:35,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +17: [2023-05-11 00:16:35,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +17: [2023-05-11 00:16:35,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +11: [2023-05-11 00:16:35,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +22: [2023-05-11 00:16:35,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +29: [2023-05-11 00:16:35,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +29: [2023-05-11 00:16:35,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 3: [2023-05-11 00:16:35,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +22: [2023-05-11 00:16:35,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +24: [2023-05-11 00:16:35,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +17: [2023-05-11 00:16:35,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +17: [2023-05-11 00:16:35,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +16: [2023-05-11 00:16:35,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +16: [2023-05-11 00:16:35,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 7: [2023-05-11 00:16:35,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +27: [2023-05-11 00:16:35,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +29: [2023-05-11 00:16:35,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +24: [2023-05-11 00:16:35,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +24: [2023-05-11 00:16:35,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +24: [2023-05-11 00:16:35,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +24: [2023-05-11 00:16:35,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +24: [2023-05-11 00:16:35,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +29: [2023-05-11 00:16:35,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... +25: [2023-05-11 00:16:35,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +17: [2023-05-11 00:16:35,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +29: [2023-05-11 00:16:35,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +27: [2023-05-11 00:16:35,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +25: [2023-05-11 00:16:35,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +27: [2023-05-11 00:16:35,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +22: [2023-05-11 00:16:35,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +19: [2023-05-11 00:16:35,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +22: [2023-05-11 00:16:35,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +14: [2023-05-11 00:16:35,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +14: [2023-05-11 00:16:35,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +17: [2023-05-11 00:16:35,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +12: [2023-05-11 00:16:35,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +19: [2023-05-11 00:16:35,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 3: [2023-05-11 00:16:35,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +19: [2023-05-11 00:16:35,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +22: [2023-05-11 00:16:35,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +27: [2023-05-11 00:16:35,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +27: [2023-05-11 00:16:35,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +24: [2023-05-11 00:16:35,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +19: [2023-05-11 00:16:35,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +29: [2023-05-11 00:16:35,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +31: [2023-05-11 00:16:35,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +31: [2023-05-11 00:16:35,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 7: [2023-05-11 00:16:35,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +22: [2023-05-11 00:16:35,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +22: [2023-05-11 00:16:35,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +24: [2023-05-11 00:16:35,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 3: [2023-05-11 00:16:35,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +25: [2023-05-11 00:16:35,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +25: [2023-05-11 00:16:35,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +25: [2023-05-11 00:16:35,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +16: [2023-05-11 00:16:35,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +25: [2023-05-11 00:16:35,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 2: [2023-05-11 00:16:35,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 2: [2023-05-11 00:16:35,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 2: [2023-05-11 00:16:35,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 2: [2023-05-11 00:16:35,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +27: [2023-05-11 00:16:35,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +27: [2023-05-11 00:16:35,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +25: [2023-05-11 00:16:35,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +22: [2023-05-11 00:16:35,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +25: [2023-05-11 00:16:35,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +22: [2023-05-11 00:16:35,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +16: [2023-05-11 00:16:35,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +22: [2023-05-11 00:16:35,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +19: [2023-05-11 00:16:35,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +14: [2023-05-11 00:16:35,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +31: [2023-05-11 00:16:35,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +31: [2023-05-11 00:16:35,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +19: [2023-05-11 00:16:35,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +14: [2023-05-11 00:16:35,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +22: [2023-05-11 00:16:35,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +10: [2023-05-11 00:16:35,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +10: [2023-05-11 00:16:35,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +28: [2023-05-11 00:16:35,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +28: [2023-05-11 00:16:35,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +29: [2023-05-11 00:16:35,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +22: [2023-05-11 00:16:35,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 7: [2023-05-11 00:16:35,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +10: [2023-05-11 00:16:35,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +22: [2023-05-11 00:16:35,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +27: [2023-05-11 00:16:35,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +27: [2023-05-11 00:16:35,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +10: [2023-05-11 00:16:35,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +27: [2023-05-11 00:16:35,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +27: [2023-05-11 00:16:35,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +27: [2023-05-11 00:16:35,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +27: [2023-05-11 00:16:35,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +28: [2023-05-11 00:16:35,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +10: [2023-05-11 00:16:35,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +10: [2023-05-11 00:16:35,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +10: [2023-05-11 00:16:35,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +22: [2023-05-11 00:16:35,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +28: [2023-05-11 00:16:35,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +28: [2023-05-11 00:16:35,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +28: [2023-05-11 00:16:35,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +28: [2023-05-11 00:16:35,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +31: [2023-05-11 00:16:35,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +31: [2023-05-11 00:16:35,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +31: [2023-05-11 00:16:35,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +31: [2023-05-11 00:16:35,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 2: [2023-05-11 00:16:35,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 3: [2023-05-11 00:16:35,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +10: [2023-05-11 00:16:35,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +27: [2023-05-11 00:16:35,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +28: [2023-05-11 00:16:35,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. +27: [2023-05-11 00:16:35,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... + 7: [2023-05-11 00:16:35,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +24: [2023-05-11 00:16:35,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +24: [2023-05-11 00:16:35,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +24: [2023-05-11 00:16:35,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +24: [2023-05-11 00:16:35,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +10: [2023-05-11 00:16:35,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +10: [2023-05-11 00:16:35,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +27: [2023-05-11 00:16:35,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +24: [2023-05-11 00:16:35,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +24: [2023-05-11 00:16:35,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +28: [2023-05-11 00:16:35,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +27: [2023-05-11 00:16:35,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +27: [2023-05-11 00:16:35,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +24: [2023-05-11 00:16:35,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +24: [2023-05-11 00:16:35,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +28: [2023-05-11 00:16:35,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +15: [2023-05-11 00:16:35,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +15: [2023-05-11 00:16:35,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +15: [2023-05-11 00:16:35,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +15: [2023-05-11 00:16:35,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +28: [2023-05-11 00:16:35,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +27: [2023-05-11 00:16:35,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +15: [2023-05-11 00:16:35,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +15: [2023-05-11 00:16:35,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +10: [2023-05-11 00:16:35,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +10: [2023-05-11 00:16:35,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +15: [2023-05-11 00:16:35,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +15: [2023-05-11 00:16:35,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt... +27: [2023-05-11 00:16:35,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +27: [2023-05-11 00:16:35,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +24: [2023-05-11 00:16:35,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +24: [2023-05-11 00:16:35,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +10: [2023-05-11 00:16:35,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +28: [2023-05-11 00:16:35,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +28: [2023-05-11 00:16:35,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +10: [2023-05-11 00:16:35,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +28: [2023-05-11 00:16:35,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +27: [2023-05-11 00:16:35,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +24: [2023-05-11 00:16:35,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +11: [2023-05-11 00:16:35,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +11: [2023-05-11 00:16:35,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +16: [2023-05-11 00:16:35,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +16: [2023-05-11 00:16:35,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +24: [2023-05-11 00:16:35,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +16: [2023-05-11 00:16:35,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +16: [2023-05-11 00:16:35,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +16: [2023-05-11 00:16:35,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +16: [2023-05-11 00:16:35,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +16: [2023-05-11 00:16:35,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +16: [2023-05-11 00:16:35,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +11: [2023-05-11 00:16:35,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +10: [2023-05-11 00:16:35,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +11: [2023-05-11 00:16:35,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +11: [2023-05-11 00:16:35,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +11: [2023-05-11 00:16:35,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +11: [2023-05-11 00:16:35,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +19: [2023-05-11 00:16:35,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +19: [2023-05-11 00:16:35,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +19: [2023-05-11 00:16:35,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +19: [2023-05-11 00:16:35,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +11: [2023-05-11 00:16:35,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +10: [2023-05-11 00:16:35,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +19: [2023-05-11 00:16:35,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +19: [2023-05-11 00:16:35,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +19: [2023-05-11 00:16:35,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +21: [2023-05-11 00:16:35,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +21: [2023-05-11 00:16:35,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +24: [2023-05-11 00:16:35,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +28: [2023-05-11 00:16:35,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +21: [2023-05-11 00:16:35,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +21: [2023-05-11 00:16:35,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +27: [2023-05-11 00:16:35,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +28: [2023-05-11 00:16:35,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +19: [2023-05-11 00:16:35,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +15: [2023-05-11 00:16:35,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +21: [2023-05-11 00:16:35,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +21: [2023-05-11 00:16:35,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +15: [2023-05-11 00:16:35,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +21: [2023-05-11 00:16:35,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +21: [2023-05-11 00:16:35,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +30: [2023-05-11 00:16:35,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +30: [2023-05-11 00:16:35,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +30: [2023-05-11 00:16:35,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +30: [2023-05-11 00:16:35,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +30: [2023-05-11 00:16:35,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +30: [2023-05-11 00:16:35,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +15: [2023-05-11 00:16:35,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +30: [2023-05-11 00:16:35,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +30: [2023-05-11 00:16:35,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +24: [2023-05-11 00:16:35,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +16: [2023-05-11 00:16:35,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +15: [2023-05-11 00:16:35,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +11: [2023-05-11 00:16:35,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +24: [2023-05-11 00:16:35,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +16: [2023-05-11 00:16:35,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +19: [2023-05-11 00:16:35,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +19: [2023-05-11 00:16:35,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +19: [2023-05-11 00:16:35,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +15: [2023-05-11 00:16:35,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +15: [2023-05-11 00:16:35,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +16: [2023-05-11 00:16:35,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +11: [2023-05-11 00:16:35,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +24: [2023-05-11 00:16:35,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +18: [2023-05-11 00:16:35,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +18: [2023-05-11 00:16:35,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +18: [2023-05-11 00:16:35,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +18: [2023-05-11 00:16:35,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +18: [2023-05-11 00:16:35,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +18: [2023-05-11 00:16:35,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +18: [2023-05-11 00:16:35,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +15: [2023-05-11 00:16:35,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +20: [2023-05-11 00:16:35,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +20: [2023-05-11 00:16:35,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +14: [2023-05-11 00:16:35,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +14: [2023-05-11 00:16:35,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +20: [2023-05-11 00:16:35,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +20: [2023-05-11 00:16:35,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +15: [2023-05-11 00:16:35,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +21: [2023-05-11 00:16:35,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +18: [2023-05-11 00:16:35,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +23: [2023-05-11 00:16:35,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +23: [2023-05-11 00:16:35,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +23: [2023-05-11 00:16:35,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +23: [2023-05-11 00:16:35,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +16: [2023-05-11 00:16:35,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +16: [2023-05-11 00:16:35,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +30: [2023-05-11 00:16:35,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +30: [2023-05-11 00:16:35,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +11: [2023-05-11 00:16:35,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +19: [2023-05-11 00:16:35,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +23: [2023-05-11 00:16:35,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +23: [2023-05-11 00:16:35,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +23: [2023-05-11 00:16:35,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +14: [2023-05-11 00:16:35,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +14: [2023-05-11 00:16:35,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +14: [2023-05-11 00:16:35,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +14: [2023-05-11 00:16:35,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +14: [2023-05-11 00:16:35,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +20: [2023-05-11 00:16:35,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +20: [2023-05-11 00:16:35,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +20: [2023-05-11 00:16:35,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +14: [2023-05-11 00:16:35,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +11: [2023-05-11 00:16:35,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +19: [2023-05-11 00:16:35,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +19: [2023-05-11 00:16:35,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +21: [2023-05-11 00:16:35,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +21: [2023-05-11 00:16:35,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +11: [2023-05-11 00:16:35,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +20: [2023-05-11 00:16:35,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +30: [2023-05-11 00:16:35,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +30: [2023-05-11 00:16:35,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +16: [2023-05-11 00:16:35,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +23: [2023-05-11 00:16:35,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +16: [2023-05-11 00:16:35,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +21: [2023-05-11 00:16:35,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +14: [2023-05-11 00:16:35,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +29: [2023-05-11 00:16:35,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +29: [2023-05-11 00:16:35,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +29: [2023-05-11 00:16:35,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +29: [2023-05-11 00:16:35,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +29: [2023-05-11 00:16:35,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +29: [2023-05-11 00:16:35,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +18: [2023-05-11 00:16:35,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +21: [2023-05-11 00:16:35,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +11: [2023-05-11 00:16:35,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +30: [2023-05-11 00:16:35,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +29: [2023-05-11 00:16:35,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +18: [2023-05-11 00:16:35,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +18: [2023-05-11 00:16:35,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +16: [2023-05-11 00:16:35,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +11: [2023-05-11 00:16:35,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +29: [2023-05-11 00:16:35,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +30: [2023-05-11 00:16:35,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +20: [2023-05-11 00:16:35,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +30: [2023-05-11 00:16:35,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +23: [2023-05-11 00:16:35,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +23: [2023-05-11 00:16:35,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +23: [2023-05-11 00:16:35,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +30: [2023-05-11 00:16:35,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +19: [2023-05-11 00:16:35,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +21: [2023-05-11 00:16:35,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +21: [2023-05-11 00:16:35,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +20: [2023-05-11 00:16:35,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +20: [2023-05-11 00:16:35,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +11: [2023-05-11 00:16:35,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +25: [2023-05-11 00:16:35,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +25: [2023-05-11 00:16:35,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +25: [2023-05-11 00:16:35,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +25: [2023-05-11 00:16:35,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +21: [2023-05-11 00:16:35,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +14: [2023-05-11 00:16:35,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +14: [2023-05-11 00:16:35,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +14: [2023-05-11 00:16:35,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +14: [2023-05-11 00:16:35,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +18: [2023-05-11 00:16:35,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +25: [2023-05-11 00:16:35,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +25: [2023-05-11 00:16:35,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +25: [2023-05-11 00:16:35,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +25: [2023-05-11 00:16:35,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +20: [2023-05-11 00:16:35,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +18: [2023-05-11 00:16:35,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +18: [2023-05-11 00:16:35,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +23: [2023-05-11 00:16:35,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +23: [2023-05-11 00:16:35,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +23: [2023-05-11 00:16:35,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +13: [2023-05-11 00:16:35,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +13: [2023-05-11 00:16:35,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +13: [2023-05-11 00:16:35,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +13: [2023-05-11 00:16:35,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +31: [2023-05-11 00:16:35,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +31: [2023-05-11 00:16:35,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +31: [2023-05-11 00:16:35,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +13: [2023-05-11 00:16:35,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +31: [2023-05-11 00:16:35,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +31: [2023-05-11 00:16:35,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +29: [2023-05-11 00:16:35,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +19: [2023-05-11 00:16:35,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +31: [2023-05-11 00:16:35,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +20: [2023-05-11 00:16:35,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +31: [2023-05-11 00:16:35,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +13: [2023-05-11 00:16:35,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +20: [2023-05-11 00:16:35,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +18: [2023-05-11 00:16:35,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +31: [2023-05-11 00:16:35,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +13: [2023-05-11 00:16:35,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +13: [2023-05-11 00:16:35,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +14: [2023-05-11 00:16:35,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +14: [2023-05-11 00:16:35,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +14: [2023-05-11 00:16:35,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +29: [2023-05-11 00:16:35,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +29: [2023-05-11 00:16:35,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +17: [2023-05-11 00:16:35,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +17: [2023-05-11 00:16:35,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +17: [2023-05-11 00:16:35,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +17: [2023-05-11 00:16:35,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +17: [2023-05-11 00:16:35,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +17: [2023-05-11 00:16:35,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +29: [2023-05-11 00:16:35,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +17: [2023-05-11 00:16:35,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +17: [2023-05-11 00:16:35,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +23: [2023-05-11 00:16:35,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +18: [2023-05-11 00:16:35,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +20: [2023-05-11 00:16:35,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +13: [2023-05-11 00:16:35,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +13: [2023-05-11 00:16:35,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +31: [2023-05-11 00:16:35,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +29: [2023-05-11 00:16:35,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +29: [2023-05-11 00:16:35,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +23: [2023-05-11 00:16:35,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +20: [2023-05-11 00:16:35,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +27: [2023-05-11 00:16:35,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +27: [2023-05-11 00:16:35,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +10: [2023-05-11 00:16:35,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +10: [2023-05-11 00:16:35,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +10: [2023-05-11 00:16:35,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +10: [2023-05-11 00:16:35,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +10: [2023-05-11 00:16:35,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +10: [2023-05-11 00:16:35,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +10: [2023-05-11 00:16:35,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +31: [2023-05-11 00:16:35,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 6: [2023-05-11 00:16:35,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 6: [2023-05-11 00:16:35,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +13: [2023-05-11 00:16:35,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +10: [2023-05-11 00:16:35,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +17: [2023-05-11 00:16:35,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +17: [2023-05-11 00:16:35,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +27: [2023-05-11 00:16:35,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +27: [2023-05-11 00:16:35,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +29: [2023-05-11 00:16:35,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +28: [2023-05-11 00:16:35,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +28: [2023-05-11 00:16:35,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +28: [2023-05-11 00:16:35,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +28: [2023-05-11 00:16:35,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +28: [2023-05-11 00:16:35,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +28: [2023-05-11 00:16:35,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +28: [2023-05-11 00:16:35,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +28: [2023-05-11 00:16:35,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +26: [2023-05-11 00:16:35,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +26: [2023-05-11 00:16:35,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +13: [2023-05-11 00:16:35,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +26: [2023-05-11 00:16:35,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +26: [2023-05-11 00:16:35,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +26: [2023-05-11 00:16:35,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +26: [2023-05-11 00:16:35,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +26: [2023-05-11 00:16:35,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +31: [2023-05-11 00:16:35,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +27: [2023-05-11 00:16:35,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +27: [2023-05-11 00:16:35,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 9: [2023-05-11 00:16:35,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +27: [2023-05-11 00:16:35,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +27: [2023-05-11 00:16:35,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +26: [2023-05-11 00:16:35,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +31: [2023-05-11 00:16:35,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +31: [2023-05-11 00:16:35,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +29: [2023-05-11 00:16:35,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +17: [2023-05-11 00:16:35,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +10: [2023-05-11 00:16:35,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +31: [2023-05-11 00:16:35,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +27: [2023-05-11 00:16:35,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +10: [2023-05-11 00:16:35,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +13: [2023-05-11 00:16:35,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +13: [2023-05-11 00:16:35,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +27: [2023-05-11 00:16:35,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +25: [2023-05-11 00:16:35,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +25: [2023-05-11 00:16:35,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +25: [2023-05-11 00:16:35,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +25: [2023-05-11 00:16:35,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +25: [2023-05-11 00:16:35,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +25: [2023-05-11 00:16:35,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +25: [2023-05-11 00:16:35,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +25: [2023-05-11 00:16:35,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +17: [2023-05-11 00:16:35,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +28: [2023-05-11 00:16:35,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +15: [2023-05-11 00:16:35,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +15: [2023-05-11 00:16:35,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +17: [2023-05-11 00:16:35,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +17: [2023-05-11 00:16:35,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +15: [2023-05-11 00:16:35,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +15: [2023-05-11 00:16:35,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 8: [2023-05-11 00:16:35,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +28: [2023-05-11 00:16:35,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +28: [2023-05-11 00:16:35,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +28: [2023-05-11 00:16:35,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +31: [2023-05-11 00:16:35,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +10: [2023-05-11 00:16:35,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +31: [2023-05-11 00:16:35,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +13: [2023-05-11 00:16:35,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +10: [2023-05-11 00:16:35,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +27: [2023-05-11 00:16:35,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +10: [2023-05-11 00:16:35,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +13: [2023-05-11 00:16:35,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +27: [2023-05-11 00:16:35,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +17: [2023-05-11 00:16:35,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +17: [2023-05-11 00:16:35,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +15: [2023-05-11 00:16:35,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +15: [2023-05-11 00:16:35,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +26: [2023-05-11 00:16:35,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +26: [2023-05-11 00:16:35,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +26: [2023-05-11 00:16:35,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +15: [2023-05-11 00:16:35,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +15: [2023-05-11 00:16:35,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +10: [2023-05-11 00:16:35,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +28: [2023-05-11 00:16:35,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 4: [2023-05-11 00:16:35,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 4: [2023-05-11 00:16:35,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 6: [2023-05-11 00:16:35,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 6: [2023-05-11 00:16:35,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +28: [2023-05-11 00:16:35,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +28: [2023-05-11 00:16:35,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +10: [2023-05-11 00:16:35,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +28: [2023-05-11 00:16:35,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +10: [2023-05-11 00:16:35,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +10: [2023-05-11 00:16:35,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +10: [2023-05-11 00:16:35,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +27: [2023-05-11 00:16:35,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +20: [2023-05-11 00:16:35,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +20: [2023-05-11 00:16:35,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +20: [2023-05-11 00:16:35,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +30: [2023-05-11 00:16:35,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +30: [2023-05-11 00:16:35,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +30: [2023-05-11 00:16:35,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +20: [2023-05-11 00:16:35,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +30: [2023-05-11 00:16:35,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +27: [2023-05-11 00:16:35,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +21: [2023-05-11 00:16:35,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 8: [2023-05-11 00:16:35,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +10: [2023-05-11 00:16:35,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +21: [2023-05-11 00:16:35,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 8: [2023-05-11 00:16:35,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 1: [2023-05-11 00:16:35,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +12: [2023-05-11 00:16:35,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +12: [2023-05-11 00:16:35,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +12: [2023-05-11 00:16:35,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +12: [2023-05-11 00:16:35,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +13: [2023-05-11 00:16:35,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +21: [2023-05-11 00:16:35,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 1: [2023-05-11 00:16:35,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 0: [2023-05-11 00:16:35,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 0: [2023-05-11 00:16:35,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +21: [2023-05-11 00:16:35,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +13: [2023-05-11 00:16:35,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +26: [2023-05-11 00:16:35,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +26: [2023-05-11 00:16:35,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 7: [2023-05-11 00:16:35,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 7: [2023-05-11 00:16:35,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +26: [2023-05-11 00:16:35,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +23: [2023-05-11 00:16:35,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +23: [2023-05-11 00:16:35,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +23: [2023-05-11 00:16:35,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +23: [2023-05-11 00:16:35,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 6: [2023-05-11 00:16:35,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +16: [2023-05-11 00:16:35,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 7: [2023-05-11 00:16:35,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 4: [2023-05-11 00:16:35,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +16: [2023-05-11 00:16:35,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +16: [2023-05-11 00:16:35,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +16: [2023-05-11 00:16:35,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +26: [2023-05-11 00:16:35,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +26: [2023-05-11 00:16:35,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +26: [2023-05-11 00:16:35,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 4: [2023-05-11 00:16:35,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +13: [2023-05-11 00:16:35,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 6: [2023-05-11 00:16:35,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +15: [2023-05-11 00:16:35,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +13: [2023-05-11 00:16:35,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 8: [2023-05-11 00:16:35,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +15: [2023-05-11 00:16:35,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 8: [2023-05-11 00:16:35,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 8: [2023-05-11 00:16:35,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +27: [2023-05-11 00:16:35,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +10: [2023-05-11 00:16:35,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +15: [2023-05-11 00:16:35,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +15: [2023-05-11 00:16:35,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +10: [2023-05-11 00:16:35,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +10: [2023-05-11 00:16:35,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +16: [2023-05-11 00:16:35,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +10: [2023-05-11 00:16:35,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +27: [2023-05-11 00:16:35,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +26: [2023-05-11 00:16:35,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +16: [2023-05-11 00:16:35,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +16: [2023-05-11 00:16:35,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +30: [2023-05-11 00:16:35,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +20: [2023-05-11 00:16:35,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +20: [2023-05-11 00:16:35,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +15: [2023-05-11 00:16:35,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +20: [2023-05-11 00:16:35,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +15: [2023-05-11 00:16:35,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +20: [2023-05-11 00:16:35,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +16: [2023-05-11 00:16:35,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +12: [2023-05-11 00:16:35,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +13: [2023-05-11 00:16:35,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +10: [2023-05-11 00:16:35,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +30: [2023-05-11 00:16:35,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +12: [2023-05-11 00:16:35,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +15: [2023-05-11 00:16:35,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +15: [2023-05-11 00:16:35,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +13: [2023-05-11 00:16:35,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +21: [2023-05-11 00:16:35,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +21: [2023-05-11 00:16:35,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +21: [2023-05-11 00:16:35,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +12: [2023-05-11 00:16:35,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +12: [2023-05-11 00:16:35,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +23: [2023-05-11 00:16:35,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +23: [2023-05-11 00:16:35,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +26: [2023-05-11 00:16:35,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +26: [2023-05-11 00:16:35,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +21: [2023-05-11 00:16:35,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +23: [2023-05-11 00:16:35,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +26: [2023-05-11 00:16:35,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +23: [2023-05-11 00:16:35,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +30: [2023-05-11 00:16:35,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +30: [2023-05-11 00:16:35,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +26: [2023-05-11 00:16:35,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +13: [2023-05-11 00:16:35,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +13: [2023-05-11 00:16:35,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 8: [2023-05-11 00:16:35,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +26: [2023-05-11 00:16:35,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +10: [2023-05-11 00:16:35,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +10: [2023-05-11 00:16:35,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +16: [2023-05-11 00:16:35,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +26: [2023-05-11 00:16:35,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +16: [2023-05-11 00:16:35,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +13: [2023-05-11 00:16:35,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +28: [2023-05-11 00:16:35,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +28: [2023-05-11 00:16:35,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +28: [2023-05-11 00:16:35,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +28: [2023-05-11 00:16:35,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 7: [2023-05-11 00:16:35,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +12: [2023-05-11 00:16:35,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +12: [2023-05-11 00:16:35,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 1: [2023-05-11 00:16:35,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +23: [2023-05-11 00:16:35,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +13: [2023-05-11 00:16:35,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +21: [2023-05-11 00:16:35,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +13: [2023-05-11 00:16:35,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +13: [2023-05-11 00:16:35,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 1: [2023-05-11 00:16:35,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +30: [2023-05-11 00:16:35,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +21: [2023-05-11 00:16:35,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +30: [2023-05-11 00:16:35,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +21: [2023-05-11 00:16:35,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +21: [2023-05-11 00:16:35,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 8: [2023-05-11 00:16:35,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +10: [2023-05-11 00:16:35,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +10: [2023-05-11 00:16:35,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +23: [2023-05-11 00:16:35,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +23: [2023-05-11 00:16:35,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +23: [2023-05-11 00:16:35,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +26: [2023-05-11 00:16:35,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +26: [2023-05-11 00:16:35,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +23: [2023-05-11 00:16:35,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +23: [2023-05-11 00:16:35,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +10: [2023-05-11 00:16:35,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +10: [2023-05-11 00:16:35,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +16: [2023-05-11 00:16:35,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +20: [2023-05-11 00:16:35,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +12: [2023-05-11 00:16:35,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +20: [2023-05-11 00:16:35,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +12: [2023-05-11 00:16:35,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +12: [2023-05-11 00:16:35,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +12: [2023-05-11 00:16:35,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +20: [2023-05-11 00:16:35,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +20: [2023-05-11 00:16:35,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +20: [2023-05-11 00:16:35,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +20: [2023-05-11 00:16:35,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 7: [2023-05-11 00:16:35,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 7: [2023-05-11 00:16:35,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +10: [2023-05-11 00:16:35,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +21: [2023-05-11 00:16:35,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +21: [2023-05-11 00:16:35,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +26: [2023-05-11 00:16:35,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +26: [2023-05-11 00:16:35,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 4: [2023-05-11 00:16:35,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +13: [2023-05-11 00:16:35,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +13: [2023-05-11 00:16:35,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +13: [2023-05-11 00:16:35,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 4: [2023-05-11 00:16:35,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +13: [2023-05-11 00:16:35,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 7: [2023-05-11 00:16:35,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +10: [2023-05-11 00:16:35,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +26: [2023-05-11 00:16:35,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +26: [2023-05-11 00:16:35,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 0: [2023-05-11 00:16:35,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 4: [2023-05-11 00:16:35,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +11: [2023-05-11 00:16:35,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +11: [2023-05-11 00:16:35,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +11: [2023-05-11 00:16:35,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +11: [2023-05-11 00:16:35,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +28: [2023-05-11 00:16:35,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +16: [2023-05-11 00:16:35,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +28: [2023-05-11 00:16:35,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +20: [2023-05-11 00:16:35,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +28: [2023-05-11 00:16:35,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +28: [2023-05-11 00:16:35,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +26: [2023-05-11 00:16:35,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +26: [2023-05-11 00:16:35,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +16: [2023-05-11 00:16:35,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +16: [2023-05-11 00:16:35,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +17: [2023-05-11 00:16:35,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +17: [2023-05-11 00:16:35,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +17: [2023-05-11 00:16:35,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +20: [2023-05-11 00:16:35,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +20: [2023-05-11 00:16:35,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +20: [2023-05-11 00:16:35,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +20: [2023-05-11 00:16:35,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +20: [2023-05-11 00:16:35,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +20: [2023-05-11 00:16:35,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +20: [2023-05-11 00:16:35,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +17: [2023-05-11 00:16:35,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +20: [2023-05-11 00:16:35,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +20: [2023-05-11 00:16:35,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +18: [2023-05-11 00:16:35,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +18: [2023-05-11 00:16:35,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +18: [2023-05-11 00:16:35,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +18: [2023-05-11 00:16:35,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +11: [2023-05-11 00:16:35,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +11: [2023-05-11 00:16:35,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +11: [2023-05-11 00:16:35,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +11: [2023-05-11 00:16:35,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +12: [2023-05-11 00:16:35,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +21: [2023-05-11 00:16:35,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +12: [2023-05-11 00:16:35,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 1: [2023-05-11 00:16:35,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +23: [2023-05-11 00:16:35,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +18: [2023-05-11 00:16:35,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +21: [2023-05-11 00:16:35,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +16: [2023-05-11 00:16:35,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +30: [2023-05-11 00:16:35,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +18: [2023-05-11 00:16:35,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +16: [2023-05-11 00:16:35,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +30: [2023-05-11 00:16:35,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +23: [2023-05-11 00:16:35,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +18: [2023-05-11 00:16:35,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +17: [2023-05-11 00:16:35,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +17: [2023-05-11 00:16:35,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +18: [2023-05-11 00:16:35,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +30: [2023-05-11 00:16:35,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +30: [2023-05-11 00:16:35,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +28: [2023-05-11 00:16:35,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +28: [2023-05-11 00:16:35,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +30: [2023-05-11 00:16:35,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +17: [2023-05-11 00:16:35,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +17: [2023-05-11 00:16:35,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +30: [2023-05-11 00:16:35,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 6: [2023-05-11 00:16:35,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +11: [2023-05-11 00:16:35,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +11: [2023-05-11 00:16:35,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +20: [2023-05-11 00:16:35,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +11: [2023-05-11 00:16:35,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +20: [2023-05-11 00:16:35,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +20: [2023-05-11 00:16:35,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +28: [2023-05-11 00:16:35,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +28: [2023-05-11 00:16:35,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +28: [2023-05-11 00:16:35,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +28: [2023-05-11 00:16:35,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +20: [2023-05-11 00:16:35,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +11: [2023-05-11 00:16:35,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +18: [2023-05-11 00:16:35,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +28: [2023-05-11 00:16:35,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +28: [2023-05-11 00:16:35,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +18: [2023-05-11 00:16:35,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +18: [2023-05-11 00:16:35,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +23: [2023-05-11 00:16:35,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +23: [2023-05-11 00:16:35,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +18: [2023-05-11 00:16:35,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +17: [2023-05-11 00:16:35,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +23: [2023-05-11 00:16:35,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +23: [2023-05-11 00:16:35,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +17: [2023-05-11 00:16:35,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +17: [2023-05-11 00:16:35,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +17: [2023-05-11 00:16:35,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +18: [2023-05-11 00:16:35,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +16: [2023-05-11 00:16:35,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +16: [2023-05-11 00:16:35,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +16: [2023-05-11 00:16:35,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +23: [2023-05-11 00:16:35,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +23: [2023-05-11 00:16:35,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +11: [2023-05-11 00:16:35,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +20: [2023-05-11 00:16:35,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +20: [2023-05-11 00:16:35,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +18: [2023-05-11 00:16:35,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +11: [2023-05-11 00:16:35,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +20: [2023-05-11 00:16:35,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +20: [2023-05-11 00:16:35,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +23: [2023-05-11 00:16:35,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +11: [2023-05-11 00:16:35,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +23: [2023-05-11 00:16:35,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +11: [2023-05-11 00:16:35,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +11: [2023-05-11 00:16:35,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +16: [2023-05-11 00:16:35,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +11: [2023-05-11 00:16:35,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +18: [2023-05-11 00:16:35,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +11: [2023-05-11 00:16:35,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +11: [2023-05-11 00:16:35,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +17: [2023-05-11 00:16:35,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +13: [2023-05-11 00:16:35,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +13: [2023-05-11 00:16:35,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +18: [2023-05-11 00:16:35,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +18: [2023-05-11 00:16:35,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +13: [2023-05-11 00:16:35,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +13: [2023-05-11 00:16:35,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +13: [2023-05-11 00:16:35,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +13: [2023-05-11 00:16:35,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +17: [2023-05-11 00:16:35,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +13: [2023-05-11 00:16:35,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +18: [2023-05-11 00:16:35,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +13: [2023-05-11 00:16:35,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +23: [2023-05-11 00:16:35,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +18: [2023-05-11 00:16:35,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +17: [2023-05-11 00:16:35,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +11: [2023-05-11 00:16:35,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +23: [2023-05-11 00:16:35,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +18: [2023-05-11 00:16:35,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +11: [2023-05-11 00:16:35,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +17: [2023-05-11 00:16:35,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +11: [2023-05-11 00:16:35,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +16: [2023-05-11 00:16:35,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +16: [2023-05-11 00:16:35,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +31: [2023-05-11 00:16:35,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +16: [2023-05-11 00:16:35,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +31: [2023-05-11 00:16:35,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +31: [2023-05-11 00:16:35,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +31: [2023-05-11 00:16:35,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +31: [2023-05-11 00:16:35,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +11: [2023-05-11 00:16:35,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +16: [2023-05-11 00:16:35,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +31: [2023-05-11 00:16:35,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +31: [2023-05-11 00:16:35,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +31: [2023-05-11 00:16:35,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +13: [2023-05-11 00:16:35,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +25: [2023-05-11 00:16:35,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +25: [2023-05-11 00:16:35,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 6: [2023-05-11 00:16:35,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +25: [2023-05-11 00:16:35,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +25: [2023-05-11 00:16:35,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +25: [2023-05-11 00:16:35,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +25: [2023-05-11 00:16:35,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +18: [2023-05-11 00:16:35,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +25: [2023-05-11 00:16:35,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +25: [2023-05-11 00:16:35,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +19: [2023-05-11 00:16:35,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +19: [2023-05-11 00:16:35,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +23: [2023-05-11 00:16:35,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +23: [2023-05-11 00:16:35,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +19: [2023-05-11 00:16:35,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +19: [2023-05-11 00:16:35,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +14: [2023-05-11 00:16:35,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +13: [2023-05-11 00:16:35,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +13: [2023-05-11 00:16:35,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +14: [2023-05-11 00:16:35,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +14: [2023-05-11 00:16:35,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +14: [2023-05-11 00:16:35,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +27: [2023-05-11 00:16:35,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +27: [2023-05-11 00:16:35,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +27: [2023-05-11 00:16:35,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +27: [2023-05-11 00:16:35,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 2: [2023-05-11 00:16:35,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 2: [2023-05-11 00:16:35,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 2: [2023-05-11 00:16:35,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +18: [2023-05-11 00:16:35,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +18: [2023-05-11 00:16:35,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +16: [2023-05-11 00:16:35,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +31: [2023-05-11 00:16:35,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +16: [2023-05-11 00:16:35,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +18: [2023-05-11 00:16:35,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +18: [2023-05-11 00:16:35,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +18: [2023-05-11 00:16:35,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +18: [2023-05-11 00:16:35,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +23: [2023-05-11 00:16:35,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +18: [2023-05-11 00:16:35,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +11: [2023-05-11 00:16:35,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +11: [2023-05-11 00:16:35,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +23: [2023-05-11 00:16:35,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +11: [2023-05-11 00:16:35,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +11: [2023-05-11 00:16:35,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +18: [2023-05-11 00:16:35,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +11: [2023-05-11 00:16:35,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +18: [2023-05-11 00:16:35,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +18: [2023-05-11 00:16:35,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +11: [2023-05-11 00:16:35,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 2: [2023-05-11 00:16:35,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 2: [2023-05-11 00:16:35,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +16: [2023-05-11 00:16:35,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +16: [2023-05-11 00:16:35,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +16: [2023-05-11 00:16:35,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +16: [2023-05-11 00:16:35,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 0: [2023-05-11 00:16:35,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 6: [2023-05-11 00:16:35,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +11: [2023-05-11 00:16:35,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +19: [2023-05-11 00:16:35,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +11: [2023-05-11 00:16:35,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +19: [2023-05-11 00:16:35,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +25: [2023-05-11 00:16:35,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +13: [2023-05-11 00:16:35,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +13: [2023-05-11 00:16:35,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +31: [2023-05-11 00:16:35,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +25: [2023-05-11 00:16:35,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +13: [2023-05-11 00:16:35,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +19: [2023-05-11 00:16:35,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +31: [2023-05-11 00:16:35,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +13: [2023-05-11 00:16:35,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +14: [2023-05-11 00:16:35,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +14: [2023-05-11 00:16:35,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +27: [2023-05-11 00:16:35,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +14: [2023-05-11 00:16:35,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +19: [2023-05-11 00:16:35,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +25: [2023-05-11 00:16:35,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +27: [2023-05-11 00:16:35,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +23: [2023-05-11 00:16:35,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +23: [2023-05-11 00:16:35,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +18: [2023-05-11 00:16:35,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +14: [2023-05-11 00:16:35,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 5: [2023-05-11 00:16:35,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 5: [2023-05-11 00:16:35,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 5: [2023-05-11 00:16:35,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 6: [2023-05-11 00:16:35,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +25: [2023-05-11 00:16:35,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +27: [2023-05-11 00:16:35,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +27: [2023-05-11 00:16:35,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +25: [2023-05-11 00:16:35,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +22: [2023-05-11 00:16:35,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +22: [2023-05-11 00:16:35,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 2: [2023-05-11 00:16:35,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +31: [2023-05-11 00:16:35,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +25: [2023-05-11 00:16:35,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +31: [2023-05-11 00:16:35,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +13: [2023-05-11 00:16:35,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +31: [2023-05-11 00:16:35,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +31: [2023-05-11 00:16:35,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +16: [2023-05-11 00:16:35,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +22: [2023-05-11 00:16:35,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +22: [2023-05-11 00:16:35,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +16: [2023-05-11 00:16:35,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 5: [2023-05-11 00:16:35,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +19: [2023-05-11 00:16:35,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +19: [2023-05-11 00:16:35,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +25: [2023-05-11 00:16:35,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 3: [2023-05-11 00:16:35,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 3: [2023-05-11 00:16:35,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 3: [2023-05-11 00:16:35,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +14: [2023-05-11 00:16:35,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +14: [2023-05-11 00:16:35,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +14: [2023-05-11 00:16:35,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +14: [2023-05-11 00:16:35,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +25: [2023-05-11 00:16:35,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +19: [2023-05-11 00:16:35,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +19: [2023-05-11 00:16:35,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 5: [2023-05-11 00:16:35,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +22: [2023-05-11 00:16:35,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +31: [2023-05-11 00:16:35,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +27: [2023-05-11 00:16:35,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +22: [2023-05-11 00:16:35,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +27: [2023-05-11 00:16:35,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +14: [2023-05-11 00:16:35,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +14: [2023-05-11 00:16:35,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +26: [2023-05-11 00:16:35,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +26: [2023-05-11 00:16:35,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +26: [2023-05-11 00:16:35,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +26: [2023-05-11 00:16:35,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +26: [2023-05-11 00:16:35,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +26: [2023-05-11 00:16:35,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +26: [2023-05-11 00:16:35,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +27: [2023-05-11 00:16:35,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +27: [2023-05-11 00:16:35,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +22: [2023-05-11 00:16:35,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +31: [2023-05-11 00:16:35,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +22: [2023-05-11 00:16:35,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +26: [2023-05-11 00:16:35,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +31: [2023-05-11 00:16:35,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +31: [2023-05-11 00:16:35,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +31: [2023-05-11 00:16:35,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. + 0: [2023-05-11 00:16:35,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +19: [2023-05-11 00:16:35,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +19: [2023-05-11 00:16:35,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 0: [2023-05-11 00:16:35,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +14: [2023-05-11 00:16:35,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +14: [2023-05-11 00:16:35,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 5: [2023-05-11 00:16:35,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +19: [2023-05-11 00:16:35,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +19: [2023-05-11 00:16:35,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +25: [2023-05-11 00:16:35,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +25: [2023-05-11 00:16:35,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +25: [2023-05-11 00:16:35,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +25: [2023-05-11 00:16:35,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt. +24: [2023-05-11 00:16:35,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +31: [2023-05-11 00:16:35,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +24: [2023-05-11 00:16:35,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +22: [2023-05-11 00:16:35,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +31: [2023-05-11 00:16:35,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 0: [2023-05-11 00:16:35,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +31: [2023-05-11 00:16:35,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +22: [2023-05-11 00:16:35,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 2: [2023-05-11 00:16:35,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 5: [2023-05-11 00:16:35,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +24: [2023-05-11 00:16:35,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +22: [2023-05-11 00:16:35,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +22: [2023-05-11 00:16:35,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 2: [2023-05-11 00:16:35,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +24: [2023-05-11 00:16:35,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +22: [2023-05-11 00:16:35,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +22: [2023-05-11 00:16:35,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +27: [2023-05-11 00:16:35,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +27: [2023-05-11 00:16:35,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +27: [2023-05-11 00:16:35,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +27: [2023-05-11 00:16:35,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +31: [2023-05-11 00:16:35,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +25: [2023-05-11 00:16:35,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +25: [2023-05-11 00:16:35,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +17: [2023-05-11 00:16:35,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +17: [2023-05-11 00:16:35,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +17: [2023-05-11 00:16:35,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +17: [2023-05-11 00:16:35,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +17: [2023-05-11 00:16:35,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +17: [2023-05-11 00:16:35,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +22: [2023-05-11 00:16:35,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +25: [2023-05-11 00:16:35,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... +26: [2023-05-11 00:16:35,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +26: [2023-05-11 00:16:35,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 0: [2023-05-11 00:16:35,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +17: [2023-05-11 00:16:35,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +17: [2023-05-11 00:16:35,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +22: [2023-05-11 00:16:35,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 3: [2023-05-11 00:16:35,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +24: [2023-05-11 00:16:35,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 3: [2023-05-11 00:16:35,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +24: [2023-05-11 00:16:35,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 3: [2023-05-11 00:16:35,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 5: [2023-05-11 00:16:35,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +25: [2023-05-11 00:16:35,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +24: [2023-05-11 00:16:35,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 3: [2023-05-11 00:16:35,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +24: [2023-05-11 00:16:35,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +26: [2023-05-11 00:16:35,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 2: [2023-05-11 00:16:35,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 2: [2023-05-11 00:16:35,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 0: [2023-05-11 00:16:35,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 2: [2023-05-11 00:16:35,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +26: [2023-05-11 00:16:35,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +25: [2023-05-11 00:16:35,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +26: [2023-05-11 00:16:35,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +25: [2023-05-11 00:16:35,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +25: [2023-05-11 00:16:35,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +25: [2023-05-11 00:16:35,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 8: [2023-05-11 00:16:35,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +31: [2023-05-11 00:16:35,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +31: [2023-05-11 00:16:35,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +31: [2023-05-11 00:16:35,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +31: [2023-05-11 00:16:35,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +24: [2023-05-11 00:16:35,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +17: [2023-05-11 00:16:35,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +17: [2023-05-11 00:16:35,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +24: [2023-05-11 00:16:35,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +17: [2023-05-11 00:16:35,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +17: [2023-05-11 00:16:35,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +25: [2023-05-11 00:16:35,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +24: [2023-05-11 00:16:35,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +24: [2023-05-11 00:16:35,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +31: [2023-05-11 00:16:35,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +25: [2023-05-11 00:16:35,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +24: [2023-05-11 00:16:35,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +24: [2023-05-11 00:16:35,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +26: [2023-05-11 00:16:35,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +31: [2023-05-11 00:16:35,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... +24: [2023-05-11 00:16:35,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +24: [2023-05-11 00:16:35,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +26: [2023-05-11 00:16:35,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +17: [2023-05-11 00:16:35,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +17: [2023-05-11 00:16:35,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +17: [2023-05-11 00:16:35,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +17: [2023-05-11 00:16:35,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +31: [2023-05-11 00:16:35,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +31: [2023-05-11 00:16:35,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 8: [2023-05-11 00:16:35,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +26: [2023-05-11 00:16:35,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +25: [2023-05-11 00:16:35,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt. +25: [2023-05-11 00:16:35,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt... + 1: [2023-05-11 00:16:35,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 1: [2023-05-11 00:16:35,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 9: [2023-05-11 00:16:35,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 9: [2023-05-11 00:16:35,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 9: [2023-05-11 00:16:35,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 1: [2023-05-11 00:16:35,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 1: [2023-05-11 00:16:35,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 9: [2023-05-11 00:16:35,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 9: [2023-05-11 00:16:35,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 9: [2023-05-11 00:16:35,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 8: [2023-05-11 00:16:35,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 8: [2023-05-11 00:16:35,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 8: [2023-05-11 00:16:35,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 8: [2023-05-11 00:16:35,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 9: [2023-05-11 00:16:35,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +24: [2023-05-11 00:16:35,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +24: [2023-05-11 00:16:35,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +24: [2023-05-11 00:16:35,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +24: [2023-05-11 00:16:35,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +24: [2023-05-11 00:16:35,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +24: [2023-05-11 00:16:35,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 9: [2023-05-11 00:16:35,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +24: [2023-05-11 00:16:35,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +24: [2023-05-11 00:16:35,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +24: [2023-05-11 00:16:35,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +24: [2023-05-11 00:16:35,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +24: [2023-05-11 00:16:35,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 8: [2023-05-11 00:16:35,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +24: [2023-05-11 00:16:35,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 8: [2023-05-11 00:16:35,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +24: [2023-05-11 00:16:35,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +24: [2023-05-11 00:16:35,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 8: [2023-05-11 00:16:35,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +29: [2023-05-11 00:16:35,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +29: [2023-05-11 00:16:35,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +29: [2023-05-11 00:16:35,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +29: [2023-05-11 00:16:35,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +24: [2023-05-11 00:16:35,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +10: [2023-05-11 00:16:35,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +10: [2023-05-11 00:16:35,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +10: [2023-05-11 00:16:35,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +10: [2023-05-11 00:16:35,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 7: [2023-05-11 00:16:35,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 7: [2023-05-11 00:16:35,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 7: [2023-05-11 00:16:35,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 7: [2023-05-11 00:16:35,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +29: [2023-05-11 00:16:35,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +29: [2023-05-11 00:16:35,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +29: [2023-05-11 00:16:35,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +24: [2023-05-11 00:16:35,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +29: [2023-05-11 00:16:35,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +22: [2023-05-11 00:16:35,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +22: [2023-05-11 00:16:35,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +22: [2023-05-11 00:16:35,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +22: [2023-05-11 00:16:35,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +10: [2023-05-11 00:16:35,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +15: [2023-05-11 00:16:35,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +15: [2023-05-11 00:16:35,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +15: [2023-05-11 00:16:35,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +15: [2023-05-11 00:16:35,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +10: [2023-05-11 00:16:35,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +22: [2023-05-11 00:16:35,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +22: [2023-05-11 00:16:35,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +15: [2023-05-11 00:16:35,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +15: [2023-05-11 00:16:35,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +15: [2023-05-11 00:16:35,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +15: [2023-05-11 00:16:35,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +15: [2023-05-11 00:16:35,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +15: [2023-05-11 00:16:35,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +15: [2023-05-11 00:16:35,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +15: [2023-05-11 00:16:35,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +22: [2023-05-11 00:16:35,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +22: [2023-05-11 00:16:35,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +10: [2023-05-11 00:16:35,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +10: [2023-05-11 00:16:35,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +29: [2023-05-11 00:16:35,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +29: [2023-05-11 00:16:35,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +29: [2023-05-11 00:16:35,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +29: [2023-05-11 00:16:35,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +29: [2023-05-11 00:16:35,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +29: [2023-05-11 00:16:35,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +10: [2023-05-11 00:16:35,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +15: [2023-05-11 00:16:35,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +10: [2023-05-11 00:16:35,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +22: [2023-05-11 00:16:35,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +22: [2023-05-11 00:16:35,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +15: [2023-05-11 00:16:35,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +15: [2023-05-11 00:16:35,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +15: [2023-05-11 00:16:35,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +15: [2023-05-11 00:16:35,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 7: [2023-05-11 00:16:35,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 7: [2023-05-11 00:16:35,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +10: [2023-05-11 00:16:35,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +10: [2023-05-11 00:16:35,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +29: [2023-05-11 00:16:35,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +29: [2023-05-11 00:16:35,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +22: [2023-05-11 00:16:35,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +22: [2023-05-11 00:16:35,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +15: [2023-05-11 00:16:35,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:35,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +23: [2023-05-11 00:16:35,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +12: [2023-05-11 00:16:35,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +26: [2023-05-11 00:16:35,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +12: [2023-05-11 00:16:35,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +26: [2023-05-11 00:16:35,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +26: [2023-05-11 00:16:35,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +15: [2023-05-11 00:16:35,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:35,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +23: [2023-05-11 00:16:35,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +26: [2023-05-11 00:16:35,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +23: [2023-05-11 00:16:35,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +23: [2023-05-11 00:16:35,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +15: [2023-05-11 00:16:35,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +15: [2023-05-11 00:16:35,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +10: [2023-05-11 00:16:35,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 7: [2023-05-11 00:16:35,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +22: [2023-05-11 00:16:35,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +10: [2023-05-11 00:16:35,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +22: [2023-05-11 00:16:35,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 4: [2023-05-11 00:16:35,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 4: [2023-05-11 00:16:35,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +14: [2023-05-11 00:16:35,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +14: [2023-05-11 00:16:35,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +14: [2023-05-11 00:16:35,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +14: [2023-05-11 00:16:35,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +21: [2023-05-11 00:16:35,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +21: [2023-05-11 00:16:35,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +15: [2023-05-11 00:16:35,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +12: [2023-05-11 00:16:35,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +26: [2023-05-11 00:16:35,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +21: [2023-05-11 00:16:35,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +21: [2023-05-11 00:16:35,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +21: [2023-05-11 00:16:35,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +21: [2023-05-11 00:16:35,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +15: [2023-05-11 00:16:35,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +15: [2023-05-11 00:16:35,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +12: [2023-05-11 00:16:35,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +12: [2023-05-11 00:16:35,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +26: [2023-05-11 00:16:35,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +21: [2023-05-11 00:16:35,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:35,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +14: [2023-05-11 00:16:35,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +14: [2023-05-11 00:16:35,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +10: [2023-05-11 00:16:35,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +26: [2023-05-11 00:16:35,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +23: [2023-05-11 00:16:35,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +21: [2023-05-11 00:16:35,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +14: [2023-05-11 00:16:35,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +23: [2023-05-11 00:16:35,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +14: [2023-05-11 00:16:35,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +10: [2023-05-11 00:16:35,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +23: [2023-05-11 00:16:35,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +23: [2023-05-11 00:16:35,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +15: [2023-05-11 00:16:35,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +30: [2023-05-11 00:16:35,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +15: [2023-05-11 00:16:35,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +15: [2023-05-11 00:16:35,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 7: [2023-05-11 00:16:35,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +26: [2023-05-11 00:16:35,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +30: [2023-05-11 00:16:35,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +19: [2023-05-11 00:16:35,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +30: [2023-05-11 00:16:35,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +19: [2023-05-11 00:16:35,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +19: [2023-05-11 00:16:35,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +30: [2023-05-11 00:16:35,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +19: [2023-05-11 00:16:35,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +22: [2023-05-11 00:16:35,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 4: [2023-05-11 00:16:35,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +22: [2023-05-11 00:16:35,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +15: [2023-05-11 00:16:35,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +15: [2023-05-11 00:16:35,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +15: [2023-05-11 00:16:35,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +15: [2023-05-11 00:16:35,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 4: [2023-05-11 00:16:35,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +15: [2023-05-11 00:16:35,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +21: [2023-05-11 00:16:35,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:35,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +26: [2023-05-11 00:16:35,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +12: [2023-05-11 00:16:35,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +26: [2023-05-11 00:16:35,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 7: [2023-05-11 00:16:35,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +26: [2023-05-11 00:16:35,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +26: [2023-05-11 00:16:35,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 4: [2023-05-11 00:16:35,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +23: [2023-05-11 00:16:35,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +23: [2023-05-11 00:16:35,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 7: [2023-05-11 00:16:35,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +26: [2023-05-11 00:16:35,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +26: [2023-05-11 00:16:35,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +19: [2023-05-11 00:16:35,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 7: [2023-05-11 00:16:35,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +30: [2023-05-11 00:16:35,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +30: [2023-05-11 00:16:35,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +23: [2023-05-11 00:16:35,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +23: [2023-05-11 00:16:35,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +19: [2023-05-11 00:16:35,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +30: [2023-05-11 00:16:35,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +30: [2023-05-11 00:16:35,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +21: [2023-05-11 00:16:35,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +21: [2023-05-11 00:16:35,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +19: [2023-05-11 00:16:35,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +21: [2023-05-11 00:16:35,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +14: [2023-05-11 00:16:35,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:35,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +12: [2023-05-11 00:16:35,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +14: [2023-05-11 00:16:35,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +14: [2023-05-11 00:16:35,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +19: [2023-05-11 00:16:35,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +12: [2023-05-11 00:16:35,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +12: [2023-05-11 00:16:35,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 5: [2023-05-11 00:16:35,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 5: [2023-05-11 00:16:35,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 5: [2023-05-11 00:16:35,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +24: [2023-05-11 00:16:36,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +24: [2023-05-11 00:16:36,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +24: [2023-05-11 00:16:36,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 7: [2023-05-11 00:16:36,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +24: [2023-05-11 00:16:36,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +23: [2023-05-11 00:16:36,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +23: [2023-05-11 00:16:36,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 3: [2023-05-11 00:16:36,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:36,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +12: [2023-05-11 00:16:36,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 3: [2023-05-11 00:16:36,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +23: [2023-05-11 00:16:36,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +28: [2023-05-11 00:16:36,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +21: [2023-05-11 00:16:36,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +28: [2023-05-11 00:16:36,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +28: [2023-05-11 00:16:36,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +28: [2023-05-11 00:16:36,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +21: [2023-05-11 00:16:36,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +14: [2023-05-11 00:16:36,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +23: [2023-05-11 00:16:36,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 4: [2023-05-11 00:16:36,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +24: [2023-05-11 00:16:36,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +14: [2023-05-11 00:16:36,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +14: [2023-05-11 00:16:36,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +24: [2023-05-11 00:16:36,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +24: [2023-05-11 00:16:36,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +24: [2023-05-11 00:16:36,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +26: [2023-05-11 00:16:36,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +14: [2023-05-11 00:16:36,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +26: [2023-05-11 00:16:36,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 4: [2023-05-11 00:16:36,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 3: [2023-05-11 00:16:36,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +21: [2023-05-11 00:16:36,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +19: [2023-05-11 00:16:36,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +19: [2023-05-11 00:16:36,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +19: [2023-05-11 00:16:36,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +19: [2023-05-11 00:16:36,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +30: [2023-05-11 00:16:36,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +30: [2023-05-11 00:16:36,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +30: [2023-05-11 00:16:36,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +30: [2023-05-11 00:16:36,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +28: [2023-05-11 00:16:36,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +28: [2023-05-11 00:16:36,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +19: [2023-05-11 00:16:36,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +19: [2023-05-11 00:16:36,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +19: [2023-05-11 00:16:36,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +19: [2023-05-11 00:16:36,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +19: [2023-05-11 00:16:36,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +19: [2023-05-11 00:16:36,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +28: [2023-05-11 00:16:36,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +28: [2023-05-11 00:16:36,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +14: [2023-05-11 00:16:36,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +30: [2023-05-11 00:16:36,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +30: [2023-05-11 00:16:36,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +30: [2023-05-11 00:16:36,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +30: [2023-05-11 00:16:36,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 9: [2023-05-11 00:16:36,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +21: [2023-05-11 00:16:36,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +24: [2023-05-11 00:16:36,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +24: [2023-05-11 00:16:36,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +24: [2023-05-11 00:16:36,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +24: [2023-05-11 00:16:36,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +24: [2023-05-11 00:16:36,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +24: [2023-05-11 00:16:36,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +19: [2023-05-11 00:16:36,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +19: [2023-05-11 00:16:36,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +19: [2023-05-11 00:16:36,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +24: [2023-05-11 00:16:36,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +24: [2023-05-11 00:16:36,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 3: [2023-05-11 00:16:36,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +19: [2023-05-11 00:16:36,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +28: [2023-05-11 00:16:36,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +28: [2023-05-11 00:16:36,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +19: [2023-05-11 00:16:36,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 3: [2023-05-11 00:16:36,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +19: [2023-05-11 00:16:36,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +21: [2023-05-11 00:16:36,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +21: [2023-05-11 00:16:36,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +21: [2023-05-11 00:16:36,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +21: [2023-05-11 00:16:36,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 9: [2023-05-11 00:16:36,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +19: [2023-05-11 00:16:36,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +28: [2023-05-11 00:16:36,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +28: [2023-05-11 00:16:36,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +28: [2023-05-11 00:16:36,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +28: [2023-05-11 00:16:36,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +28: [2023-05-11 00:16:36,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +28: [2023-05-11 00:16:36,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +28: [2023-05-11 00:16:36,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +28: [2023-05-11 00:16:36,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 3: [2023-05-11 00:16:36,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +19: [2023-05-11 00:16:36,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +28: [2023-05-11 00:16:36,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +28: [2023-05-11 00:16:36,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 5: [2023-05-11 00:16:36,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 5: [2023-05-11 00:16:36,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 5: [2023-05-11 00:16:36,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 5: [2023-05-11 00:16:36,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 5: [2023-05-11 00:16:36,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 5: [2023-05-11 00:16:36,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 5: [2023-05-11 00:16:36,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 9: [2023-05-11 00:16:36,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +21: [2023-05-11 00:16:36,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +21: [2023-05-11 00:16:36,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +19: [2023-05-11 00:16:36,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +19: [2023-05-11 00:16:36,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +16: [2023-05-11 00:16:36,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +16: [2023-05-11 00:16:36,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +11: [2023-05-11 00:16:36,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +11: [2023-05-11 00:16:36,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 1: [2023-05-11 00:16:36,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +19: [2023-05-11 00:16:36,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +21: [2023-05-11 00:16:36,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 1: [2023-05-11 00:16:36,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +16: [2023-05-11 00:16:36,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +16: [2023-05-11 00:16:36,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 1: [2023-05-11 00:16:36,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +20: [2023-05-11 00:16:36,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +20: [2023-05-11 00:16:36,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +20: [2023-05-11 00:16:36,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +11: [2023-05-11 00:16:36,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +11: [2023-05-11 00:16:36,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +21: [2023-05-11 00:16:36,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +19: [2023-05-11 00:16:36,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 9: [2023-05-11 00:16:36,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +20: [2023-05-11 00:16:36,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +19: [2023-05-11 00:16:36,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +27: [2023-05-11 00:16:36,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +27: [2023-05-11 00:16:36,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +19: [2023-05-11 00:16:36,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +28: [2023-05-11 00:16:36,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +10: [2023-05-11 00:16:36,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +10: [2023-05-11 00:16:36,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +10: [2023-05-11 00:16:36,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +10: [2023-05-11 00:16:36,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +10: [2023-05-11 00:16:36,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +10: [2023-05-11 00:16:36,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 9: [2023-05-11 00:16:36,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 9: [2023-05-11 00:16:36,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 9: [2023-05-11 00:16:36,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +27: [2023-05-11 00:16:36,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +27: [2023-05-11 00:16:36,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +10: [2023-05-11 00:16:36,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +28: [2023-05-11 00:16:36,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +11: [2023-05-11 00:16:36,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +16: [2023-05-11 00:16:36,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +16: [2023-05-11 00:16:36,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +10: [2023-05-11 00:16:36,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +16: [2023-05-11 00:16:36,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +27: [2023-05-11 00:16:36,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +20: [2023-05-11 00:16:36,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +27: [2023-05-11 00:16:36,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +27: [2023-05-11 00:16:36,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +11: [2023-05-11 00:16:36,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +11: [2023-05-11 00:16:36,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +16: [2023-05-11 00:16:36,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +20: [2023-05-11 00:16:36,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +20: [2023-05-11 00:16:36,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +11: [2023-05-11 00:16:36,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +27: [2023-05-11 00:16:36,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +28: [2023-05-11 00:16:36,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +20: [2023-05-11 00:16:36,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 0: [2023-05-11 00:16:36,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 6: [2023-05-11 00:16:36,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 6: [2023-05-11 00:16:36,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 6: [2023-05-11 00:16:36,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +17: [2023-05-11 00:16:36,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +17: [2023-05-11 00:16:36,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 0: [2023-05-11 00:16:36,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 0: [2023-05-11 00:16:36,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 6: [2023-05-11 00:16:36,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 8: [2023-05-11 00:16:36,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 0: [2023-05-11 00:16:36,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +21: [2023-05-11 00:16:36,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +28: [2023-05-11 00:16:36,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +21: [2023-05-11 00:16:36,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +21: [2023-05-11 00:16:36,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +21: [2023-05-11 00:16:36,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +17: [2023-05-11 00:16:36,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +17: [2023-05-11 00:16:36,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +28: [2023-05-11 00:16:36,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +14: [2023-05-11 00:16:36,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +14: [2023-05-11 00:16:36,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +14: [2023-05-11 00:16:36,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +14: [2023-05-11 00:16:36,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +18: [2023-05-11 00:16:36,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +18: [2023-05-11 00:16:36,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 9: [2023-05-11 00:16:36,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +27: [2023-05-11 00:16:36,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +28: [2023-05-11 00:16:36,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +13: [2023-05-11 00:16:36,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +13: [2023-05-11 00:16:36,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +13: [2023-05-11 00:16:36,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 9: [2023-05-11 00:16:36,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +13: [2023-05-11 00:16:36,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +22: [2023-05-11 00:16:36,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +22: [2023-05-11 00:16:36,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 2: [2023-05-11 00:16:36,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 2: [2023-05-11 00:16:36,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +22: [2023-05-11 00:16:36,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 2: [2023-05-11 00:16:36,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +10: [2023-05-11 00:16:36,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +20: [2023-05-11 00:16:36,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +20: [2023-05-11 00:16:36,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +11: [2023-05-11 00:16:36,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +22: [2023-05-11 00:16:36,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +18: [2023-05-11 00:16:36,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +20: [2023-05-11 00:16:36,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +18: [2023-05-11 00:16:36,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +10: [2023-05-11 00:16:36,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +20: [2023-05-11 00:16:36,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +11: [2023-05-11 00:16:36,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +27: [2023-05-11 00:16:36,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +28: [2023-05-11 00:16:36,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +28: [2023-05-11 00:16:36,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +20: [2023-05-11 00:16:36,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +27: [2023-05-11 00:16:36,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +16: [2023-05-11 00:16:36,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +16: [2023-05-11 00:16:36,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 8: [2023-05-11 00:16:36,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +20: [2023-05-11 00:16:36,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +27: [2023-05-11 00:16:36,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 8: [2023-05-11 00:16:36,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +17: [2023-05-11 00:16:36,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +16: [2023-05-11 00:16:36,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +16: [2023-05-11 00:16:36,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 0: [2023-05-11 00:16:36,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +10: [2023-05-11 00:16:36,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +21: [2023-05-11 00:16:36,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +21: [2023-05-11 00:16:36,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 0: [2023-05-11 00:16:36,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +11: [2023-05-11 00:16:36,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +17: [2023-05-11 00:16:36,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +11: [2023-05-11 00:16:36,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +20: [2023-05-11 00:16:36,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +28: [2023-05-11 00:16:36,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +17: [2023-05-11 00:16:36,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +21: [2023-05-11 00:16:36,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +14: [2023-05-11 00:16:36,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +13: [2023-05-11 00:16:36,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +21: [2023-05-11 00:16:36,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +13: [2023-05-11 00:16:36,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +17: [2023-05-11 00:16:36,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +14: [2023-05-11 00:16:36,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +14: [2023-05-11 00:16:36,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +14: [2023-05-11 00:16:36,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +10: [2023-05-11 00:16:36,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +22: [2023-05-11 00:16:36,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +18: [2023-05-11 00:16:36,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +22: [2023-05-11 00:16:36,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +20: [2023-05-11 00:16:36,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +20: [2023-05-11 00:16:36,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +22: [2023-05-11 00:16:36,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +20: [2023-05-11 00:16:36,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 4: [2023-05-11 00:16:36,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +27: [2023-05-11 00:16:36,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +18: [2023-05-11 00:16:36,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +28: [2023-05-11 00:16:36,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +18: [2023-05-11 00:16:36,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +18: [2023-05-11 00:16:36,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +27: [2023-05-11 00:16:36,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +11: [2023-05-11 00:16:36,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +11: [2023-05-11 00:16:36,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +10: [2023-05-11 00:16:36,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +13: [2023-05-11 00:16:36,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +13: [2023-05-11 00:16:36,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +27: [2023-05-11 00:16:36,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +22: [2023-05-11 00:16:36,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +27: [2023-05-11 00:16:36,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +27: [2023-05-11 00:16:36,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +27: [2023-05-11 00:16:36,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +16: [2023-05-11 00:16:36,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +11: [2023-05-11 00:16:36,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +16: [2023-05-11 00:16:36,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +16: [2023-05-11 00:16:36,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +28: [2023-05-11 00:16:36,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +16: [2023-05-11 00:16:36,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +20: [2023-05-11 00:16:36,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +11: [2023-05-11 00:16:36,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 1: [2023-05-11 00:16:36,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +27: [2023-05-11 00:16:36,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +10: [2023-05-11 00:16:36,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +17: [2023-05-11 00:16:36,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +17: [2023-05-11 00:16:36,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 1: [2023-05-11 00:16:36,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +13: [2023-05-11 00:16:36,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +17: [2023-05-11 00:16:36,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +17: [2023-05-11 00:16:36,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +13: [2023-05-11 00:16:36,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +13: [2023-05-11 00:16:36,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +13: [2023-05-11 00:16:36,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 6: [2023-05-11 00:16:36,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +10: [2023-05-11 00:16:36,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +14: [2023-05-11 00:16:36,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +14: [2023-05-11 00:16:36,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 6: [2023-05-11 00:16:36,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 4: [2023-05-11 00:16:36,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 0: [2023-05-11 00:16:36,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +22: [2023-05-11 00:16:36,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +14: [2023-05-11 00:16:36,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +14: [2023-05-11 00:16:36,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 9: [2023-05-11 00:16:36,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +22: [2023-05-11 00:16:36,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +17: [2023-05-11 00:16:36,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +17: [2023-05-11 00:16:36,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +18: [2023-05-11 00:16:36,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +22: [2023-05-11 00:16:36,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +22: [2023-05-11 00:16:36,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +18: [2023-05-11 00:16:36,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +20: [2023-05-11 00:16:36,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +20: [2023-05-11 00:16:36,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 9: [2023-05-11 00:16:36,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +28: [2023-05-11 00:16:36,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +20: [2023-05-11 00:16:36,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +20: [2023-05-11 00:16:36,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +27: [2023-05-11 00:16:36,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +18: [2023-05-11 00:16:36,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +18: [2023-05-11 00:16:36,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 4: [2023-05-11 00:16:36,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +27: [2023-05-11 00:16:36,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +27: [2023-05-11 00:16:36,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +27: [2023-05-11 00:16:36,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +20: [2023-05-11 00:16:36,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +20: [2023-05-11 00:16:36,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +20: [2023-05-11 00:16:36,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 3: [2023-05-11 00:16:36,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +27: [2023-05-11 00:16:36,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 3: [2023-05-11 00:16:36,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +20: [2023-05-11 00:16:36,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +10: [2023-05-11 00:16:36,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 9: [2023-05-11 00:16:36,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +18: [2023-05-11 00:16:36,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +18: [2023-05-11 00:16:36,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +13: [2023-05-11 00:16:36,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +13: [2023-05-11 00:16:36,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +13: [2023-05-11 00:16:36,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +13: [2023-05-11 00:16:36,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +17: [2023-05-11 00:16:36,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +27: [2023-05-11 00:16:36,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 6: [2023-05-11 00:16:36,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +20: [2023-05-11 00:16:36,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +17: [2023-05-11 00:16:36,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +30: [2023-05-11 00:16:36,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +30: [2023-05-11 00:16:36,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +30: [2023-05-11 00:16:36,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +30: [2023-05-11 00:16:36,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +14: [2023-05-11 00:16:36,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +30: [2023-05-11 00:16:36,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +30: [2023-05-11 00:16:36,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +14: [2023-05-11 00:16:36,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 0: [2023-05-11 00:16:36,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +22: [2023-05-11 00:16:36,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +30: [2023-05-11 00:16:36,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +30: [2023-05-11 00:16:36,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +22: [2023-05-11 00:16:36,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 0: [2023-05-11 00:16:36,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 3: [2023-05-11 00:16:36,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +20: [2023-05-11 00:16:36,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 2: [2023-05-11 00:16:36,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 4: [2023-05-11 00:16:36,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 4: [2023-05-11 00:16:36,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +14: [2023-05-11 00:16:36,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +18: [2023-05-11 00:16:36,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +27: [2023-05-11 00:16:36,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +14: [2023-05-11 00:16:36,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +18: [2023-05-11 00:16:36,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +20: [2023-05-11 00:16:36,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +22: [2023-05-11 00:16:36,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +22: [2023-05-11 00:16:36,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +12: [2023-05-11 00:16:36,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:36,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:36,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +12: [2023-05-11 00:16:36,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +30: [2023-05-11 00:16:36,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +30: [2023-05-11 00:16:36,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:36,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:36,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:36,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +27: [2023-05-11 00:16:36,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +27: [2023-05-11 00:16:36,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 1: [2023-05-11 00:16:36,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +20: [2023-05-11 00:16:36,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +12: [2023-05-11 00:16:36,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +27: [2023-05-11 00:16:36,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +27: [2023-05-11 00:16:36,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +27: [2023-05-11 00:16:36,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +27: [2023-05-11 00:16:36,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 1: [2023-05-11 00:16:36,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +30: [2023-05-11 00:16:36,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +30: [2023-05-11 00:16:36,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 3: [2023-05-11 00:16:36,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... + 2: [2023-05-11 00:16:36,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +20: [2023-05-11 00:16:36,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +30: [2023-05-11 00:16:36,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +25: [2023-05-11 00:16:36,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +25: [2023-05-11 00:16:36,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +18: [2023-05-11 00:16:36,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +18: [2023-05-11 00:16:36,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +18: [2023-05-11 00:16:36,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +18: [2023-05-11 00:16:36,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +18: [2023-05-11 00:16:36,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +18: [2023-05-11 00:16:36,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +18: [2023-05-11 00:16:36,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +25: [2023-05-11 00:16:36,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +12: [2023-05-11 00:16:36,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +25: [2023-05-11 00:16:36,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +16: [2023-05-11 00:16:36,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +16: [2023-05-11 00:16:36,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +18: [2023-05-11 00:16:36,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +16: [2023-05-11 00:16:36,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +16: [2023-05-11 00:16:36,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +30: [2023-05-11 00:16:36,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:36,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +16: [2023-05-11 00:16:36,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +16: [2023-05-11 00:16:36,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +16: [2023-05-11 00:16:36,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +30: [2023-05-11 00:16:36,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +16: [2023-05-11 00:16:36,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +11: [2023-05-11 00:16:36,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +11: [2023-05-11 00:16:36,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +12: [2023-05-11 00:16:36,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +11: [2023-05-11 00:16:36,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +11: [2023-05-11 00:16:36,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +11: [2023-05-11 00:16:36,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +11: [2023-05-11 00:16:36,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +25: [2023-05-11 00:16:36,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +11: [2023-05-11 00:16:36,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +25: [2023-05-11 00:16:36,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +11: [2023-05-11 00:16:36,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +30: [2023-05-11 00:16:36,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +12: [2023-05-11 00:16:36,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +25: [2023-05-11 00:16:36,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +25: [2023-05-11 00:16:36,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +12: [2023-05-11 00:16:36,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +18: [2023-05-11 00:16:36,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +18: [2023-05-11 00:16:36,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +18: [2023-05-11 00:16:36,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:36,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +16: [2023-05-11 00:16:36,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:36,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +31: [2023-05-11 00:16:36,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +31: [2023-05-11 00:16:36,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. +16: [2023-05-11 00:16:36,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +16: [2023-05-11 00:16:36,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +11: [2023-05-11 00:16:36,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +11: [2023-05-11 00:16:36,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +25: [2023-05-11 00:16:36,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +31: [2023-05-11 00:16:36,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 6: [2023-05-11 00:16:36,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +25: [2023-05-11 00:16:36,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +31: [2023-05-11 00:16:36,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt. + 6: [2023-05-11 00:16:36,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +25: [2023-05-11 00:16:36,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +25: [2023-05-11 00:16:36,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +18: [2023-05-11 00:16:36,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +18: [2023-05-11 00:16:36,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +23: [2023-05-11 00:16:36,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +23: [2023-05-11 00:16:36,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +23: [2023-05-11 00:16:36,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +23: [2023-05-11 00:16:36,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +23: [2023-05-11 00:16:36,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +23: [2023-05-11 00:16:36,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +23: [2023-05-11 00:16:36,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:36,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +29: [2023-05-11 00:16:36,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +29: [2023-05-11 00:16:36,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +29: [2023-05-11 00:16:36,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +29: [2023-05-11 00:16:36,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +29: [2023-05-11 00:16:36,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +16: [2023-05-11 00:16:36,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +29: [2023-05-11 00:16:36,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +29: [2023-05-11 00:16:36,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +25: [2023-05-11 00:16:36,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +25: [2023-05-11 00:16:36,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +25: [2023-05-11 00:16:36,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +25: [2023-05-11 00:16:36,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +23: [2023-05-11 00:16:36,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +18: [2023-05-11 00:16:36,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +31: [2023-05-11 00:16:36,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +11: [2023-05-11 00:16:36,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +31: [2023-05-11 00:16:36,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +31: [2023-05-11 00:16:36,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +16: [2023-05-11 00:16:36,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +29: [2023-05-11 00:16:36,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +11: [2023-05-11 00:16:36,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +18: [2023-05-11 00:16:36,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +31: [2023-05-11 00:16:36,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt... +16: [2023-05-11 00:16:36,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +11: [2023-05-11 00:16:36,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +11: [2023-05-11 00:16:36,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +24: [2023-05-11 00:16:36,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +24: [2023-05-11 00:16:36,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +16: [2023-05-11 00:16:36,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +24: [2023-05-11 00:16:36,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +24: [2023-05-11 00:16:36,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +23: [2023-05-11 00:16:36,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +23: [2023-05-11 00:16:36,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +23: [2023-05-11 00:16:36,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +23: [2023-05-11 00:16:36,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +24: [2023-05-11 00:16:36,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +24: [2023-05-11 00:16:36,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +29: [2023-05-11 00:16:36,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +18: [2023-05-11 00:16:36,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +24: [2023-05-11 00:16:36,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +24: [2023-05-11 00:16:36,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +23: [2023-05-11 00:16:36,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +23: [2023-05-11 00:16:36,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:36,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +12: [2023-05-11 00:16:36,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +12: [2023-05-11 00:16:36,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +23: [2023-05-11 00:16:36,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +12: [2023-05-11 00:16:36,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +29: [2023-05-11 00:16:36,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +29: [2023-05-11 00:16:36,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +11: [2023-05-11 00:16:36,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +11: [2023-05-11 00:16:36,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +16: [2023-05-11 00:16:36,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +31: [2023-05-11 00:16:36,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +31: [2023-05-11 00:16:36,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +23: [2023-05-11 00:16:36,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +23: [2023-05-11 00:16:36,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +31: [2023-05-11 00:16:36,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +31: [2023-05-11 00:16:36,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +31: [2023-05-11 00:16:36,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +31: [2023-05-11 00:16:36,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +23: [2023-05-11 00:16:36,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +31: [2023-05-11 00:16:36,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt. +31: [2023-05-11 00:16:36,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt... +29: [2023-05-11 00:16:36,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +12: [2023-05-11 00:16:36,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +17: [2023-05-11 00:16:36,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +17: [2023-05-11 00:16:36,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +29: [2023-05-11 00:16:36,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +23: [2023-05-11 00:16:36,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +23: [2023-05-11 00:16:36,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 7: [2023-05-11 00:16:36,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 7: [2023-05-11 00:16:36,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +12: [2023-05-11 00:16:36,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +17: [2023-05-11 00:16:36,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:36,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +12: [2023-05-11 00:16:36,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +29: [2023-05-11 00:16:36,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +17: [2023-05-11 00:16:36,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +17: [2023-05-11 00:16:36,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +17: [2023-05-11 00:16:36,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +17: [2023-05-11 00:16:36,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +23: [2023-05-11 00:16:36,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +23: [2023-05-11 00:16:36,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +17: [2023-05-11 00:16:36,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +24: [2023-05-11 00:16:36,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +24: [2023-05-11 00:16:36,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +29: [2023-05-11 00:16:36,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +29: [2023-05-11 00:16:36,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +29: [2023-05-11 00:16:36,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +31: [2023-05-11 00:16:36,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +31: [2023-05-11 00:16:36,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +31: [2023-05-11 00:16:36,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +31: [2023-05-11 00:16:36,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +31: [2023-05-11 00:16:36,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +31: [2023-05-11 00:16:36,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +10: [2023-05-11 00:16:36,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +10: [2023-05-11 00:16:36,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +10: [2023-05-11 00:16:36,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +31: [2023-05-11 00:16:36,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +10: [2023-05-11 00:16:36,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +29: [2023-05-11 00:16:36,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +22: [2023-05-11 00:16:36,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +22: [2023-05-11 00:16:36,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +22: [2023-05-11 00:16:36,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +22: [2023-05-11 00:16:36,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +22: [2023-05-11 00:16:36,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +22: [2023-05-11 00:16:36,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +22: [2023-05-11 00:16:36,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +22: [2023-05-11 00:16:36,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +31: [2023-05-11 00:16:36,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +23: [2023-05-11 00:16:36,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +12: [2023-05-11 00:16:36,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:36,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +24: [2023-05-11 00:16:36,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +24: [2023-05-11 00:16:36,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +23: [2023-05-11 00:16:36,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +17: [2023-05-11 00:16:36,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +23: [2023-05-11 00:16:36,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +29: [2023-05-11 00:16:36,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +29: [2023-05-11 00:16:36,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +23: [2023-05-11 00:16:36,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +23: [2023-05-11 00:16:36,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +29: [2023-05-11 00:16:36,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +29: [2023-05-11 00:16:36,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +29: [2023-05-11 00:16:36,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +24: [2023-05-11 00:16:36,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +17: [2023-05-11 00:16:36,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +15: [2023-05-11 00:16:36,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +15: [2023-05-11 00:16:36,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +15: [2023-05-11 00:16:36,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +15: [2023-05-11 00:16:36,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +15: [2023-05-11 00:16:36,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +15: [2023-05-11 00:16:36,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +25: [2023-05-11 00:16:36,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +25: [2023-05-11 00:16:36,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +25: [2023-05-11 00:16:36,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +15: [2023-05-11 00:16:36,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +10: [2023-05-11 00:16:36,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +10: [2023-05-11 00:16:36,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +25: [2023-05-11 00:16:36,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +10: [2023-05-11 00:16:36,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +10: [2023-05-11 00:16:36,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +23: [2023-05-11 00:16:36,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +23: [2023-05-11 00:16:36,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +23: [2023-05-11 00:16:36,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +12: [2023-05-11 00:16:36,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:36,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:36,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +12: [2023-05-11 00:16:36,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +15: [2023-05-11 00:16:36,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +22: [2023-05-11 00:16:36,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +22: [2023-05-11 00:16:36,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +25: [2023-05-11 00:16:36,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +25: [2023-05-11 00:16:36,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +25: [2023-05-11 00:16:36,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +31: [2023-05-11 00:16:36,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +31: [2023-05-11 00:16:36,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 4: [2023-05-11 00:16:36,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 4: [2023-05-11 00:16:36,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +25: [2023-05-11 00:16:36,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +24: [2023-05-11 00:16:36,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +22: [2023-05-11 00:16:36,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +22: [2023-05-11 00:16:36,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +17: [2023-05-11 00:16:36,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +12: [2023-05-11 00:16:36,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +23: [2023-05-11 00:16:36,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:36,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 7: [2023-05-11 00:16:36,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +23: [2023-05-11 00:16:36,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 7: [2023-05-11 00:16:36,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +30: [2023-05-11 00:16:36,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +30: [2023-05-11 00:16:36,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +30: [2023-05-11 00:16:36,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +31: [2023-05-11 00:16:36,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +30: [2023-05-11 00:16:36,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +29: [2023-05-11 00:16:36,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +17: [2023-05-11 00:16:36,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +24: [2023-05-11 00:16:36,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +17: [2023-05-11 00:16:36,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +15: [2023-05-11 00:16:36,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +15: [2023-05-11 00:16:36,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +29: [2023-05-11 00:16:36,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +31: [2023-05-11 00:16:36,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +29: [2023-05-11 00:16:36,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +31: [2023-05-11 00:16:36,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +25: [2023-05-11 00:16:36,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +22: [2023-05-11 00:16:36,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +25: [2023-05-11 00:16:36,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +25: [2023-05-11 00:16:36,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +17: [2023-05-11 00:16:36,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +29: [2023-05-11 00:16:36,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +29: [2023-05-11 00:16:36,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 4: [2023-05-11 00:16:36,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +22: [2023-05-11 00:16:36,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +22: [2023-05-11 00:16:36,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +22: [2023-05-11 00:16:36,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 9: [2023-05-11 00:16:36,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +10: [2023-05-11 00:16:36,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +31: [2023-05-11 00:16:36,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +10: [2023-05-11 00:16:36,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +29: [2023-05-11 00:16:36,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +29: [2023-05-11 00:16:36,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +15: [2023-05-11 00:16:36,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +17: [2023-05-11 00:16:36,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +15: [2023-05-11 00:16:36,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 8: [2023-05-11 00:16:36,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 8: [2023-05-11 00:16:36,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 9: [2023-05-11 00:16:36,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 8: [2023-05-11 00:16:36,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 8: [2023-05-11 00:16:36,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +24: [2023-05-11 00:16:36,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +15: [2023-05-11 00:16:36,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +31: [2023-05-11 00:16:36,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +25: [2023-05-11 00:16:36,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +30: [2023-05-11 00:16:36,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +30: [2023-05-11 00:16:36,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +30: [2023-05-11 00:16:36,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +30: [2023-05-11 00:16:36,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +25: [2023-05-11 00:16:36,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +26: [2023-05-11 00:16:36,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +26: [2023-05-11 00:16:36,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +26: [2023-05-11 00:16:36,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +26: [2023-05-11 00:16:36,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +26: [2023-05-11 00:16:36,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +26: [2023-05-11 00:16:36,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +26: [2023-05-11 00:16:36,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +26: [2023-05-11 00:16:36,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +17: [2023-05-11 00:16:36,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +24: [2023-05-11 00:16:36,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +24: [2023-05-11 00:16:36,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +24: [2023-05-11 00:16:36,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +24: [2023-05-11 00:16:36,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +10: [2023-05-11 00:16:36,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +10: [2023-05-11 00:16:36,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +10: [2023-05-11 00:16:36,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +10: [2023-05-11 00:16:36,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +25: [2023-05-11 00:16:36,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +10: [2023-05-11 00:16:36,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +29: [2023-05-11 00:16:36,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +10: [2023-05-11 00:16:36,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +29: [2023-05-11 00:16:36,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 9: [2023-05-11 00:16:36,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +15: [2023-05-11 00:16:36,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +31: [2023-05-11 00:16:36,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +15: [2023-05-11 00:16:36,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +15: [2023-05-11 00:16:36,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +15: [2023-05-11 00:16:36,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +15: [2023-05-11 00:16:36,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +15: [2023-05-11 00:16:36,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +25: [2023-05-11 00:16:36,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +24: [2023-05-11 00:16:36,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +24: [2023-05-11 00:16:36,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +24: [2023-05-11 00:16:36,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +24: [2023-05-11 00:16:36,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 4: [2023-05-11 00:16:36,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 7: [2023-05-11 00:16:36,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +28: [2023-05-11 00:16:36,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +28: [2023-05-11 00:16:36,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +15: [2023-05-11 00:16:36,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +28: [2023-05-11 00:16:36,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 7: [2023-05-11 00:16:36,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +28: [2023-05-11 00:16:36,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 7: [2023-05-11 00:16:36,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +25: [2023-05-11 00:16:36,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +30: [2023-05-11 00:16:36,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +30: [2023-05-11 00:16:36,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +30: [2023-05-11 00:16:36,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +30: [2023-05-11 00:16:36,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +15: [2023-05-11 00:16:36,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +15: [2023-05-11 00:16:36,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +24: [2023-05-11 00:16:36,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +26: [2023-05-11 00:16:36,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +26: [2023-05-11 00:16:36,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +24: [2023-05-11 00:16:36,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +26: [2023-05-11 00:16:36,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +24: [2023-05-11 00:16:36,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +24: [2023-05-11 00:16:36,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +15: [2023-05-11 00:16:36,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +14: [2023-05-11 00:16:36,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +14: [2023-05-11 00:16:36,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +15: [2023-05-11 00:16:36,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +14: [2023-05-11 00:16:36,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +14: [2023-05-11 00:16:36,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +14: [2023-05-11 00:16:36,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +14: [2023-05-11 00:16:36,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +14: [2023-05-11 00:16:36,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +14: [2023-05-11 00:16:36,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +28: [2023-05-11 00:16:36,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +24: [2023-05-11 00:16:36,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +24: [2023-05-11 00:16:36,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +24: [2023-05-11 00:16:36,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +24: [2023-05-11 00:16:36,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 4: [2023-05-11 00:16:36,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +30: [2023-05-11 00:16:36,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +30: [2023-05-11 00:16:36,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +28: [2023-05-11 00:16:36,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +28: [2023-05-11 00:16:36,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +28: [2023-05-11 00:16:36,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +26: [2023-05-11 00:16:36,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 5: [2023-05-11 00:16:36,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 5: [2023-05-11 00:16:36,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 5: [2023-05-11 00:16:36,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +26: [2023-05-11 00:16:36,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +26: [2023-05-11 00:16:36,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +26: [2023-05-11 00:16:36,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +30: [2023-05-11 00:16:36,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +15: [2023-05-11 00:16:36,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +30: [2023-05-11 00:16:36,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +14: [2023-05-11 00:16:36,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +13: [2023-05-11 00:16:36,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +13: [2023-05-11 00:16:36,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +15: [2023-05-11 00:16:36,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +28: [2023-05-11 00:16:36,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +28: [2023-05-11 00:16:36,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +15: [2023-05-11 00:16:36,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +28: [2023-05-11 00:16:36,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +15: [2023-05-11 00:16:36,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +28: [2023-05-11 00:16:36,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +13: [2023-05-11 00:16:36,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +13: [2023-05-11 00:16:36,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +26: [2023-05-11 00:16:36,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +13: [2023-05-11 00:16:36,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +13: [2023-05-11 00:16:36,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +21: [2023-05-11 00:16:36,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +21: [2023-05-11 00:16:36,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +21: [2023-05-11 00:16:36,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +21: [2023-05-11 00:16:36,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +13: [2023-05-11 00:16:36,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +21: [2023-05-11 00:16:36,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +14: [2023-05-11 00:16:36,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +14: [2023-05-11 00:16:36,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +21: [2023-05-11 00:16:36,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +13: [2023-05-11 00:16:36,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +28: [2023-05-11 00:16:36,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +28: [2023-05-11 00:16:36,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +21: [2023-05-11 00:16:36,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +28: [2023-05-11 00:16:36,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +28: [2023-05-11 00:16:36,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +21: [2023-05-11 00:16:36,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +14: [2023-05-11 00:16:36,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +15: [2023-05-11 00:16:36,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +13: [2023-05-11 00:16:36,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +15: [2023-05-11 00:16:36,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +15: [2023-05-11 00:16:36,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +14: [2023-05-11 00:16:36,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +15: [2023-05-11 00:16:36,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 7: [2023-05-11 00:16:36,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +19: [2023-05-11 00:16:36,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +19: [2023-05-11 00:16:36,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +14: [2023-05-11 00:16:36,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +14: [2023-05-11 00:16:36,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +13: [2023-05-11 00:16:36,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +19: [2023-05-11 00:16:36,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +13: [2023-05-11 00:16:36,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +19: [2023-05-11 00:16:36,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 0: [2023-05-11 00:16:36,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +14: [2023-05-11 00:16:36,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +21: [2023-05-11 00:16:36,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +21: [2023-05-11 00:16:36,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +19: [2023-05-11 00:16:36,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +13: [2023-05-11 00:16:36,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +19: [2023-05-11 00:16:36,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 5: [2023-05-11 00:16:36,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 5: [2023-05-11 00:16:36,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +19: [2023-05-11 00:16:36,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +19: [2023-05-11 00:16:36,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +13: [2023-05-11 00:16:36,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +21: [2023-05-11 00:16:36,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +13: [2023-05-11 00:16:36,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +21: [2023-05-11 00:16:36,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +21: [2023-05-11 00:16:36,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +28: [2023-05-11 00:16:36,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +28: [2023-05-11 00:16:36,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +28: [2023-05-11 00:16:36,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +28: [2023-05-11 00:16:36,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +21: [2023-05-11 00:16:36,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +28: [2023-05-11 00:16:36,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +28: [2023-05-11 00:16:36,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +28: [2023-05-11 00:16:36,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +28: [2023-05-11 00:16:36,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +21: [2023-05-11 00:16:36,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +13: [2023-05-11 00:16:36,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +13: [2023-05-11 00:16:36,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +11: [2023-05-11 00:16:36,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +11: [2023-05-11 00:16:36,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +11: [2023-05-11 00:16:36,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +11: [2023-05-11 00:16:36,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +21: [2023-05-11 00:16:36,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 5: [2023-05-11 00:16:36,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +11: [2023-05-11 00:16:36,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 9: [2023-05-11 00:16:36,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +19: [2023-05-11 00:16:36,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +19: [2023-05-11 00:16:36,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +11: [2023-05-11 00:16:36,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +11: [2023-05-11 00:16:36,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +11: [2023-05-11 00:16:36,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +19: [2023-05-11 00:16:36,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +19: [2023-05-11 00:16:36,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +19: [2023-05-11 00:16:36,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +19: [2023-05-11 00:16:36,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +19: [2023-05-11 00:16:36,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +19: [2023-05-11 00:16:36,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +19: [2023-05-11 00:16:36,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +19: [2023-05-11 00:16:36,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +19: [2023-05-11 00:16:36,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +19: [2023-05-11 00:16:36,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +19: [2023-05-11 00:16:36,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +19: [2023-05-11 00:16:36,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +19: [2023-05-11 00:16:36,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +19: [2023-05-11 00:16:36,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +11: [2023-05-11 00:16:36,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +11: [2023-05-11 00:16:36,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +10: [2023-05-11 00:16:36,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +10: [2023-05-11 00:16:36,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +10: [2023-05-11 00:16:36,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +10: [2023-05-11 00:16:36,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +10: [2023-05-11 00:16:36,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +10: [2023-05-11 00:16:36,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +10: [2023-05-11 00:16:36,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +11: [2023-05-11 00:16:36,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +11: [2023-05-11 00:16:36,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +10: [2023-05-11 00:16:36,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +11: [2023-05-11 00:16:36,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +11: [2023-05-11 00:16:36,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 9: [2023-05-11 00:16:36,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 6: [2023-05-11 00:16:36,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +19: [2023-05-11 00:16:36,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 5: [2023-05-11 00:16:36,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +19: [2023-05-11 00:16:36,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +28: [2023-05-11 00:16:36,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +28: [2023-05-11 00:16:36,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +28: [2023-05-11 00:16:36,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +28: [2023-05-11 00:16:36,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +28: [2023-05-11 00:16:36,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +28: [2023-05-11 00:16:36,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +28: [2023-05-11 00:16:36,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +28: [2023-05-11 00:16:36,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +10: [2023-05-11 00:16:36,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +10: [2023-05-11 00:16:36,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +10: [2023-05-11 00:16:36,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +11: [2023-05-11 00:16:36,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +11: [2023-05-11 00:16:36,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 6: [2023-05-11 00:16:36,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +19: [2023-05-11 00:16:36,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +19: [2023-05-11 00:16:36,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +27: [2023-05-11 00:16:36,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +27: [2023-05-11 00:16:36,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +27: [2023-05-11 00:16:36,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +27: [2023-05-11 00:16:36,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +27: [2023-05-11 00:16:36,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +27: [2023-05-11 00:16:36,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +27: [2023-05-11 00:16:36,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +19: [2023-05-11 00:16:36,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +27: [2023-05-11 00:16:36,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +10: [2023-05-11 00:16:36,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +10: [2023-05-11 00:16:36,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +10: [2023-05-11 00:16:36,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +19: [2023-05-11 00:16:36,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +30: [2023-05-11 00:16:36,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +30: [2023-05-11 00:16:36,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +30: [2023-05-11 00:16:36,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +30: [2023-05-11 00:16:36,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +30: [2023-05-11 00:16:36,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +30: [2023-05-11 00:16:36,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +30: [2023-05-11 00:16:36,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +30: [2023-05-11 00:16:36,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +19: [2023-05-11 00:16:36,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +27: [2023-05-11 00:16:36,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +27: [2023-05-11 00:16:36,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +20: [2023-05-11 00:16:36,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +27: [2023-05-11 00:16:36,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +20: [2023-05-11 00:16:36,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +10: [2023-05-11 00:16:36,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +20: [2023-05-11 00:16:36,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +20: [2023-05-11 00:16:36,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +20: [2023-05-11 00:16:36,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +20: [2023-05-11 00:16:36,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +20: [2023-05-11 00:16:36,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +20: [2023-05-11 00:16:36,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +19: [2023-05-11 00:16:36,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +14: [2023-05-11 00:16:36,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +14: [2023-05-11 00:16:36,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 6: [2023-05-11 00:16:36,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 6: [2023-05-11 00:16:36,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +27: [2023-05-11 00:16:36,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +30: [2023-05-11 00:16:36,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +14: [2023-05-11 00:16:36,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +10: [2023-05-11 00:16:36,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +27: [2023-05-11 00:16:36,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +14: [2023-05-11 00:16:36,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 6: [2023-05-11 00:16:36,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +22: [2023-05-11 00:16:36,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +22: [2023-05-11 00:16:36,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +20: [2023-05-11 00:16:36,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +20: [2023-05-11 00:16:36,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +20: [2023-05-11 00:16:36,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +20: [2023-05-11 00:16:36,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +27: [2023-05-11 00:16:36,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +27: [2023-05-11 00:16:36,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 6: [2023-05-11 00:16:36,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +30: [2023-05-11 00:16:36,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +30: [2023-05-11 00:16:36,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +27: [2023-05-11 00:16:36,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +22: [2023-05-11 00:16:36,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 6: [2023-05-11 00:16:36,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +30: [2023-05-11 00:16:36,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +27: [2023-05-11 00:16:36,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +22: [2023-05-11 00:16:36,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +27: [2023-05-11 00:16:36,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +27: [2023-05-11 00:16:36,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +20: [2023-05-11 00:16:36,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +14: [2023-05-11 00:16:36,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +17: [2023-05-11 00:16:36,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +17: [2023-05-11 00:16:36,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +17: [2023-05-11 00:16:36,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +30: [2023-05-11 00:16:36,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +14: [2023-05-11 00:16:36,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +14: [2023-05-11 00:16:36,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +17: [2023-05-11 00:16:36,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +14: [2023-05-11 00:16:36,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +27: [2023-05-11 00:16:36,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +30: [2023-05-11 00:16:36,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +20: [2023-05-11 00:16:36,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +20: [2023-05-11 00:16:36,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +27: [2023-05-11 00:16:36,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +27: [2023-05-11 00:16:36,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +20: [2023-05-11 00:16:36,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +20: [2023-05-11 00:16:36,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +30: [2023-05-11 00:16:36,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +27: [2023-05-11 00:16:36,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +27: [2023-05-11 00:16:36,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +22: [2023-05-11 00:16:36,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +22: [2023-05-11 00:16:36,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +22: [2023-05-11 00:16:36,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +22: [2023-05-11 00:16:36,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +30: [2023-05-11 00:16:36,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +20: [2023-05-11 00:16:36,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +20: [2023-05-11 00:16:36,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +12: [2023-05-11 00:16:36,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +12: [2023-05-11 00:16:36,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +12: [2023-05-11 00:16:36,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +12: [2023-05-11 00:16:36,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +12: [2023-05-11 00:16:36,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +12: [2023-05-11 00:16:36,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +12: [2023-05-11 00:16:36,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +17: [2023-05-11 00:16:36,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +17: [2023-05-11 00:16:36,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +14: [2023-05-11 00:16:36,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +14: [2023-05-11 00:16:36,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +20: [2023-05-11 00:16:36,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +14: [2023-05-11 00:16:36,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +14: [2023-05-11 00:16:36,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 6: [2023-05-11 00:16:36,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +12: [2023-05-11 00:16:36,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +20: [2023-05-11 00:16:36,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +17: [2023-05-11 00:16:36,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +17: [2023-05-11 00:16:36,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +18: [2023-05-11 00:16:36,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +18: [2023-05-11 00:16:36,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +18: [2023-05-11 00:16:36,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 2: [2023-05-11 00:16:36,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 2: [2023-05-11 00:16:36,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 2: [2023-05-11 00:16:36,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 2: [2023-05-11 00:16:36,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +14: [2023-05-11 00:16:36,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +14: [2023-05-11 00:16:36,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +21: [2023-05-11 00:16:36,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +18: [2023-05-11 00:16:36,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +27: [2023-05-11 00:16:36,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +21: [2023-05-11 00:16:36,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +22: [2023-05-11 00:16:36,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +22: [2023-05-11 00:16:36,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +27: [2023-05-11 00:16:36,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 3: [2023-05-11 00:16:36,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 3: [2023-05-11 00:16:36,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 3: [2023-05-11 00:16:36,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 3: [2023-05-11 00:16:36,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +22: [2023-05-11 00:16:36,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +22: [2023-05-11 00:16:36,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +22: [2023-05-11 00:16:36,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +21: [2023-05-11 00:16:36,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +22: [2023-05-11 00:16:36,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +21: [2023-05-11 00:16:36,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 6: [2023-05-11 00:16:36,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 0: [2023-05-11 00:16:36,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +13: [2023-05-11 00:16:36,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +26: [2023-05-11 00:16:36,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +26: [2023-05-11 00:16:36,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +13: [2023-05-11 00:16:36,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +13: [2023-05-11 00:16:36,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +20: [2023-05-11 00:16:36,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +13: [2023-05-11 00:16:36,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +20: [2023-05-11 00:16:36,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +20: [2023-05-11 00:16:36,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 1: [2023-05-11 00:16:36,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +31: [2023-05-11 00:16:36,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +26: [2023-05-11 00:16:36,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 1: [2023-05-11 00:16:36,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +26: [2023-05-11 00:16:36,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 1: [2023-05-11 00:16:36,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +17: [2023-05-11 00:16:36,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +17: [2023-05-11 00:16:36,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +12: [2023-05-11 00:16:36,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +12: [2023-05-11 00:16:36,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +12: [2023-05-11 00:16:36,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +11: [2023-05-11 00:16:36,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +11: [2023-05-11 00:16:36,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +11: [2023-05-11 00:16:36,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +11: [2023-05-11 00:16:36,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +18: [2023-05-11 00:16:36,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +18: [2023-05-11 00:16:36,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +24: [2023-05-11 00:16:36,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +24: [2023-05-11 00:16:36,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +31: [2023-05-11 00:16:36,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +18: [2023-05-11 00:16:36,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +12: [2023-05-11 00:16:36,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +18: [2023-05-11 00:16:36,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +27: [2023-05-11 00:16:36,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +27: [2023-05-11 00:16:36,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 1: [2023-05-11 00:16:36,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 1: [2023-05-11 00:16:36,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +24: [2023-05-11 00:16:36,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +24: [2023-05-11 00:16:36,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +24: [2023-05-11 00:16:36,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +24: [2023-05-11 00:16:36,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +31: [2023-05-11 00:16:36,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 1: [2023-05-11 00:16:36,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +27: [2023-05-11 00:16:36,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +27: [2023-05-11 00:16:36,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +21: [2023-05-11 00:16:36,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +31: [2023-05-11 00:16:36,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +18: [2023-05-11 00:16:36,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 3: [2023-05-11 00:16:36,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +18: [2023-05-11 00:16:36,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +11: [2023-05-11 00:16:36,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +11: [2023-05-11 00:16:36,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +11: [2023-05-11 00:16:36,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +11: [2023-05-11 00:16:36,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +18: [2023-05-11 00:16:36,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +24: [2023-05-11 00:16:36,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +17: [2023-05-11 00:16:36,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +17: [2023-05-11 00:16:36,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 3: [2023-05-11 00:16:36,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +22: [2023-05-11 00:16:36,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +14: [2023-05-11 00:16:36,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +16: [2023-05-11 00:16:36,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +26: [2023-05-11 00:16:36,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +12: [2023-05-11 00:16:36,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +21: [2023-05-11 00:16:36,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +16: [2023-05-11 00:16:36,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +16: [2023-05-11 00:16:36,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +16: [2023-05-11 00:16:36,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +18: [2023-05-11 00:16:36,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +14: [2023-05-11 00:16:36,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 3: [2023-05-11 00:16:36,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +24: [2023-05-11 00:16:36,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +18: [2023-05-11 00:16:36,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +18: [2023-05-11 00:16:36,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +18: [2023-05-11 00:16:36,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +16: [2023-05-11 00:16:36,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +16: [2023-05-11 00:16:36,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +16: [2023-05-11 00:16:36,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +22: [2023-05-11 00:16:36,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 0: [2023-05-11 00:16:36,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +27: [2023-05-11 00:16:36,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +27: [2023-05-11 00:16:36,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 8: [2023-05-11 00:16:36,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +21: [2023-05-11 00:16:36,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +20: [2023-05-11 00:16:36,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +20: [2023-05-11 00:16:36,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +20: [2023-05-11 00:16:36,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +20: [2023-05-11 00:16:36,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +13: [2023-05-11 00:16:36,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +21: [2023-05-11 00:16:36,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +25: [2023-05-11 00:16:36,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +25: [2023-05-11 00:16:36,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +25: [2023-05-11 00:16:36,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +25: [2023-05-11 00:16:36,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +25: [2023-05-11 00:16:36,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +25: [2023-05-11 00:16:36,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +25: [2023-05-11 00:16:36,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +25: [2023-05-11 00:16:36,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +13: [2023-05-11 00:16:36,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +13: [2023-05-11 00:16:36,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +16: [2023-05-11 00:16:36,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +13: [2023-05-11 00:16:36,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +20: [2023-05-11 00:16:36,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +29: [2023-05-11 00:16:36,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +29: [2023-05-11 00:16:36,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +29: [2023-05-11 00:16:36,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +12: [2023-05-11 00:16:36,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +29: [2023-05-11 00:16:36,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +18: [2023-05-11 00:16:36,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +26: [2023-05-11 00:16:36,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +31: [2023-05-11 00:16:36,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +12: [2023-05-11 00:16:36,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +12: [2023-05-11 00:16:36,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +17: [2023-05-11 00:16:36,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +17: [2023-05-11 00:16:36,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 1: [2023-05-11 00:16:36,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +12: [2023-05-11 00:16:36,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +12: [2023-05-11 00:16:36,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +29: [2023-05-11 00:16:36,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +24: [2023-05-11 00:16:36,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +29: [2023-05-11 00:16:36,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +29: [2023-05-11 00:16:36,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +26: [2023-05-11 00:16:36,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +26: [2023-05-11 00:16:36,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +31: [2023-05-11 00:16:36,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +29: [2023-05-11 00:16:36,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +20: [2023-05-11 00:16:36,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +31: [2023-05-11 00:16:36,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +20: [2023-05-11 00:16:36,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 4: [2023-05-11 00:16:36,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +31: [2023-05-11 00:16:36,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +12: [2023-05-11 00:16:36,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +17: [2023-05-11 00:16:36,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +17: [2023-05-11 00:16:36,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +17: [2023-05-11 00:16:36,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +17: [2023-05-11 00:16:36,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +26: [2023-05-11 00:16:36,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +24: [2023-05-11 00:16:36,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +29: [2023-05-11 00:16:36,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +29: [2023-05-11 00:16:36,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +17: [2023-05-11 00:16:36,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +17: [2023-05-11 00:16:36,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +17: [2023-05-11 00:16:36,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +17: [2023-05-11 00:16:36,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:36,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +21: [2023-05-11 00:16:36,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +16: [2023-05-11 00:16:36,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +16: [2023-05-11 00:16:36,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +16: [2023-05-11 00:16:36,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +17: [2023-05-11 00:16:36,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +12: [2023-05-11 00:16:36,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +25: [2023-05-11 00:16:36,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +21: [2023-05-11 00:16:36,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +12: [2023-05-11 00:16:36,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 3: [2023-05-11 00:16:36,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 3: [2023-05-11 00:16:36,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +26: [2023-05-11 00:16:36,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +17: [2023-05-11 00:16:36,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 3: [2023-05-11 00:16:36,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +20: [2023-05-11 00:16:36,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +26: [2023-05-11 00:16:36,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +26: [2023-05-11 00:16:36,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +13: [2023-05-11 00:16:36,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +21: [2023-05-11 00:16:36,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +21: [2023-05-11 00:16:36,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +29: [2023-05-11 00:16:36,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +25: [2023-05-11 00:16:36,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +21: [2023-05-11 00:16:36,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +13: [2023-05-11 00:16:36,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +15: [2023-05-11 00:16:36,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +15: [2023-05-11 00:16:36,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +15: [2023-05-11 00:16:36,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +15: [2023-05-11 00:16:36,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +29: [2023-05-11 00:16:36,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +16: [2023-05-11 00:16:36,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +21: [2023-05-11 00:16:36,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +24: [2023-05-11 00:16:36,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 8: [2023-05-11 00:16:36,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 8: [2023-05-11 00:16:36,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +13: [2023-05-11 00:16:36,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +13: [2023-05-11 00:16:36,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +31: [2023-05-11 00:16:36,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +31: [2023-05-11 00:16:36,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 8: [2023-05-11 00:16:36,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +15: [2023-05-11 00:16:36,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +15: [2023-05-11 00:16:36,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +25: [2023-05-11 00:16:36,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +15: [2023-05-11 00:16:36,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +18: [2023-05-11 00:16:36,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +15: [2023-05-11 00:16:36,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +18: [2023-05-11 00:16:36,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +25: [2023-05-11 00:16:36,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. + 8: [2023-05-11 00:16:36,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +25: [2023-05-11 00:16:36,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +25: [2023-05-11 00:16:36,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +24: [2023-05-11 00:16:36,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +18: [2023-05-11 00:16:36,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +25: [2023-05-11 00:16:36,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +18: [2023-05-11 00:16:36,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +21: [2023-05-11 00:16:36,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +21: [2023-05-11 00:16:36,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +26: [2023-05-11 00:16:36,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +26: [2023-05-11 00:16:36,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 0: [2023-05-11 00:16:36,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +13: [2023-05-11 00:16:36,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +13: [2023-05-11 00:16:36,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +26: [2023-05-11 00:16:36,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +26: [2023-05-11 00:16:36,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 0: [2023-05-11 00:16:36,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 1: [2023-05-11 00:16:36,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +25: [2023-05-11 00:16:36,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +24: [2023-05-11 00:16:36,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +18: [2023-05-11 00:16:36,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +18: [2023-05-11 00:16:36,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +18: [2023-05-11 00:16:36,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 2: [2023-05-11 00:16:36,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +16: [2023-05-11 00:16:36,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +16: [2023-05-11 00:16:36,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +17: [2023-05-11 00:16:36,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +16: [2023-05-11 00:16:36,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt. +13: [2023-05-11 00:16:36,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +13: [2023-05-11 00:16:36,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +12: [2023-05-11 00:16:36,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +12: [2023-05-11 00:16:36,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +29: [2023-05-11 00:16:36,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +12: [2023-05-11 00:16:36,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +16: [2023-05-11 00:16:36,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +29: [2023-05-11 00:16:36,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +29: [2023-05-11 00:16:36,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +16: [2023-05-11 00:16:36,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 2: [2023-05-11 00:16:36,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +31: [2023-05-11 00:16:36,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +31: [2023-05-11 00:16:36,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +31: [2023-05-11 00:16:36,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +31: [2023-05-11 00:16:36,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +29: [2023-05-11 00:16:36,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +29: [2023-05-11 00:16:36,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +25: [2023-05-11 00:16:36,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +25: [2023-05-11 00:16:36,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +17: [2023-05-11 00:16:36,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +16: [2023-05-11 00:16:36,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +31: [2023-05-11 00:16:36,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +31: [2023-05-11 00:16:36,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +31: [2023-05-11 00:16:36,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +31: [2023-05-11 00:16:36,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +31: [2023-05-11 00:16:36,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +31: [2023-05-11 00:16:36,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 4: [2023-05-11 00:16:36,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 4: [2023-05-11 00:16:36,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 4: [2023-05-11 00:16:36,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 8: [2023-05-11 00:16:36,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +24: [2023-05-11 00:16:36,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +15: [2023-05-11 00:16:36,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +29: [2023-05-11 00:16:36,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +18: [2023-05-11 00:16:36,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +15: [2023-05-11 00:16:36,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +16: [2023-05-11 00:16:36,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +12: [2023-05-11 00:16:36,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +12: [2023-05-11 00:16:36,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 8: [2023-05-11 00:16:36,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +17: [2023-05-11 00:16:36,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +18: [2023-05-11 00:16:36,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +15: [2023-05-11 00:16:36,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +18: [2023-05-11 00:16:36,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +18: [2023-05-11 00:16:36,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +15: [2023-05-11 00:16:36,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +25: [2023-05-11 00:16:36,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +16: [2023-05-11 00:16:36,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +25: [2023-05-11 00:16:36,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +16: [2023-05-11 00:16:36,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +17: [2023-05-11 00:16:36,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +18: [2023-05-11 00:16:36,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +29: [2023-05-11 00:16:36,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:36,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 2: [2023-05-11 00:16:36,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +11: [2023-05-11 00:16:36,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +11: [2023-05-11 00:16:36,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +11: [2023-05-11 00:16:36,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +11: [2023-05-11 00:16:36,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +11: [2023-05-11 00:16:36,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +11: [2023-05-11 00:16:36,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +11: [2023-05-11 00:16:36,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +11: [2023-05-11 00:16:36,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +31: [2023-05-11 00:16:36,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +31: [2023-05-11 00:16:36,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +12: [2023-05-11 00:16:36,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +12: [2023-05-11 00:16:36,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +25: [2023-05-11 00:16:36,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +24: [2023-05-11 00:16:36,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +16: [2023-05-11 00:16:36,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +25: [2023-05-11 00:16:36,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +16: [2023-05-11 00:16:36,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +17: [2023-05-11 00:16:36,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +29: [2023-05-11 00:16:36,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +12: [2023-05-11 00:16:36,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +18: [2023-05-11 00:16:36,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +29: [2023-05-11 00:16:36,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +29: [2023-05-11 00:16:36,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +15: [2023-05-11 00:16:36,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +25: [2023-05-11 00:16:36,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt... +25: [2023-05-11 00:16:36,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +12: [2023-05-11 00:16:36,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +15: [2023-05-11 00:16:36,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +31: [2023-05-11 00:16:36,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +18: [2023-05-11 00:16:36,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +16: [2023-05-11 00:16:36,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +17: [2023-05-11 00:16:36,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +17: [2023-05-11 00:16:36,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +16: [2023-05-11 00:16:36,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +18: [2023-05-11 00:16:36,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +31: [2023-05-11 00:16:36,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +15: [2023-05-11 00:16:36,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +24: [2023-05-11 00:16:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +15: [2023-05-11 00:16:36,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +16: [2023-05-11 00:16:36,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +29: [2023-05-11 00:16:36,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +29: [2023-05-11 00:16:36,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +31: [2023-05-11 00:16:36,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +31: [2023-05-11 00:16:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +22: [2023-05-11 00:16:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +22: [2023-05-11 00:16:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +22: [2023-05-11 00:16:36,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +22: [2023-05-11 00:16:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +22: [2023-05-11 00:16:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +22: [2023-05-11 00:16:36,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +16: [2023-05-11 00:16:36,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +16: [2023-05-11 00:16:36,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 4: [2023-05-11 00:16:36,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +22: [2023-05-11 00:16:36,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +16: [2023-05-11 00:16:36,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +29: [2023-05-11 00:16:36,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +16: [2023-05-11 00:16:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +29: [2023-05-11 00:16:36,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +25: [2023-05-11 00:16:36,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +29: [2023-05-11 00:16:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 8: [2023-05-11 00:16:36,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +25: [2023-05-11 00:16:36,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +29: [2023-05-11 00:16:36,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +28: [2023-05-11 00:16:36,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +28: [2023-05-11 00:16:36,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +28: [2023-05-11 00:16:36,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +28: [2023-05-11 00:16:36,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +29: [2023-05-11 00:16:36,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 4: [2023-05-11 00:16:36,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +22: [2023-05-11 00:16:36,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +25: [2023-05-11 00:16:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +31: [2023-05-11 00:16:36,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +25: [2023-05-11 00:16:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 4: [2023-05-11 00:16:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +17: [2023-05-11 00:16:36,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +23: [2023-05-11 00:16:36,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +23: [2023-05-11 00:16:36,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +23: [2023-05-11 00:16:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +23: [2023-05-11 00:16:36,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +23: [2023-05-11 00:16:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +23: [2023-05-11 00:16:36,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +18: [2023-05-11 00:16:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +31: [2023-05-11 00:16:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... + 7: [2023-05-11 00:16:36,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +23: [2023-05-11 00:16:36,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +23: [2023-05-11 00:16:36,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +31: [2023-05-11 00:16:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +31: [2023-05-11 00:16:36,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +31: [2023-05-11 00:16:36,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +29: [2023-05-11 00:16:36,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +22: [2023-05-11 00:16:36,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +28: [2023-05-11 00:16:36,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +28: [2023-05-11 00:16:36,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +16: [2023-05-11 00:16:36,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +14: [2023-05-11 00:16:36,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +14: [2023-05-11 00:16:36,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +14: [2023-05-11 00:16:36,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +14: [2023-05-11 00:16:36,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +14: [2023-05-11 00:16:36,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +14: [2023-05-11 00:16:36,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +25: [2023-05-11 00:16:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. +14: [2023-05-11 00:16:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +28: [2023-05-11 00:16:36,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +31: [2023-05-11 00:16:36,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +22: [2023-05-11 00:16:36,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +16: [2023-05-11 00:16:36,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +22: [2023-05-11 00:16:36,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +25: [2023-05-11 00:16:36,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +14: [2023-05-11 00:16:36,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +28: [2023-05-11 00:16:36,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +24: [2023-05-11 00:16:36,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +24: [2023-05-11 00:16:36,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +24: [2023-05-11 00:16:36,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +24: [2023-05-11 00:16:36,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 8: [2023-05-11 00:16:36,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +23: [2023-05-11 00:16:36,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +29: [2023-05-11 00:16:36,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +23: [2023-05-11 00:16:36,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 4: [2023-05-11 00:16:36,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +23: [2023-05-11 00:16:36,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 8: [2023-05-11 00:16:36,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 8: [2023-05-11 00:16:36,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 4: [2023-05-11 00:16:36,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +22: [2023-05-11 00:16:36,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +29: [2023-05-11 00:16:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 2: [2023-05-11 00:16:36,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +23: [2023-05-11 00:16:36,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +23: [2023-05-11 00:16:36,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +23: [2023-05-11 00:16:36,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 7: [2023-05-11 00:16:36,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +25: [2023-05-11 00:16:36,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 9: [2023-05-11 00:16:36,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 9: [2023-05-11 00:16:36,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +28: [2023-05-11 00:16:36,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +22: [2023-05-11 00:16:36,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +26: [2023-05-11 00:16:36,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +26: [2023-05-11 00:16:36,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +26: [2023-05-11 00:16:36,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +26: [2023-05-11 00:16:36,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +28: [2023-05-11 00:16:36,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +25: [2023-05-11 00:16:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt... +21: [2023-05-11 00:16:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +22: [2023-05-11 00:16:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +26: [2023-05-11 00:16:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +21: [2023-05-11 00:16:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +26: [2023-05-11 00:16:36,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +26: [2023-05-11 00:16:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +26: [2023-05-11 00:16:36,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +28: [2023-05-11 00:16:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +28: [2023-05-11 00:16:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 4: [2023-05-11 00:16:36,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 5: [2023-05-11 00:16:36,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 5: [2023-05-11 00:16:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +24: [2023-05-11 00:16:36,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +21: [2023-05-11 00:16:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +21: [2023-05-11 00:16:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +21: [2023-05-11 00:16:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +21: [2023-05-11 00:16:36,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +24: [2023-05-11 00:16:36,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +24: [2023-05-11 00:16:36,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +21: [2023-05-11 00:16:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +21: [2023-05-11 00:16:36,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +23: [2023-05-11 00:16:36,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +24: [2023-05-11 00:16:36,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +23: [2023-05-11 00:16:36,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +13: [2023-05-11 00:16:36,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +13: [2023-05-11 00:16:36,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +13: [2023-05-11 00:16:36,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +13: [2023-05-11 00:16:36,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +23: [2023-05-11 00:16:36,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +22: [2023-05-11 00:16:36,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +23: [2023-05-11 00:16:36,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +15: [2023-05-11 00:16:36,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +23: [2023-05-11 00:16:36,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +23: [2023-05-11 00:16:36,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +15: [2023-05-11 00:16:36,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +15: [2023-05-11 00:16:36,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 0: [2023-05-11 00:16:36,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +13: [2023-05-11 00:16:36,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +13: [2023-05-11 00:16:36,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +13: [2023-05-11 00:16:36,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +14: [2023-05-11 00:16:36,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +14: [2023-05-11 00:16:36,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +14: [2023-05-11 00:16:36,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +15: [2023-05-11 00:16:36,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 9: [2023-05-11 00:16:36,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +13: [2023-05-11 00:16:36,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +21: [2023-05-11 00:16:36,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +24: [2023-05-11 00:16:36,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +24: [2023-05-11 00:16:36,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +28: [2023-05-11 00:16:36,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +14: [2023-05-11 00:16:36,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +24: [2023-05-11 00:16:36,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +24: [2023-05-11 00:16:36,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +28: [2023-05-11 00:16:36,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +28: [2023-05-11 00:16:36,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +23: [2023-05-11 00:16:36,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +23: [2023-05-11 00:16:36,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +22: [2023-05-11 00:16:36,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +10: [2023-05-11 00:16:36,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +10: [2023-05-11 00:16:36,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +10: [2023-05-11 00:16:36,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +15: [2023-05-11 00:16:36,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +15: [2023-05-11 00:16:36,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +28: [2023-05-11 00:16:36,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 7: [2023-05-11 00:16:36,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 7: [2023-05-11 00:16:36,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 7: [2023-05-11 00:16:36,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +15: [2023-05-11 00:16:36,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +14: [2023-05-11 00:16:36,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +13: [2023-05-11 00:16:36,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +15: [2023-05-11 00:16:36,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +23: [2023-05-11 00:16:36,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +21: [2023-05-11 00:16:36,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +13: [2023-05-11 00:16:36,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +23: [2023-05-11 00:16:36,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +14: [2023-05-11 00:16:36,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +10: [2023-05-11 00:16:36,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +26: [2023-05-11 00:16:36,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +26: [2023-05-11 00:16:36,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +14: [2023-05-11 00:16:36,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +26: [2023-05-11 00:16:36,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +26: [2023-05-11 00:16:36,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +14: [2023-05-11 00:16:36,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +21: [2023-05-11 00:16:36,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +24: [2023-05-11 00:16:36,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +13: [2023-05-11 00:16:36,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +23: [2023-05-11 00:16:36,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +23: [2023-05-11 00:16:36,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +24: [2023-05-11 00:16:36,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +24: [2023-05-11 00:16:36,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +24: [2023-05-11 00:16:36,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +23: [2023-05-11 00:16:36,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +23: [2023-05-11 00:16:36,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 7: [2023-05-11 00:16:36,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +10: [2023-05-11 00:16:36,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +19: [2023-05-11 00:16:36,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +19: [2023-05-11 00:16:36,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 7: [2023-05-11 00:16:36,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +10: [2023-05-11 00:16:36,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +19: [2023-05-11 00:16:36,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +10: [2023-05-11 00:16:36,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +15: [2023-05-11 00:16:36,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +23: [2023-05-11 00:16:36,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +26: [2023-05-11 00:16:36,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +26: [2023-05-11 00:16:36,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +26: [2023-05-11 00:16:36,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +13: [2023-05-11 00:16:36,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +19: [2023-05-11 00:16:36,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +23: [2023-05-11 00:16:36,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +15: [2023-05-11 00:16:36,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +13: [2023-05-11 00:16:36,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +21: [2023-05-11 00:16:36,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +21: [2023-05-11 00:16:36,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +21: [2023-05-11 00:16:36,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +26: [2023-05-11 00:16:36,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +13: [2023-05-11 00:16:36,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +10: [2023-05-11 00:16:36,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +13: [2023-05-11 00:16:36,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +15: [2023-05-11 00:16:36,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +15: [2023-05-11 00:16:36,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +23: [2023-05-11 00:16:36,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +15: [2023-05-11 00:16:36,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +19: [2023-05-11 00:16:36,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +15: [2023-05-11 00:16:36,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +15: [2023-05-11 00:16:36,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +15: [2023-05-11 00:16:36,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +23: [2023-05-11 00:16:36,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 0: [2023-05-11 00:16:36,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +19: [2023-05-11 00:16:36,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +19: [2023-05-11 00:16:36,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +21: [2023-05-11 00:16:36,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +21: [2023-05-11 00:16:36,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 7: [2023-05-11 00:16:36,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +19: [2023-05-11 00:16:36,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +13: [2023-05-11 00:16:36,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +10: [2023-05-11 00:16:36,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +10: [2023-05-11 00:16:36,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +10: [2023-05-11 00:16:36,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +10: [2023-05-11 00:16:36,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 0: [2023-05-11 00:16:36,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 3: [2023-05-11 00:16:36,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +10: [2023-05-11 00:16:36,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +10: [2023-05-11 00:16:36,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 7: [2023-05-11 00:16:36,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +11: [2023-05-11 00:16:36,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +11: [2023-05-11 00:16:36,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +11: [2023-05-11 00:16:36,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 7: [2023-05-11 00:16:36,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +11: [2023-05-11 00:16:36,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 6: [2023-05-11 00:16:36,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 6: [2023-05-11 00:16:36,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 6: [2023-05-11 00:16:36,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 6: [2023-05-11 00:16:36,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 3: [2023-05-11 00:16:36,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +10: [2023-05-11 00:16:36,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 5: [2023-05-11 00:16:36,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 5: [2023-05-11 00:16:36,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 5: [2023-05-11 00:16:36,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +10: [2023-05-11 00:16:36,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 5: [2023-05-11 00:16:36,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +11: [2023-05-11 00:16:36,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +11: [2023-05-11 00:16:36,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +11: [2023-05-11 00:16:36,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +11: [2023-05-11 00:16:36,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +11: [2023-05-11 00:16:36,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +11: [2023-05-11 00:16:36,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 3: [2023-05-11 00:16:36,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +19: [2023-05-11 00:16:36,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +19: [2023-05-11 00:16:36,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +19: [2023-05-11 00:16:36,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +19: [2023-05-11 00:16:36,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +19: [2023-05-11 00:16:36,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +19: [2023-05-11 00:16:36,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +19: [2023-05-11 00:16:36,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +19: [2023-05-11 00:16:36,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +19: [2023-05-11 00:16:36,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +11: [2023-05-11 00:16:36,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +19: [2023-05-11 00:16:36,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +19: [2023-05-11 00:16:36,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +11: [2023-05-11 00:16:36,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +10: [2023-05-11 00:16:36,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +10: [2023-05-11 00:16:36,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +10: [2023-05-11 00:16:36,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +10: [2023-05-11 00:16:36,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +11: [2023-05-11 00:16:36,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +19: [2023-05-11 00:16:36,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +19: [2023-05-11 00:16:36,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +19: [2023-05-11 00:16:36,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +19: [2023-05-11 00:16:36,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +19: [2023-05-11 00:16:36,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +11: [2023-05-11 00:16:36,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +11: [2023-05-11 00:16:36,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 9: [2023-05-11 00:16:36,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +11: [2023-05-11 00:16:36,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +12: [2023-05-11 00:16:36,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +12: [2023-05-11 00:16:36,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +12: [2023-05-11 00:16:36,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +12: [2023-05-11 00:16:36,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 6: [2023-05-11 00:16:36,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 6: [2023-05-11 00:16:36,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 9: [2023-05-11 00:16:36,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 9: [2023-05-11 00:16:36,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 9: [2023-05-11 00:16:36,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 3: [2023-05-11 00:16:36,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +10: [2023-05-11 00:16:36,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +10: [2023-05-11 00:16:36,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +10: [2023-05-11 00:16:36,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +10: [2023-05-11 00:16:36,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +19: [2023-05-11 00:16:36,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +19: [2023-05-11 00:16:36,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +12: [2023-05-11 00:16:36,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +12: [2023-05-11 00:16:36,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +12: [2023-05-11 00:16:36,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +12: [2023-05-11 00:16:36,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +29: [2023-05-11 00:16:36,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +29: [2023-05-11 00:16:36,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +29: [2023-05-11 00:16:36,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +10: [2023-05-11 00:16:36,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +10: [2023-05-11 00:16:36,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +19: [2023-05-11 00:16:36,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +29: [2023-05-11 00:16:36,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 8: [2023-05-11 00:16:36,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 8: [2023-05-11 00:16:36,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +19: [2023-05-11 00:16:36,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 8: [2023-05-11 00:16:36,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +10: [2023-05-11 00:16:36,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +19: [2023-05-11 00:16:36,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +19: [2023-05-11 00:16:36,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +10: [2023-05-11 00:16:36,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +29: [2023-05-11 00:16:36,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +29: [2023-05-11 00:16:36,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +29: [2023-05-11 00:16:36,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +29: [2023-05-11 00:16:36,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +19: [2023-05-11 00:16:36,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +19: [2023-05-11 00:16:36,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 8: [2023-05-11 00:16:36,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +10: [2023-05-11 00:16:36,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +10: [2023-05-11 00:16:36,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 9: [2023-05-11 00:16:36,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 7: [2023-05-11 00:16:36,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +10: [2023-05-11 00:16:36,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +15: [2023-05-11 00:16:36,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +15: [2023-05-11 00:16:36,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +15: [2023-05-11 00:16:36,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +15: [2023-05-11 00:16:36,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +15: [2023-05-11 00:16:36,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +15: [2023-05-11 00:16:36,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +15: [2023-05-11 00:16:36,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +15: [2023-05-11 00:16:36,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +14: [2023-05-11 00:16:36,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +14: [2023-05-11 00:16:36,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +14: [2023-05-11 00:16:36,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +30: [2023-05-11 00:16:36,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +30: [2023-05-11 00:16:36,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +30: [2023-05-11 00:16:36,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +30: [2023-05-11 00:16:36,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +14: [2023-05-11 00:16:36,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +22: [2023-05-11 00:16:36,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +22: [2023-05-11 00:16:36,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +22: [2023-05-11 00:16:36,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +22: [2023-05-11 00:16:36,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +10: [2023-05-11 00:16:36,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +27: [2023-05-11 00:16:36,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +27: [2023-05-11 00:16:36,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +27: [2023-05-11 00:16:36,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +27: [2023-05-11 00:16:36,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +28: [2023-05-11 00:16:36,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +28: [2023-05-11 00:16:36,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +28: [2023-05-11 00:16:36,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +28: [2023-05-11 00:16:36,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +15: [2023-05-11 00:16:36,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +23: [2023-05-11 00:16:36,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +23: [2023-05-11 00:16:36,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +15: [2023-05-11 00:16:36,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +24: [2023-05-11 00:16:36,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +24: [2023-05-11 00:16:36,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +24: [2023-05-11 00:16:36,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +24: [2023-05-11 00:16:36,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +14: [2023-05-11 00:16:36,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +14: [2023-05-11 00:16:36,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +28: [2023-05-11 00:16:36,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +28: [2023-05-11 00:16:36,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +27: [2023-05-11 00:16:36,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +22: [2023-05-11 00:16:36,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +27: [2023-05-11 00:16:36,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +15: [2023-05-11 00:16:36,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +22: [2023-05-11 00:16:36,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +22: [2023-05-11 00:16:36,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +24: [2023-05-11 00:16:36,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +30: [2023-05-11 00:16:36,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +24: [2023-05-11 00:16:36,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +30: [2023-05-11 00:16:36,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +30: [2023-05-11 00:16:36,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +14: [2023-05-11 00:16:36,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +14: [2023-05-11 00:16:36,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +30: [2023-05-11 00:16:36,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +27: [2023-05-11 00:16:36,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +23: [2023-05-11 00:16:36,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +24: [2023-05-11 00:16:36,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +23: [2023-05-11 00:16:36,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +15: [2023-05-11 00:16:36,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +27: [2023-05-11 00:16:36,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +28: [2023-05-11 00:16:36,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +28: [2023-05-11 00:16:36,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +28: [2023-05-11 00:16:36,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +24: [2023-05-11 00:16:36,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +22: [2023-05-11 00:16:36,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +28: [2023-05-11 00:16:36,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +28: [2023-05-11 00:16:36,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +23: [2023-05-11 00:16:36,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +28: [2023-05-11 00:16:36,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +27: [2023-05-11 00:16:36,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +27: [2023-05-11 00:16:36,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +23: [2023-05-11 00:16:36,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +15: [2023-05-11 00:16:36,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +15: [2023-05-11 00:16:36,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +28: [2023-05-11 00:16:36,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +28: [2023-05-11 00:16:36,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +23: [2023-05-11 00:16:36,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +28: [2023-05-11 00:16:36,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +23: [2023-05-11 00:16:36,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +14: [2023-05-11 00:16:36,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +14: [2023-05-11 00:16:36,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +15: [2023-05-11 00:16:36,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +15: [2023-05-11 00:16:36,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +28: [2023-05-11 00:16:36,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +22: [2023-05-11 00:16:36,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +22: [2023-05-11 00:16:36,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +24: [2023-05-11 00:16:36,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +28: [2023-05-11 00:16:36,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +10: [2023-05-11 00:16:36,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +10: [2023-05-11 00:16:36,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +22: [2023-05-11 00:16:36,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +22: [2023-05-11 00:16:36,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +10: [2023-05-11 00:16:36,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 7: [2023-05-11 00:16:36,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +15: [2023-05-11 00:16:36,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +15: [2023-05-11 00:16:36,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +15: [2023-05-11 00:16:36,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +15: [2023-05-11 00:16:36,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +15: [2023-05-11 00:16:36,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +27: [2023-05-11 00:16:36,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +15: [2023-05-11 00:16:36,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +15: [2023-05-11 00:16:36,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +15: [2023-05-11 00:16:36,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +10: [2023-05-11 00:16:36,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 7: [2023-05-11 00:16:36,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +15: [2023-05-11 00:16:36,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +15: [2023-05-11 00:16:36,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +15: [2023-05-11 00:16:36,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +15: [2023-05-11 00:16:36,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +27: [2023-05-11 00:16:36,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +15: [2023-05-11 00:16:36,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +15: [2023-05-11 00:16:36,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +15: [2023-05-11 00:16:36,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +15: [2023-05-11 00:16:36,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +24: [2023-05-11 00:16:36,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +14: [2023-05-11 00:16:36,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +22: [2023-05-11 00:16:36,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +14: [2023-05-11 00:16:36,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +14: [2023-05-11 00:16:36,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +30: [2023-05-11 00:16:36,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +30: [2023-05-11 00:16:36,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +28: [2023-05-11 00:16:36,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +30: [2023-05-11 00:16:36,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +30: [2023-05-11 00:16:36,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +28: [2023-05-11 00:16:36,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +14: [2023-05-11 00:16:36,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +22: [2023-05-11 00:16:36,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +28: [2023-05-11 00:16:36,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +24: [2023-05-11 00:16:36,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +28: [2023-05-11 00:16:36,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +24: [2023-05-11 00:16:36,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +10: [2023-05-11 00:16:36,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +30: [2023-05-11 00:16:36,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +10: [2023-05-11 00:16:36,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +27: [2023-05-11 00:16:36,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +10: [2023-05-11 00:16:36,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +30: [2023-05-11 00:16:36,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +30: [2023-05-11 00:16:36,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +30: [2023-05-11 00:16:36,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +10: [2023-05-11 00:16:36,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +14: [2023-05-11 00:16:36,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +24: [2023-05-11 00:16:36,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +24: [2023-05-11 00:16:36,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +24: [2023-05-11 00:16:36,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +27: [2023-05-11 00:16:36,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +22: [2023-05-11 00:16:36,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +14: [2023-05-11 00:16:36,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +15: [2023-05-11 00:16:36,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +24: [2023-05-11 00:16:36,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +24: [2023-05-11 00:16:36,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +24: [2023-05-11 00:16:36,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +22: [2023-05-11 00:16:36,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +24: [2023-05-11 00:16:36,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +15: [2023-05-11 00:16:36,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +15: [2023-05-11 00:16:36,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +24: [2023-05-11 00:16:36,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +15: [2023-05-11 00:16:36,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +24: [2023-05-11 00:16:36,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +24: [2023-05-11 00:16:36,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +27: [2023-05-11 00:16:36,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +24: [2023-05-11 00:16:36,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +28: [2023-05-11 00:16:36,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +24: [2023-05-11 00:16:36,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +28: [2023-05-11 00:16:36,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +28: [2023-05-11 00:16:36,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +22: [2023-05-11 00:16:36,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +22: [2023-05-11 00:16:36,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +27: [2023-05-11 00:16:36,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +22: [2023-05-11 00:16:36,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +22: [2023-05-11 00:16:36,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +22: [2023-05-11 00:16:36,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +22: [2023-05-11 00:16:36,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +22: [2023-05-11 00:16:36,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +22: [2023-05-11 00:16:36,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +10: [2023-05-11 00:16:36,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +10: [2023-05-11 00:16:36,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +28: [2023-05-11 00:16:36,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +28: [2023-05-11 00:16:36,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 1: [2023-05-11 00:16:36,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 1: [2023-05-11 00:16:36,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 1: [2023-05-11 00:16:36,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 1: [2023-05-11 00:16:36,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +15: [2023-05-11 00:16:36,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +15: [2023-05-11 00:16:36,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +24: [2023-05-11 00:16:36,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +24: [2023-05-11 00:16:36,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +24: [2023-05-11 00:16:36,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +24: [2023-05-11 00:16:36,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +15: [2023-05-11 00:16:36,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +15: [2023-05-11 00:16:36,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +10: [2023-05-11 00:16:36,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +10: [2023-05-11 00:16:36,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +10: [2023-05-11 00:16:36,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +10: [2023-05-11 00:16:36,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +15: [2023-05-11 00:16:36,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +15: [2023-05-11 00:16:36,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +28: [2023-05-11 00:16:36,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +28: [2023-05-11 00:16:36,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +28: [2023-05-11 00:16:36,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +10: [2023-05-11 00:16:36,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +10: [2023-05-11 00:16:36,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 1: [2023-05-11 00:16:36,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt... + 7: [2023-05-11 00:16:36,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt... + 7: [2023-05-11 00:16:36,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt... + 7: [2023-05-11 00:16:36,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt... +22: [2023-05-11 00:16:36,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +28: [2023-05-11 00:16:36,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +22: [2023-05-11 00:16:36,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +22: [2023-05-11 00:16:36,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +28: [2023-05-11 00:16:36,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +15: [2023-05-11 00:16:36,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +15: [2023-05-11 00:16:36,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt... +15: [2023-05-11 00:16:36,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt... +15: [2023-05-11 00:16:36,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt... +15: [2023-05-11 00:16:36,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt... +15: [2023-05-11 00:16:36,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +30: [2023-05-11 00:16:36,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +30: [2023-05-11 00:16:36,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +24: [2023-05-11 00:16:36,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt... +24: [2023-05-11 00:16:36,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt... +24: [2023-05-11 00:16:36,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt... +24: [2023-05-11 00:16:36,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt... +30: [2023-05-11 00:16:36,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +30: [2023-05-11 00:16:36,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +30: [2023-05-11 00:16:36,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 7: [2023-05-11 00:16:36,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 7: [2023-05-11 00:16:36,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 7: [2023-05-11 00:16:36,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +30: [2023-05-11 00:16:36,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +22: [2023-05-11 00:16:36,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +22: [2023-05-11 00:16:36,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +30: [2023-05-11 00:16:36,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 6: [2023-05-11 00:16:36,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +30: [2023-05-11 00:16:36,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +22: [2023-05-11 00:16:36,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +28: [2023-05-11 00:16:36,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +11: [2023-05-11 00:16:36,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +22: [2023-05-11 00:16:36,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +11: [2023-05-11 00:16:36,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +11: [2023-05-11 00:16:36,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +22: [2023-05-11 00:16:36,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +22: [2023-05-11 00:16:36,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +22: [2023-05-11 00:16:36,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +22: [2023-05-11 00:16:36,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +22: [2023-05-11 00:16:36,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +22: [2023-05-11 00:16:36,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +22: [2023-05-11 00:16:36,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +22: [2023-05-11 00:16:36,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +11: [2023-05-11 00:16:36,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +22: [2023-05-11 00:16:36,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +18: [2023-05-11 00:16:36,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 1: [2023-05-11 00:16:36,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 1: [2023-05-11 00:16:36,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 1: [2023-05-11 00:16:36,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 1: [2023-05-11 00:16:36,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +18: [2023-05-11 00:16:36,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 7: [2023-05-11 00:16:36,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +17: [2023-05-11 00:16:36,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +17: [2023-05-11 00:16:36,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 7: [2023-05-11 00:16:36,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +18: [2023-05-11 00:16:36,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +17: [2023-05-11 00:16:36,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +22: [2023-05-11 00:16:36,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +18: [2023-05-11 00:16:36,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +11: [2023-05-11 00:16:36,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +17: [2023-05-11 00:16:36,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +22: [2023-05-11 00:16:36,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +30: [2023-05-11 00:16:36,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +22: [2023-05-11 00:16:36,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +22: [2023-05-11 00:16:36,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +11: [2023-05-11 00:16:36,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +11: [2023-05-11 00:16:36,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +11: [2023-05-11 00:16:36,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +18: [2023-05-11 00:16:36,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +26: [2023-05-11 00:16:36,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +26: [2023-05-11 00:16:36,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +26: [2023-05-11 00:16:36,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +26: [2023-05-11 00:16:36,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +18: [2023-05-11 00:16:36,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +21: [2023-05-11 00:16:36,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +21: [2023-05-11 00:16:36,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +17: [2023-05-11 00:16:36,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +21: [2023-05-11 00:16:36,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +21: [2023-05-11 00:16:36,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +17: [2023-05-11 00:16:36,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +18: [2023-05-11 00:16:36,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +17: [2023-05-11 00:16:36,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +30: [2023-05-11 00:16:36,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +18: [2023-05-11 00:16:36,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +18: [2023-05-11 00:16:36,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +18: [2023-05-11 00:16:36,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +18: [2023-05-11 00:16:36,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +18: [2023-05-11 00:16:36,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +17: [2023-05-11 00:16:36,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +18: [2023-05-11 00:16:36,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +18: [2023-05-11 00:16:36,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +18: [2023-05-11 00:16:36,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +21: [2023-05-11 00:16:36,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +21: [2023-05-11 00:16:36,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +21: [2023-05-11 00:16:36,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +30: [2023-05-11 00:16:36,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +31: [2023-05-11 00:16:36,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 7: [2023-05-11 00:16:36,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +13: [2023-05-11 00:16:36,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +13: [2023-05-11 00:16:36,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +13: [2023-05-11 00:16:36,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +27: [2023-05-11 00:16:36,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +31: [2023-05-11 00:16:36,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +31: [2023-05-11 00:16:36,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +21: [2023-05-11 00:16:36,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +13: [2023-05-11 00:16:36,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +27: [2023-05-11 00:16:36,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +27: [2023-05-11 00:16:36,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +31: [2023-05-11 00:16:36,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 0: [2023-05-11 00:16:36,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 0: [2023-05-11 00:16:36,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +21: [2023-05-11 00:16:36,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 0: [2023-05-11 00:16:36,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +21: [2023-05-11 00:16:36,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +26: [2023-05-11 00:16:36,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +20: [2023-05-11 00:16:36,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +20: [2023-05-11 00:16:36,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +20: [2023-05-11 00:16:36,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 2: [2023-05-11 00:16:36,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 2: [2023-05-11 00:16:36,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 2: [2023-05-11 00:16:36,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +20: [2023-05-11 00:16:36,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +18: [2023-05-11 00:16:36,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +26: [2023-05-11 00:16:36,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +16: [2023-05-11 00:16:36,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +16: [2023-05-11 00:16:36,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +16: [2023-05-11 00:16:36,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +25: [2023-05-11 00:16:36,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +25: [2023-05-11 00:16:36,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +25: [2023-05-11 00:16:36,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +16: [2023-05-11 00:16:36,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +25: [2023-05-11 00:16:36,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +27: [2023-05-11 00:16:36,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 7: [2023-05-11 00:16:36,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +21: [2023-05-11 00:16:36,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +30: [2023-05-11 00:16:36,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +30: [2023-05-11 00:16:36,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +26: [2023-05-11 00:16:36,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +21: [2023-05-11 00:16:36,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +26: [2023-05-11 00:16:36,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +17: [2023-05-11 00:16:36,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 7: [2023-05-11 00:16:36,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +31: [2023-05-11 00:16:36,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +31: [2023-05-11 00:16:36,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +17: [2023-05-11 00:16:36,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +30: [2023-05-11 00:16:36,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +31: [2023-05-11 00:16:36,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +20: [2023-05-11 00:16:36,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +16: [2023-05-11 00:16:36,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +25: [2023-05-11 00:16:36,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +27: [2023-05-11 00:16:36,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +21: [2023-05-11 00:16:36,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +20: [2023-05-11 00:16:36,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +20: [2023-05-11 00:16:36,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +27: [2023-05-11 00:16:36,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +20: [2023-05-11 00:16:36,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +21: [2023-05-11 00:16:36,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +31: [2023-05-11 00:16:36,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +18: [2023-05-11 00:16:36,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +17: [2023-05-11 00:16:36,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 4: [2023-05-11 00:16:36,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +17: [2023-05-11 00:16:36,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 4: [2023-05-11 00:16:36,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +18: [2023-05-11 00:16:36,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 4: [2023-05-11 00:16:36,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 4: [2023-05-11 00:16:36,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +16: [2023-05-11 00:16:36,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +26: [2023-05-11 00:16:36,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +16: [2023-05-11 00:16:36,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +27: [2023-05-11 00:16:36,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +29: [2023-05-11 00:16:36,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +29: [2023-05-11 00:16:36,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +29: [2023-05-11 00:16:36,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +29: [2023-05-11 00:16:36,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +25: [2023-05-11 00:16:36,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +29: [2023-05-11 00:16:36,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +13: [2023-05-11 00:16:36,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +29: [2023-05-11 00:16:36,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +26: [2023-05-11 00:16:36,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +13: [2023-05-11 00:16:36,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +13: [2023-05-11 00:16:36,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +21: [2023-05-11 00:16:36,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +13: [2023-05-11 00:16:36,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 7: [2023-05-11 00:16:36,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +25: [2023-05-11 00:16:36,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +29: [2023-05-11 00:16:36,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +25: [2023-05-11 00:16:36,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +16: [2023-05-11 00:16:36,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +27: [2023-05-11 00:16:36,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +29: [2023-05-11 00:16:36,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +29: [2023-05-11 00:16:36,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +29: [2023-05-11 00:16:36,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +29: [2023-05-11 00:16:36,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +30: [2023-05-11 00:16:36,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +29: [2023-05-11 00:16:36,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +29: [2023-05-11 00:16:36,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 4: [2023-05-11 00:16:36,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 1: [2023-05-11 00:16:36,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +17: [2023-05-11 00:16:36,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +17: [2023-05-11 00:16:36,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 2: [2023-05-11 00:16:36,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +26: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +22: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt... +29: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +26: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +22: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt... +22: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt... + 7: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +22: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt... + 6: [2023-05-11 00:16:36,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 6: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +17: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +17: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +17: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +17: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +17: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +18: [2023-05-11 00:16:36,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +17: [2023-05-11 00:16:36,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +18: [2023-05-11 00:16:36,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +17: [2023-05-11 00:16:36,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +17: [2023-05-11 00:16:36,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +29: [2023-05-11 00:16:36,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +14: [2023-05-11 00:16:36,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +14: [2023-05-11 00:16:36,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +14: [2023-05-11 00:16:36,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +29: [2023-05-11 00:16:36,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +14: [2023-05-11 00:16:36,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +14: [2023-05-11 00:16:36,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +14: [2023-05-11 00:16:36,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +14: [2023-05-11 00:16:36,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +12: [2023-05-11 00:16:36,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +12: [2023-05-11 00:16:36,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +12: [2023-05-11 00:16:36,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +12: [2023-05-11 00:16:36,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +12: [2023-05-11 00:16:36,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +25: [2023-05-11 00:16:36,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +12: [2023-05-11 00:16:36,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +12: [2023-05-11 00:16:36,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +12: [2023-05-11 00:16:36,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +12: [2023-05-11 00:16:36,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +12: [2023-05-11 00:16:36,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +12: [2023-05-11 00:16:36,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +25: [2023-05-11 00:16:36,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +12: [2023-05-11 00:16:36,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +12: [2023-05-11 00:16:36,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +12: [2023-05-11 00:16:36,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +14: [2023-05-11 00:16:36,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +25: [2023-05-11 00:16:36,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +20: [2023-05-11 00:16:36,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +20: [2023-05-11 00:16:36,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +20: [2023-05-11 00:16:36,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +12: [2023-05-11 00:16:36,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +21: [2023-05-11 00:16:36,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +30: [2023-05-11 00:16:36,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +26: [2023-05-11 00:16:36,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +13: [2023-05-11 00:16:36,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +26: [2023-05-11 00:16:36,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +13: [2023-05-11 00:16:36,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +25: [2023-05-11 00:16:36,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +13: [2023-05-11 00:16:36,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +20: [2023-05-11 00:16:36,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +13: [2023-05-11 00:16:36,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +25: [2023-05-11 00:16:36,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +21: [2023-05-11 00:16:36,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +21: [2023-05-11 00:16:36,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +26: [2023-05-11 00:16:36,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 8: [2023-05-11 00:16:36,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +25: [2023-05-11 00:16:36,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +12: [2023-05-11 00:16:36,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +26: [2023-05-11 00:16:36,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +31: [2023-05-11 00:16:36,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +31: [2023-05-11 00:16:36,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +26: [2023-05-11 00:16:36,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +26: [2023-05-11 00:16:36,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +31: [2023-05-11 00:16:36,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +25: [2023-05-11 00:16:36,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +13: [2023-05-11 00:16:36,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +13: [2023-05-11 00:16:36,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +13: [2023-05-11 00:16:36,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +13: [2023-05-11 00:16:36,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +26: [2023-05-11 00:16:36,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +20: [2023-05-11 00:16:36,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +20: [2023-05-11 00:16:36,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +20: [2023-05-11 00:16:36,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +20: [2023-05-11 00:16:36,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +26: [2023-05-11 00:16:36,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +31: [2023-05-11 00:16:36,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +31: [2023-05-11 00:16:36,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +31: [2023-05-11 00:16:36,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +31: [2023-05-11 00:16:36,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +18: [2023-05-11 00:16:36,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +18: [2023-05-11 00:16:36,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 3: [2023-05-11 00:16:36,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +18: [2023-05-11 00:16:36,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +18: [2023-05-11 00:16:36,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +18: [2023-05-11 00:16:36,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +18: [2023-05-11 00:16:36,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 4: [2023-05-11 00:16:36,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +31: [2023-05-11 00:16:36,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +29: [2023-05-11 00:16:36,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +21: [2023-05-11 00:16:36,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +25: [2023-05-11 00:16:36,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +21: [2023-05-11 00:16:36,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +25: [2023-05-11 00:16:36,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +25: [2023-05-11 00:16:36,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +17: [2023-05-11 00:16:36,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +18: [2023-05-11 00:16:36,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +16: [2023-05-11 00:16:36,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +27: [2023-05-11 00:16:36,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +27: [2023-05-11 00:16:36,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +16: [2023-05-11 00:16:36,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 0: [2023-05-11 00:16:36,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +14: [2023-05-11 00:16:36,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +16: [2023-05-11 00:16:36,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +16: [2023-05-11 00:16:36,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 0: [2023-05-11 00:16:36,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +17: [2023-05-11 00:16:36,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +21: [2023-05-11 00:16:36,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +29: [2023-05-11 00:16:36,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +21: [2023-05-11 00:16:36,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +21: [2023-05-11 00:16:36,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 5: [2023-05-11 00:16:36,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +14: [2023-05-11 00:16:36,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 9: [2023-05-11 00:16:36,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 9: [2023-05-11 00:16:36,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 0: [2023-05-11 00:16:36,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +26: [2023-05-11 00:16:36,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +21: [2023-05-11 00:16:36,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +27: [2023-05-11 00:16:36,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +27: [2023-05-11 00:16:36,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +26: [2023-05-11 00:16:36,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +17: [2023-05-11 00:16:36,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +17: [2023-05-11 00:16:36,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +17: [2023-05-11 00:16:36,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +25: [2023-05-11 00:16:36,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +18: [2023-05-11 00:16:36,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +13: [2023-05-11 00:16:36,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +13: [2023-05-11 00:16:36,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +17: [2023-05-11 00:16:36,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +18: [2023-05-11 00:16:36,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +26: [2023-05-11 00:16:36,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +13: [2023-05-11 00:16:36,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +13: [2023-05-11 00:16:36,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +31: [2023-05-11 00:16:36,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +29: [2023-05-11 00:16:36,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +18: [2023-05-11 00:16:36,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +21: [2023-05-11 00:16:36,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +26: [2023-05-11 00:16:36,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 6: [2023-05-11 00:16:36,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +14: [2023-05-11 00:16:36,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 6: [2023-05-11 00:16:36,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +25: [2023-05-11 00:16:36,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +25: [2023-05-11 00:16:36,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +25: [2023-05-11 00:16:36,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 8: [2023-05-11 00:16:36,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +25: [2023-05-11 00:16:36,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +27: [2023-05-11 00:16:36,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +27: [2023-05-11 00:16:36,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +31: [2023-05-11 00:16:36,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 6: [2023-05-11 00:16:36,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +31: [2023-05-11 00:16:36,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 0: [2023-05-11 00:16:36,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 5: [2023-05-11 00:16:36,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +21: [2023-05-11 00:16:36,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +21: [2023-05-11 00:16:36,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +31: [2023-05-11 00:16:36,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +20: [2023-05-11 00:16:36,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +31: [2023-05-11 00:16:36,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +31: [2023-05-11 00:16:36,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +12: [2023-05-11 00:16:36,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +12: [2023-05-11 00:16:36,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +31: [2023-05-11 00:16:36,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 5: [2023-05-11 00:16:36,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +21: [2023-05-11 00:16:36,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +21: [2023-05-11 00:16:36,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +21: [2023-05-11 00:16:36,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +20: [2023-05-11 00:16:36,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +21: [2023-05-11 00:16:36,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +21: [2023-05-11 00:16:36,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +21: [2023-05-11 00:16:36,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +13: [2023-05-11 00:16:36,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +21: [2023-05-11 00:16:36,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +20: [2023-05-11 00:16:36,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 3: [2023-05-11 00:16:36,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +27: [2023-05-11 00:16:36,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +27: [2023-05-11 00:16:36,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +13: [2023-05-11 00:16:36,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +13: [2023-05-11 00:16:36,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +13: [2023-05-11 00:16:36,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +25: [2023-05-11 00:16:36,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 4: [2023-05-11 00:16:36,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 4: [2023-05-11 00:16:36,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +16: [2023-05-11 00:16:36,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +18: [2023-05-11 00:16:36,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +29: [2023-05-11 00:16:36,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +25: [2023-05-11 00:16:36,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +19: [2023-05-11 00:16:36,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +19: [2023-05-11 00:16:36,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +19: [2023-05-11 00:16:36,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +19: [2023-05-11 00:16:36,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +21: [2023-05-11 00:16:36,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +16: [2023-05-11 00:16:36,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +21: [2023-05-11 00:16:36,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +16: [2023-05-11 00:16:36,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +21: [2023-05-11 00:16:36,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +21: [2023-05-11 00:16:36,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +25: [2023-05-11 00:16:36,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +12: [2023-05-11 00:16:36,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +25: [2023-05-11 00:16:36,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +26: [2023-05-11 00:16:36,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +26: [2023-05-11 00:16:36,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 3: [2023-05-11 00:16:36,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 9: [2023-05-11 00:16:36,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +16: [2023-05-11 00:16:36,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +14: [2023-05-11 00:16:36,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +31: [2023-05-11 00:16:36,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +31: [2023-05-11 00:16:36,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +14: [2023-05-11 00:16:36,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +21: [2023-05-11 00:16:36,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +19: [2023-05-11 00:16:36,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +19: [2023-05-11 00:16:36,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +21: [2023-05-11 00:16:36,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +14: [2023-05-11 00:16:36,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +19: [2023-05-11 00:16:36,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +13: [2023-05-11 00:16:36,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +18: [2023-05-11 00:16:36,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +13: [2023-05-11 00:16:36,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 8: [2023-05-11 00:16:36,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +13: [2023-05-11 00:16:36,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +19: [2023-05-11 00:16:36,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +17: [2023-05-11 00:16:36,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +13: [2023-05-11 00:16:36,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +31: [2023-05-11 00:16:36,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +14: [2023-05-11 00:16:36,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +14: [2023-05-11 00:16:36,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 2: [2023-05-11 00:16:36,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 2: [2023-05-11 00:16:36,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 8: [2023-05-11 00:16:36,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 8: [2023-05-11 00:16:36,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +17: [2023-05-11 00:16:36,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +17: [2023-05-11 00:16:36,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +20: [2023-05-11 00:16:36,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +20: [2023-05-11 00:16:36,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +21: [2023-05-11 00:16:36,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +12: [2023-05-11 00:16:36,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +21: [2023-05-11 00:16:36,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +21: [2023-05-11 00:16:36,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +21: [2023-05-11 00:16:36,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +14: [2023-05-11 00:16:36,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +20: [2023-05-11 00:16:36,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +14: [2023-05-11 00:16:36,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +20: [2023-05-11 00:16:36,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +14: [2023-05-11 00:16:36,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. +20: [2023-05-11 00:16:36,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +29: [2023-05-11 00:16:36,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +17: [2023-05-11 00:16:36,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +14: [2023-05-11 00:16:36,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +25: [2023-05-11 00:16:36,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +12: [2023-05-11 00:16:36,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +14: [2023-05-11 00:16:36,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +12: [2023-05-11 00:16:36,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +13: [2023-05-11 00:16:36,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +31: [2023-05-11 00:16:36,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 6: [2023-05-11 00:16:36,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +31: [2023-05-11 00:16:36,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 6: [2023-05-11 00:16:36,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +13: [2023-05-11 00:16:36,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +25: [2023-05-11 00:16:36,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +12: [2023-05-11 00:16:36,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +20: [2023-05-11 00:16:36,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +20: [2023-05-11 00:16:36,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +20: [2023-05-11 00:16:36,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +20: [2023-05-11 00:16:36,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +20: [2023-05-11 00:16:36,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +26: [2023-05-11 00:16:36,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt. + 2: [2023-05-11 00:16:36,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +13: [2023-05-11 00:16:36,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +19: [2023-05-11 00:16:36,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +19: [2023-05-11 00:16:36,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +13: [2023-05-11 00:16:36,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +31: [2023-05-11 00:16:36,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +29: [2023-05-11 00:16:36,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +19: [2023-05-11 00:16:36,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +19: [2023-05-11 00:16:36,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 9: [2023-05-11 00:16:36,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +23: [2023-05-11 00:16:36,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +23: [2023-05-11 00:16:36,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +23: [2023-05-11 00:16:36,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +23: [2023-05-11 00:16:36,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +23: [2023-05-11 00:16:36,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +19: [2023-05-11 00:16:36,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +23: [2023-05-11 00:16:36,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +23: [2023-05-11 00:16:36,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +23: [2023-05-11 00:16:36,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +23: [2023-05-11 00:16:36,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +23: [2023-05-11 00:16:36,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +23: [2023-05-11 00:16:36,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 3: [2023-05-11 00:16:36,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +26: [2023-05-11 00:16:36,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +23: [2023-05-11 00:16:36,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +23: [2023-05-11 00:16:36,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt... + 1: [2023-05-11 00:16:36,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt... +23: [2023-05-11 00:16:36,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt... + 1: [2023-05-11 00:16:36,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt... +29: [2023-05-11 00:16:36,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +14: [2023-05-11 00:16:36,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +25: [2023-05-11 00:16:36,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +14: [2023-05-11 00:16:36,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +14: [2023-05-11 00:16:36,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +23: [2023-05-11 00:16:36,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 9: [2023-05-11 00:16:36,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +19: [2023-05-11 00:16:36,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +23: [2023-05-11 00:16:36,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +14: [2023-05-11 00:16:36,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +14: [2023-05-11 00:16:36,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +14: [2023-05-11 00:16:36,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +14: [2023-05-11 00:16:36,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +26: [2023-05-11 00:16:36,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +14: [2023-05-11 00:16:36,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +14: [2023-05-11 00:16:36,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +26: [2023-05-11 00:16:36,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +31: [2023-05-11 00:16:36,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +31: [2023-05-11 00:16:36,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +14: [2023-05-11 00:16:36,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +12: [2023-05-11 00:16:36,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +14: [2023-05-11 00:16:36,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +14: [2023-05-11 00:16:36,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... +14: [2023-05-11 00:16:36,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +25: [2023-05-11 00:16:36,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +20: [2023-05-11 00:16:36,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +20: [2023-05-11 00:16:36,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +29: [2023-05-11 00:16:36,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +20: [2023-05-11 00:16:36,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +16: [2023-05-11 00:16:36,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +16: [2023-05-11 00:16:36,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +19: [2023-05-11 00:16:36,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +28: [2023-05-11 00:16:36,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +28: [2023-05-11 00:16:36,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +28: [2023-05-11 00:16:36,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +28: [2023-05-11 00:16:36,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +11: [2023-05-11 00:16:36,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +11: [2023-05-11 00:16:36,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +11: [2023-05-11 00:16:36,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +16: [2023-05-11 00:16:36,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +16: [2023-05-11 00:16:36,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +11: [2023-05-11 00:16:36,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +11: [2023-05-11 00:16:36,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +11: [2023-05-11 00:16:36,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +11: [2023-05-11 00:16:36,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +11: [2023-05-11 00:16:36,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +11: [2023-05-11 00:16:36,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +11: [2023-05-11 00:16:36,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 9: [2023-05-11 00:16:36,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +19: [2023-05-11 00:16:36,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +19: [2023-05-11 00:16:36,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +16: [2023-05-11 00:16:36,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +16: [2023-05-11 00:16:36,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +11: [2023-05-11 00:16:36,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +11: [2023-05-11 00:16:36,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +11: [2023-05-11 00:16:36,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +31: [2023-05-11 00:16:36,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. +26: [2023-05-11 00:16:36,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +26: [2023-05-11 00:16:36,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +16: [2023-05-11 00:16:36,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +28: [2023-05-11 00:16:36,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +23: [2023-05-11 00:16:36,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +16: [2023-05-11 00:16:36,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +28: [2023-05-11 00:16:36,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +14: [2023-05-11 00:16:36,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +11: [2023-05-11 00:16:36,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +28: [2023-05-11 00:16:36,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +11: [2023-05-11 00:16:36,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +14: [2023-05-11 00:16:36,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +28: [2023-05-11 00:16:36,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +14: [2023-05-11 00:16:36,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +14: [2023-05-11 00:16:36,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +14: [2023-05-11 00:16:36,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 5: [2023-05-11 00:16:36,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +11: [2023-05-11 00:16:36,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +23: [2023-05-11 00:16:36,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +23: [2023-05-11 00:16:36,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +14: [2023-05-11 00:16:36,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +19: [2023-05-11 00:16:36,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +19: [2023-05-11 00:16:36,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +21: [2023-05-11 00:16:36,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt... +21: [2023-05-11 00:16:36,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt... +19: [2023-05-11 00:16:36,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +21: [2023-05-11 00:16:36,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt... +21: [2023-05-11 00:16:36,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt... + 0: [2023-05-11 00:16:36,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +19: [2023-05-11 00:16:36,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +19: [2023-05-11 00:16:36,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 9: [2023-05-11 00:16:36,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +19: [2023-05-11 00:16:36,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +19: [2023-05-11 00:16:36,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +19: [2023-05-11 00:16:36,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +19: [2023-05-11 00:16:36,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +25: [2023-05-11 00:16:36,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +19: [2023-05-11 00:16:36,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 2: [2023-05-11 00:16:36,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 2: [2023-05-11 00:16:36,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +11: [2023-05-11 00:16:36,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +14: [2023-05-11 00:16:36,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +14: [2023-05-11 00:16:36,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +14: [2023-05-11 00:16:36,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +14: [2023-05-11 00:16:36,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +11: [2023-05-11 00:16:36,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +19: [2023-05-11 00:16:36,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +19: [2023-05-11 00:16:36,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 5: [2023-05-11 00:16:36,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +16: [2023-05-11 00:16:36,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +23: [2023-05-11 00:16:36,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +19: [2023-05-11 00:16:36,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +19: [2023-05-11 00:16:36,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +19: [2023-05-11 00:16:36,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +19: [2023-05-11 00:16:36,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +19: [2023-05-11 00:16:36,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +23: [2023-05-11 00:16:36,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +23: [2023-05-11 00:16:36,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +16: [2023-05-11 00:16:36,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +28: [2023-05-11 00:16:36,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +28: [2023-05-11 00:16:36,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... +11: [2023-05-11 00:16:36,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 5: [2023-05-11 00:16:36,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 5: [2023-05-11 00:16:36,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 9: [2023-05-11 00:16:36,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 9: [2023-05-11 00:16:36,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +24: [2023-05-11 00:16:36,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +24: [2023-05-11 00:16:36,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +24: [2023-05-11 00:16:36,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +24: [2023-05-11 00:16:36,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +23: [2023-05-11 00:16:36,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +28: [2023-05-11 00:16:36,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +14: [2023-05-11 00:16:36,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt... +14: [2023-05-11 00:16:36,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt... +14: [2023-05-11 00:16:36,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt... +14: [2023-05-11 00:16:36,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt... + 9: [2023-05-11 00:16:36,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 3: [2023-05-11 00:16:36,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt... + 9: [2023-05-11 00:16:36,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 9: [2023-05-11 00:16:36,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +28: [2023-05-11 00:16:36,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +16: [2023-05-11 00:16:36,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +16: [2023-05-11 00:16:36,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +19: [2023-05-11 00:16:36,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +11: [2023-05-11 00:16:36,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +16: [2023-05-11 00:16:36,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +19: [2023-05-11 00:16:36,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +19: [2023-05-11 00:16:36,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +19: [2023-05-11 00:16:36,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +19: [2023-05-11 00:16:36,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +19: [2023-05-11 00:16:36,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +28: [2023-05-11 00:16:36,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +28: [2023-05-11 00:16:36,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +11: [2023-05-11 00:16:36,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +28: [2023-05-11 00:16:36,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +28: [2023-05-11 00:16:36,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 0: [2023-05-11 00:16:36,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +28: [2023-05-11 00:16:36,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +28: [2023-05-11 00:16:36,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +28: [2023-05-11 00:16:36,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 3: [2023-05-11 00:16:36,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt... + 3: [2023-05-11 00:16:36,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt... + 3: [2023-05-11 00:16:36,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt... + 3: [2023-05-11 00:16:36,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt... +23: [2023-05-11 00:16:36,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +28: [2023-05-11 00:16:36,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +24: [2023-05-11 00:16:36,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +24: [2023-05-11 00:16:36,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +24: [2023-05-11 00:16:36,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +16: [2023-05-11 00:16:36,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +24: [2023-05-11 00:16:36,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +19: [2023-05-11 00:16:36,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +19: [2023-05-11 00:16:36,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 0: [2023-05-11 00:16:36,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +11: [2023-05-11 00:16:36,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +28: [2023-05-11 00:16:36,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt... + 5: [2023-05-11 00:16:36,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt... +28: [2023-05-11 00:16:36,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +28: [2023-05-11 00:16:36,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt... + 5: [2023-05-11 00:16:36,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt... +28: [2023-05-11 00:16:36,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +10: [2023-05-11 00:16:36,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +10: [2023-05-11 00:16:36,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +10: [2023-05-11 00:16:36,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +10: [2023-05-11 00:16:36,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +10: [2023-05-11 00:16:36,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +10: [2023-05-11 00:16:36,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +16: [2023-05-11 00:16:36,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +11: [2023-05-11 00:16:36,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +28: [2023-05-11 00:16:36,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +10: [2023-05-11 00:16:36,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +28: [2023-05-11 00:16:36,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +28: [2023-05-11 00:16:36,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +28: [2023-05-11 00:16:36,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +19: [2023-05-11 00:16:36,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +10: [2023-05-11 00:16:36,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +19: [2023-05-11 00:16:36,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt... +19: [2023-05-11 00:16:36,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt... +19: [2023-05-11 00:16:36,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt... +19: [2023-05-11 00:16:36,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt... +19: [2023-05-11 00:16:36,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 5: [2023-05-11 00:16:36,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 0: [2023-05-11 00:16:36,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +27: [2023-05-11 00:16:36,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +27: [2023-05-11 00:16:36,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +27: [2023-05-11 00:16:36,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +27: [2023-05-11 00:16:36,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 5: [2023-05-11 00:16:36,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +30: [2023-05-11 00:16:36,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +30: [2023-05-11 00:16:36,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +30: [2023-05-11 00:16:36,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +30: [2023-05-11 00:16:36,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +11: [2023-05-11 00:16:36,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +16: [2023-05-11 00:16:36,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +27: [2023-05-11 00:16:36,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 0: [2023-05-11 00:16:36,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +27: [2023-05-11 00:16:36,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +27: [2023-05-11 00:16:36,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +27: [2023-05-11 00:16:36,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +24: [2023-05-11 00:16:36,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +10: [2023-05-11 00:16:36,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +24: [2023-05-11 00:16:36,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +24: [2023-05-11 00:16:36,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +24: [2023-05-11 00:16:36,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +10: [2023-05-11 00:16:36,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +10: [2023-05-11 00:16:36,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +24: [2023-05-11 00:16:36,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +24: [2023-05-11 00:16:36,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 9: [2023-05-11 00:16:36,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt... + 9: [2023-05-11 00:16:36,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt... + 9: [2023-05-11 00:16:36,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt... + 0: [2023-05-11 00:16:36,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 9: [2023-05-11 00:16:36,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt... + 5: [2023-05-11 00:16:36,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +10: [2023-05-11 00:16:36,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +30: [2023-05-11 00:16:36,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +30: [2023-05-11 00:16:36,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +30: [2023-05-11 00:16:36,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +30: [2023-05-11 00:16:36,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 1: [2023-05-11 00:16:36,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +24: [2023-05-11 00:16:36,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +27: [2023-05-11 00:16:36,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +27: [2023-05-11 00:16:36,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +27: [2023-05-11 00:16:36,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +24: [2023-05-11 00:16:36,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +10: [2023-05-11 00:16:36,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 5: [2023-05-11 00:16:36,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +10: [2023-05-11 00:16:36,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 5: [2023-05-11 00:16:36,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 1: [2023-05-11 00:16:36,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +10: [2023-05-11 00:16:36,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +10: [2023-05-11 00:16:36,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +10: [2023-05-11 00:16:36,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +10: [2023-05-11 00:16:36,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +10: [2023-05-11 00:16:36,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +10: [2023-05-11 00:16:36,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +10: [2023-05-11 00:16:36,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +10: [2023-05-11 00:16:36,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +10: [2023-05-11 00:16:36,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +10: [2023-05-11 00:16:36,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +27: [2023-05-11 00:16:36,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +27: [2023-05-11 00:16:36,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +27: [2023-05-11 00:16:36,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +28: [2023-05-11 00:16:36,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt... +28: [2023-05-11 00:16:36,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt... +28: [2023-05-11 00:16:36,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt... +28: [2023-05-11 00:16:36,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt... +27: [2023-05-11 00:16:36,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +27: [2023-05-11 00:16:36,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +27: [2023-05-11 00:16:36,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +27: [2023-05-11 00:16:36,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +27: [2023-05-11 00:16:36,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +27: [2023-05-11 00:16:36,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +27: [2023-05-11 00:16:36,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 1: [2023-05-11 00:16:36,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +27: [2023-05-11 00:16:36,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +27: [2023-05-11 00:16:36,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +27: [2023-05-11 00:16:36,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +10: [2023-05-11 00:16:36,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +30: [2023-05-11 00:16:36,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +30: [2023-05-11 00:16:36,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +30: [2023-05-11 00:16:36,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +30: [2023-05-11 00:16:36,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +30: [2023-05-11 00:16:36,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +30: [2023-05-11 00:16:36,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +10: [2023-05-11 00:16:36,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +10: [2023-05-11 00:16:36,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +27: [2023-05-11 00:16:36,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +10: [2023-05-11 00:16:36,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +27: [2023-05-11 00:16:36,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +27: [2023-05-11 00:16:36,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +27: [2023-05-11 00:16:36,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 1: [2023-05-11 00:16:36,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 1: [2023-05-11 00:16:36,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 1: [2023-05-11 00:16:37,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 1: [2023-05-11 00:16:37,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +30: [2023-05-11 00:16:37,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +30: [2023-05-11 00:16:37,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +30: [2023-05-11 00:16:37,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +30: [2023-05-11 00:16:37,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +30: [2023-05-11 00:16:37,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +30: [2023-05-11 00:16:37,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +30: [2023-05-11 00:16:37,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +30: [2023-05-11 00:16:37,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +30: [2023-05-11 00:16:37,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +30: [2023-05-11 00:16:37,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 1: [2023-05-11 00:16:37,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 1: [2023-05-11 00:16:37,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +10: [2023-05-11 00:16:37,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt... +10: [2023-05-11 00:16:37,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt... +10: [2023-05-11 00:16:37,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt... +30: [2023-05-11 00:16:37,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +27: [2023-05-11 00:16:37,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt... +27: [2023-05-11 00:16:37,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt... +27: [2023-05-11 00:16:37,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt... +10: [2023-05-11 00:16:37,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt... +30: [2023-05-11 00:16:37,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +27: [2023-05-11 00:16:37,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt... +30: [2023-05-11 00:16:37,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +30: [2023-05-11 00:16:37,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +18: [2023-05-11 00:16:37,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +18: [2023-05-11 00:16:37,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +18: [2023-05-11 00:16:37,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +18: [2023-05-11 00:16:37,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +18: [2023-05-11 00:16:37,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +18: [2023-05-11 00:16:37,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +18: [2023-05-11 00:16:37,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +18: [2023-05-11 00:16:37,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +30: [2023-05-11 00:16:37,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +30: [2023-05-11 00:16:37,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +29: [2023-05-11 00:16:37,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +29: [2023-05-11 00:16:37,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +29: [2023-05-11 00:16:37,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +29: [2023-05-11 00:16:37,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +29: [2023-05-11 00:16:37,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +29: [2023-05-11 00:16:37,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +29: [2023-05-11 00:16:37,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +29: [2023-05-11 00:16:37,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +30: [2023-05-11 00:16:37,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +30: [2023-05-11 00:16:37,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 4: [2023-05-11 00:16:37,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +30: [2023-05-11 00:16:37,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +30: [2023-05-11 00:16:37,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +30: [2023-05-11 00:16:37,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +30: [2023-05-11 00:16:37,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +30: [2023-05-11 00:16:37,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +30: [2023-05-11 00:16:37,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +30: [2023-05-11 00:16:37,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +30: [2023-05-11 00:16:37,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +30: [2023-05-11 00:16:37,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +30: [2023-05-11 00:16:37,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +30: [2023-05-11 00:16:37,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +26: [2023-05-11 00:16:37,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +26: [2023-05-11 00:16:37,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +30: [2023-05-11 00:16:37,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 4: [2023-05-11 00:16:37,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 4: [2023-05-11 00:16:37,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 4: [2023-05-11 00:16:37,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +26: [2023-05-11 00:16:37,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +26: [2023-05-11 00:16:37,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +18: [2023-05-11 00:16:37,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +18: [2023-05-11 00:16:37,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +17: [2023-05-11 00:16:37,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +17: [2023-05-11 00:16:37,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +18: [2023-05-11 00:16:37,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +10: [2023-05-11 00:16:37,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +10: [2023-05-11 00:16:37,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +10: [2023-05-11 00:16:37,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +13: [2023-05-11 00:16:37,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +13: [2023-05-11 00:16:37,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +18: [2023-05-11 00:16:37,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +10: [2023-05-11 00:16:37,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +15: [2023-05-11 00:16:37,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +15: [2023-05-11 00:16:37,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +15: [2023-05-11 00:16:37,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 4: [2023-05-11 00:16:37,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +15: [2023-05-11 00:16:37,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 4: [2023-05-11 00:16:37,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +13: [2023-05-11 00:16:37,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +17: [2023-05-11 00:16:37,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 4: [2023-05-11 00:16:37,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 4: [2023-05-11 00:16:37,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +13: [2023-05-11 00:16:37,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +17: [2023-05-11 00:16:37,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +17: [2023-05-11 00:16:37,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +26: [2023-05-11 00:16:37,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +17: [2023-05-11 00:16:37,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +13: [2023-05-11 00:16:37,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +13: [2023-05-11 00:16:37,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +13: [2023-05-11 00:16:37,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +26: [2023-05-11 00:16:37,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +17: [2023-05-11 00:16:37,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +17: [2023-05-11 00:16:37,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +13: [2023-05-11 00:16:37,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +29: [2023-05-11 00:16:37,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +22: [2023-05-11 00:16:37,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +22: [2023-05-11 00:16:37,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +26: [2023-05-11 00:16:37,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +18: [2023-05-11 00:16:37,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +26: [2023-05-11 00:16:37,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +10: [2023-05-11 00:16:37,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +18: [2023-05-11 00:16:37,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +10: [2023-05-11 00:16:37,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +15: [2023-05-11 00:16:37,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +18: [2023-05-11 00:16:37,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +18: [2023-05-11 00:16:37,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +22: [2023-05-11 00:16:37,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 4: [2023-05-11 00:16:37,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +18: [2023-05-11 00:16:37,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +23: [2023-05-11 00:16:37,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +23: [2023-05-11 00:16:37,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +22: [2023-05-11 00:16:37,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +15: [2023-05-11 00:16:37,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +15: [2023-05-11 00:16:37,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +18: [2023-05-11 00:16:37,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +29: [2023-05-11 00:16:37,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +29: [2023-05-11 00:16:37,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +29: [2023-05-11 00:16:37,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +17: [2023-05-11 00:16:37,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +10: [2023-05-11 00:16:37,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 4: [2023-05-11 00:16:37,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +15: [2023-05-11 00:16:37,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +18: [2023-05-11 00:16:37,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +10: [2023-05-11 00:16:37,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +23: [2023-05-11 00:16:37,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +29: [2023-05-11 00:16:37,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +13: [2023-05-11 00:16:37,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +18: [2023-05-11 00:16:37,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +23: [2023-05-11 00:16:37,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +22: [2023-05-11 00:16:37,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +23: [2023-05-11 00:16:37,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +23: [2023-05-11 00:16:37,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +23: [2023-05-11 00:16:37,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +22: [2023-05-11 00:16:37,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +20: [2023-05-11 00:16:37,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +20: [2023-05-11 00:16:37,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +20: [2023-05-11 00:16:37,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +20: [2023-05-11 00:16:37,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +23: [2023-05-11 00:16:37,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +22: [2023-05-11 00:16:37,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +29: [2023-05-11 00:16:37,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +29: [2023-05-11 00:16:37,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +20: [2023-05-11 00:16:37,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +20: [2023-05-11 00:16:37,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +18: [2023-05-11 00:16:37,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +29: [2023-05-11 00:16:37,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 4: [2023-05-11 00:16:37,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +12: [2023-05-11 00:16:37,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +18: [2023-05-11 00:16:37,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +12: [2023-05-11 00:16:37,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 6: [2023-05-11 00:16:37,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +12: [2023-05-11 00:16:37,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +12: [2023-05-11 00:16:37,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +13: [2023-05-11 00:16:37,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +12: [2023-05-11 00:16:37,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +12: [2023-05-11 00:16:37,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 6: [2023-05-11 00:16:37,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 6: [2023-05-11 00:16:37,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +12: [2023-05-11 00:16:37,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 6: [2023-05-11 00:16:37,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +26: [2023-05-11 00:16:37,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +17: [2023-05-11 00:16:37,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +20: [2023-05-11 00:16:37,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +22: [2023-05-11 00:16:37,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +15: [2023-05-11 00:16:37,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 6: [2023-05-11 00:16:37,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 6: [2023-05-11 00:16:37,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 4: [2023-05-11 00:16:37,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +15: [2023-05-11 00:16:37,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 4: [2023-05-11 00:16:37,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 4: [2023-05-11 00:16:37,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 4: [2023-05-11 00:16:37,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 4: [2023-05-11 00:16:37,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +20: [2023-05-11 00:16:37,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +12: [2023-05-11 00:16:37,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 4: [2023-05-11 00:16:37,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +26: [2023-05-11 00:16:37,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +26: [2023-05-11 00:16:37,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 4: [2023-05-11 00:16:37,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +23: [2023-05-11 00:16:37,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +26: [2023-05-11 00:16:37,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +25: [2023-05-11 00:16:37,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +25: [2023-05-11 00:16:37,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +25: [2023-05-11 00:16:37,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +25: [2023-05-11 00:16:37,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +29: [2023-05-11 00:16:37,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +29: [2023-05-11 00:16:37,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +29: [2023-05-11 00:16:37,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +18: [2023-05-11 00:16:37,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 6: [2023-05-11 00:16:37,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +29: [2023-05-11 00:16:37,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +29: [2023-05-11 00:16:37,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 6: [2023-05-11 00:16:37,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +13: [2023-05-11 00:16:37,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +18: [2023-05-11 00:16:37,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +29: [2023-05-11 00:16:37,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +18: [2023-05-11 00:16:37,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +18: [2023-05-11 00:16:37,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +17: [2023-05-11 00:16:37,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +18: [2023-05-11 00:16:37,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +29: [2023-05-11 00:16:37,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +17: [2023-05-11 00:16:37,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +29: [2023-05-11 00:16:37,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +18: [2023-05-11 00:16:37,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +18: [2023-05-11 00:16:37,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +29: [2023-05-11 00:16:37,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +29: [2023-05-11 00:16:37,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +18: [2023-05-11 00:16:37,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +29: [2023-05-11 00:16:37,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +26: [2023-05-11 00:16:37,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +17: [2023-05-11 00:16:37,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +26: [2023-05-11 00:16:37,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +18: [2023-05-11 00:16:37,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +29: [2023-05-11 00:16:37,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +18: [2023-05-11 00:16:37,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +25: [2023-05-11 00:16:37,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +25: [2023-05-11 00:16:37,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +25: [2023-05-11 00:16:37,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +25: [2023-05-11 00:16:37,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +30: [2023-05-11 00:16:37,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt... +30: [2023-05-11 00:16:37,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt... +30: [2023-05-11 00:16:37,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt... +30: [2023-05-11 00:16:37,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt... +31: [2023-05-11 00:16:37,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +18: [2023-05-11 00:16:37,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +31: [2023-05-11 00:16:37,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +31: [2023-05-11 00:16:37,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +31: [2023-05-11 00:16:37,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +13: [2023-05-11 00:16:37,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +31: [2023-05-11 00:16:37,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +31: [2023-05-11 00:16:37,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +13: [2023-05-11 00:16:37,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +13: [2023-05-11 00:16:37,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +18: [2023-05-11 00:16:37,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +18: [2023-05-11 00:16:37,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +18: [2023-05-11 00:16:37,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +13: [2023-05-11 00:16:37,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 4: [2023-05-11 00:16:37,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +13: [2023-05-11 00:16:37,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +26: [2023-05-11 00:16:37,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +26: [2023-05-11 00:16:37,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +13: [2023-05-11 00:16:37,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +13: [2023-05-11 00:16:37,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +13: [2023-05-11 00:16:37,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +22: [2023-05-11 00:16:37,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +31: [2023-05-11 00:16:37,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +17: [2023-05-11 00:16:37,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +31: [2023-05-11 00:16:37,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +17: [2023-05-11 00:16:37,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +10: [2023-05-11 00:16:37,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +17: [2023-05-11 00:16:37,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +22: [2023-05-11 00:16:37,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +17: [2023-05-11 00:16:37,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +10: [2023-05-11 00:16:37,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +15: [2023-05-11 00:16:37,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +15: [2023-05-11 00:16:37,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +17: [2023-05-11 00:16:37,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +22: [2023-05-11 00:16:37,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +22: [2023-05-11 00:16:37,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +18: [2023-05-11 00:16:37,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 2: [2023-05-11 00:16:37,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +10: [2023-05-11 00:16:37,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +10: [2023-05-11 00:16:37,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +15: [2023-05-11 00:16:37,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +15: [2023-05-11 00:16:37,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 4: [2023-05-11 00:16:37,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +18: [2023-05-11 00:16:37,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +17: [2023-05-11 00:16:37,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +16: [2023-05-11 00:16:37,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +16: [2023-05-11 00:16:37,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +12: [2023-05-11 00:16:37,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 6: [2023-05-11 00:16:37,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +17: [2023-05-11 00:16:37,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +22: [2023-05-11 00:16:37,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +27: [2023-05-11 00:16:37,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +27: [2023-05-11 00:16:37,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +27: [2023-05-11 00:16:37,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +22: [2023-05-11 00:16:37,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +27: [2023-05-11 00:16:37,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +26: [2023-05-11 00:16:37,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +26: [2023-05-11 00:16:37,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +26: [2023-05-11 00:16:37,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +26: [2023-05-11 00:16:37,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +26: [2023-05-11 00:16:37,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +26: [2023-05-11 00:16:37,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +23: [2023-05-11 00:16:37,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +26: [2023-05-11 00:16:37,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +18: [2023-05-11 00:16:37,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +16: [2023-05-11 00:16:37,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +16: [2023-05-11 00:16:37,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 8: [2023-05-11 00:16:37,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 8: [2023-05-11 00:16:37,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 8: [2023-05-11 00:16:37,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 8: [2023-05-11 00:16:37,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 8: [2023-05-11 00:16:37,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 8: [2023-05-11 00:16:37,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 8: [2023-05-11 00:16:37,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +20: [2023-05-11 00:16:37,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +20: [2023-05-11 00:16:37,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +22: [2023-05-11 00:16:37,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +22: [2023-05-11 00:16:37,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +18: [2023-05-11 00:16:37,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 6: [2023-05-11 00:16:37,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +25: [2023-05-11 00:16:37,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +26: [2023-05-11 00:16:37,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +10: [2023-05-11 00:16:37,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +23: [2023-05-11 00:16:37,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +23: [2023-05-11 00:16:37,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +20: [2023-05-11 00:16:37,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +23: [2023-05-11 00:16:37,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +25: [2023-05-11 00:16:37,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +15: [2023-05-11 00:16:37,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +21: [2023-05-11 00:16:37,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +21: [2023-05-11 00:16:37,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +10: [2023-05-11 00:16:37,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +21: [2023-05-11 00:16:37,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +12: [2023-05-11 00:16:37,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +12: [2023-05-11 00:16:37,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +21: [2023-05-11 00:16:37,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +15: [2023-05-11 00:16:37,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 4: [2023-05-11 00:16:37,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 8: [2023-05-11 00:16:37,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +13: [2023-05-11 00:16:37,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 4: [2023-05-11 00:16:37,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +13: [2023-05-11 00:16:37,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 4: [2023-05-11 00:16:37,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +13: [2023-05-11 00:16:37,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 4: [2023-05-11 00:16:37,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +13: [2023-05-11 00:16:37,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +13: [2023-05-11 00:16:37,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +17: [2023-05-11 00:16:37,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 4: [2023-05-11 00:16:37,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +18: [2023-05-11 00:16:37,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 4: [2023-05-11 00:16:37,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 4: [2023-05-11 00:16:37,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +16: [2023-05-11 00:16:37,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 4: [2023-05-11 00:16:37,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +25: [2023-05-11 00:16:37,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +17: [2023-05-11 00:16:37,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 6: [2023-05-11 00:16:37,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 2: [2023-05-11 00:16:37,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +20: [2023-05-11 00:16:37,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +20: [2023-05-11 00:16:37,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +20: [2023-05-11 00:16:37,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +25: [2023-05-11 00:16:37,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +17: [2023-05-11 00:16:37,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +27: [2023-05-11 00:16:37,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +17: [2023-05-11 00:16:37,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +20: [2023-05-11 00:16:37,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +18: [2023-05-11 00:16:37,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +25: [2023-05-11 00:16:37,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +17: [2023-05-11 00:16:37,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +17: [2023-05-11 00:16:37,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 2: [2023-05-11 00:16:37,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +17: [2023-05-11 00:16:37,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +17: [2023-05-11 00:16:37,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +27: [2023-05-11 00:16:37,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +27: [2023-05-11 00:16:37,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +17: [2023-05-11 00:16:37,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +17: [2023-05-11 00:16:37,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +17: [2023-05-11 00:16:37,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +23: [2023-05-11 00:16:37,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +17: [2023-05-11 00:16:37,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +17: [2023-05-11 00:16:37,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +17: [2023-05-11 00:16:37,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +25: [2023-05-11 00:16:37,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +17: [2023-05-11 00:16:37,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +17: [2023-05-11 00:16:37,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +16: [2023-05-11 00:16:37,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 2: [2023-05-11 00:16:37,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +18: [2023-05-11 00:16:37,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +25: [2023-05-11 00:16:37,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +16: [2023-05-11 00:16:37,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +16: [2023-05-11 00:16:37,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +23: [2023-05-11 00:16:37,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 6: [2023-05-11 00:16:37,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 6: [2023-05-11 00:16:37,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +10: [2023-05-11 00:16:37,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 6: [2023-05-11 00:16:37,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 6: [2023-05-11 00:16:37,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +25: [2023-05-11 00:16:37,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +20: [2023-05-11 00:16:37,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 6: [2023-05-11 00:16:37,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 6: [2023-05-11 00:16:37,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +12: [2023-05-11 00:16:37,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +27: [2023-05-11 00:16:37,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +18: [2023-05-11 00:16:37,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 0: [2023-05-11 00:16:37,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 0: [2023-05-11 00:16:37,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 6: [2023-05-11 00:16:37,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +31: [2023-05-11 00:16:37,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +31: [2023-05-11 00:16:37,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +12: [2023-05-11 00:16:37,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +31: [2023-05-11 00:16:37,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +31: [2023-05-11 00:16:37,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +25: [2023-05-11 00:16:37,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +31: [2023-05-11 00:16:37,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +31: [2023-05-11 00:16:37,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +31: [2023-05-11 00:16:37,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +31: [2023-05-11 00:16:37,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +10: [2023-05-11 00:16:37,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +23: [2023-05-11 00:16:37,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +21: [2023-05-11 00:16:37,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +21: [2023-05-11 00:16:37,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +25: [2023-05-11 00:16:37,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +20: [2023-05-11 00:16:37,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +21: [2023-05-11 00:16:37,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 8: [2023-05-11 00:16:37,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +12: [2023-05-11 00:16:37,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +12: [2023-05-11 00:16:37,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +12: [2023-05-11 00:16:37,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +12: [2023-05-11 00:16:37,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +12: [2023-05-11 00:16:37,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +12: [2023-05-11 00:16:37,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +12: [2023-05-11 00:16:37,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +12: [2023-05-11 00:16:37,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +12: [2023-05-11 00:16:37,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +20: [2023-05-11 00:16:37,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +20: [2023-05-11 00:16:37,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +12: [2023-05-11 00:16:37,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +13: [2023-05-11 00:16:37,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +20: [2023-05-11 00:16:37,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +20: [2023-05-11 00:16:37,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +13: [2023-05-11 00:16:37,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +13: [2023-05-11 00:16:37,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +20: [2023-05-11 00:16:37,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +20: [2023-05-11 00:16:37,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +13: [2023-05-11 00:16:37,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +25: [2023-05-11 00:16:37,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +25: [2023-05-11 00:16:37,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +25: [2023-05-11 00:16:37,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +21: [2023-05-11 00:16:37,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +13: [2023-05-11 00:16:37,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +13: [2023-05-11 00:16:37,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +13: [2023-05-11 00:16:37,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +25: [2023-05-11 00:16:37,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +23: [2023-05-11 00:16:37,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +25: [2023-05-11 00:16:37,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 8: [2023-05-11 00:16:37,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +13: [2023-05-11 00:16:37,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +25: [2023-05-11 00:16:37,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +25: [2023-05-11 00:16:37,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +23: [2023-05-11 00:16:37,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +23: [2023-05-11 00:16:37,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +23: [2023-05-11 00:16:37,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +17: [2023-05-11 00:16:37,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +20: [2023-05-11 00:16:37,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +26: [2023-05-11 00:16:37,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +26: [2023-05-11 00:16:37,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +17: [2023-05-11 00:16:37,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +20: [2023-05-11 00:16:37,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +20: [2023-05-11 00:16:37,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +20: [2023-05-11 00:16:37,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +23: [2023-05-11 00:16:37,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 2: [2023-05-11 00:16:37,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +20: [2023-05-11 00:16:37,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +23: [2023-05-11 00:16:37,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +25: [2023-05-11 00:16:37,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +23: [2023-05-11 00:16:37,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +23: [2023-05-11 00:16:37,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +31: [2023-05-11 00:16:37,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 0: [2023-05-11 00:16:37,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 0: [2023-05-11 00:16:37,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +20: [2023-05-11 00:16:37,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +20: [2023-05-11 00:16:37,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +20: [2023-05-11 00:16:37,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +23: [2023-05-11 00:16:37,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +23: [2023-05-11 00:16:37,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +23: [2023-05-11 00:16:37,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 0: [2023-05-11 00:16:37,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +23: [2023-05-11 00:16:37,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +29: [2023-05-11 00:16:37,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt... +29: [2023-05-11 00:16:37,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt... +29: [2023-05-11 00:16:37,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt... +29: [2023-05-11 00:16:37,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt... +18: [2023-05-11 00:16:37,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt... +18: [2023-05-11 00:16:37,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt... +18: [2023-05-11 00:16:37,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt... +18: [2023-05-11 00:16:37,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt... + 2: [2023-05-11 00:16:37,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 2: [2023-05-11 00:16:37,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 2: [2023-05-11 00:16:37,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +25: [2023-05-11 00:16:37,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 6: [2023-05-11 00:16:37,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 6: [2023-05-11 00:16:37,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +16: [2023-05-11 00:16:37,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +16: [2023-05-11 00:16:37,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 8: [2023-05-11 00:16:37,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 8: [2023-05-11 00:16:37,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +16: [2023-05-11 00:16:37,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +16: [2023-05-11 00:16:37,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +17: [2023-05-11 00:16:37,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +16: [2023-05-11 00:16:37,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +16: [2023-05-11 00:16:37,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +17: [2023-05-11 00:16:37,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +25: [2023-05-11 00:16:37,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 2: [2023-05-11 00:16:37,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +25: [2023-05-11 00:16:37,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +25: [2023-05-11 00:16:37,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +31: [2023-05-11 00:16:37,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +31: [2023-05-11 00:16:37,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +31: [2023-05-11 00:16:37,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +31: [2023-05-11 00:16:37,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 4: [2023-05-11 00:16:37,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt... + 4: [2023-05-11 00:16:37,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt... + 4: [2023-05-11 00:16:37,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt... +25: [2023-05-11 00:16:37,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 4: [2023-05-11 00:16:37,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt... +25: [2023-05-11 00:16:37,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +25: [2023-05-11 00:16:37,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +20: [2023-05-11 00:16:37,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 8: [2023-05-11 00:16:37,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +13: [2023-05-11 00:16:37,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +27: [2023-05-11 00:16:37,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +25: [2023-05-11 00:16:37,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +25: [2023-05-11 00:16:37,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +12: [2023-05-11 00:16:37,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +25: [2023-05-11 00:16:37,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +11: [2023-05-11 00:16:37,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +11: [2023-05-11 00:16:37,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +11: [2023-05-11 00:16:37,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +11: [2023-05-11 00:16:37,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +31: [2023-05-11 00:16:37,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +26: [2023-05-11 00:16:37,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +27: [2023-05-11 00:16:37,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +28: [2023-05-11 00:16:37,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +26: [2023-05-11 00:16:37,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +13: [2023-05-11 00:16:37,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +28: [2023-05-11 00:16:37,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +28: [2023-05-11 00:16:37,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +31: [2023-05-11 00:16:37,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +31: [2023-05-11 00:16:37,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +13: [2023-05-11 00:16:37,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 9: [2023-05-11 00:16:37,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 9: [2023-05-11 00:16:37,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 9: [2023-05-11 00:16:37,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +21: [2023-05-11 00:16:37,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +21: [2023-05-11 00:16:37,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 8: [2023-05-11 00:16:37,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +21: [2023-05-11 00:16:37,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +21: [2023-05-11 00:16:37,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +28: [2023-05-11 00:16:37,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 7: [2023-05-11 00:16:37,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 7: [2023-05-11 00:16:37,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 7: [2023-05-11 00:16:37,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +31: [2023-05-11 00:16:37,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +31: [2023-05-11 00:16:37,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +31: [2023-05-11 00:16:37,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +31: [2023-05-11 00:16:37,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +31: [2023-05-11 00:16:37,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 3: [2023-05-11 00:16:37,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +31: [2023-05-11 00:16:37,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +31: [2023-05-11 00:16:37,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 3: [2023-05-11 00:16:37,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. + 3: [2023-05-11 00:16:37,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +27: [2023-05-11 00:16:37,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +31: [2023-05-11 00:16:37,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +13: [2023-05-11 00:16:37,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +27: [2023-05-11 00:16:37,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 7: [2023-05-11 00:16:37,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +31: [2023-05-11 00:16:37,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +11: [2023-05-11 00:16:37,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +11: [2023-05-11 00:16:37,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +17: [2023-05-11 00:16:37,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt... +31: [2023-05-11 00:16:37,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 3: [2023-05-11 00:16:37,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt. +17: [2023-05-11 00:16:37,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt... +17: [2023-05-11 00:16:37,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt... +17: [2023-05-11 00:16:37,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt... +31: [2023-05-11 00:16:37,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +31: [2023-05-11 00:16:37,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +20: [2023-05-11 00:16:37,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +26: [2023-05-11 00:16:37,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +20: [2023-05-11 00:16:37,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +17: [2023-05-11 00:16:37,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +11: [2023-05-11 00:16:37,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +25: [2023-05-11 00:16:37,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +11: [2023-05-11 00:16:37,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 8: [2023-05-11 00:16:37,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +25: [2023-05-11 00:16:37,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 9: [2023-05-11 00:16:37,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +17: [2023-05-11 00:16:37,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +17: [2023-05-11 00:16:37,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +26: [2023-05-11 00:16:37,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +26: [2023-05-11 00:16:37,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 8: [2023-05-11 00:16:37,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 8: [2023-05-11 00:16:37,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 8: [2023-05-11 00:16:37,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 8: [2023-05-11 00:16:37,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +26: [2023-05-11 00:16:37,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +26: [2023-05-11 00:16:37,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 8: [2023-05-11 00:16:37,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 8: [2023-05-11 00:16:37,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +26: [2023-05-11 00:16:37,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +26: [2023-05-11 00:16:37,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +17: [2023-05-11 00:16:37,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 6: [2023-05-11 00:16:37,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 8: [2023-05-11 00:16:37,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 8: [2023-05-11 00:16:37,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 6: [2023-05-11 00:16:37,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 6: [2023-05-11 00:16:37,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 6: [2023-05-11 00:16:37,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 8: [2023-05-11 00:16:37,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +21: [2023-05-11 00:16:37,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +21: [2023-05-11 00:16:37,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 6: [2023-05-11 00:16:37,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 6: [2023-05-11 00:16:37,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 6: [2023-05-11 00:16:37,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 6: [2023-05-11 00:16:37,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +27: [2023-05-11 00:16:37,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +27: [2023-05-11 00:16:37,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +23: [2023-05-11 00:16:37,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt... +23: [2023-05-11 00:16:37,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt... +23: [2023-05-11 00:16:37,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt... +26: [2023-05-11 00:16:37,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +12: [2023-05-11 00:16:37,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +19: [2023-05-11 00:16:37,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +23: [2023-05-11 00:16:37,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt... +19: [2023-05-11 00:16:37,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +26: [2023-05-11 00:16:37,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +26: [2023-05-11 00:16:37,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +12: [2023-05-11 00:16:37,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +12: [2023-05-11 00:16:37,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +26: [2023-05-11 00:16:37,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +12: [2023-05-11 00:16:37,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 0: [2023-05-11 00:16:37,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 0: [2023-05-11 00:16:37,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +28: [2023-05-11 00:16:37,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 9: [2023-05-11 00:16:37,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +19: [2023-05-11 00:16:37,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 0: [2023-05-11 00:16:37,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 9: [2023-05-11 00:16:37,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 9: [2023-05-11 00:16:37,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 9: [2023-05-11 00:16:37,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 7: [2023-05-11 00:16:37,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 2: [2023-05-11 00:16:37,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 9: [2023-05-11 00:16:37,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +13: [2023-05-11 00:16:37,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 9: [2023-05-11 00:16:37,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +26: [2023-05-11 00:16:37,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 7: [2023-05-11 00:16:37,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 3: [2023-05-11 00:16:37,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 7: [2023-05-11 00:16:37,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +27: [2023-05-11 00:16:37,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +27: [2023-05-11 00:16:37,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +31: [2023-05-11 00:16:37,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +25: [2023-05-11 00:16:37,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +25: [2023-05-11 00:16:37,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 2: [2023-05-11 00:16:37,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +28: [2023-05-11 00:16:37,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 3: [2023-05-11 00:16:37,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +31: [2023-05-11 00:16:37,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +20: [2023-05-11 00:16:37,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 3: [2023-05-11 00:16:37,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... + 2: [2023-05-11 00:16:37,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 3: [2023-05-11 00:16:37,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt... +20: [2023-05-11 00:16:37,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 8: [2023-05-11 00:16:37,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +20: [2023-05-11 00:16:37,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +28: [2023-05-11 00:16:37,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +20: [2023-05-11 00:16:37,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 9: [2023-05-11 00:16:37,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 9: [2023-05-11 00:16:37,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 9: [2023-05-11 00:16:37,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +29: [2023-05-11 00:16:37,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +29: [2023-05-11 00:16:37,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 8: [2023-05-11 00:16:37,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +16: [2023-05-11 00:16:37,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +29: [2023-05-11 00:16:37,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 8: [2023-05-11 00:16:37,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +29: [2023-05-11 00:16:37,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +28: [2023-05-11 00:16:37,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 7: [2023-05-11 00:16:37,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 8: [2023-05-11 00:16:37,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 8: [2023-05-11 00:16:37,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 8: [2023-05-11 00:16:37,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 8: [2023-05-11 00:16:37,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +13: [2023-05-11 00:16:37,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +13: [2023-05-11 00:16:37,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +13: [2023-05-11 00:16:37,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +21: [2023-05-11 00:16:37,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 9: [2023-05-11 00:16:37,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 0: [2023-05-11 00:16:37,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 9: [2023-05-11 00:16:37,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 9: [2023-05-11 00:16:37,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +16: [2023-05-11 00:16:37,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +20: [2023-05-11 00:16:37,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +11: [2023-05-11 00:16:37,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +11: [2023-05-11 00:16:37,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +20: [2023-05-11 00:16:37,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +20: [2023-05-11 00:16:37,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +20: [2023-05-11 00:16:37,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +21: [2023-05-11 00:16:37,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 8: [2023-05-11 00:16:37,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +19: [2023-05-11 00:16:37,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 2: [2023-05-11 00:16:37,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 2: [2023-05-11 00:16:37,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +13: [2023-05-11 00:16:37,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt... +13: [2023-05-11 00:16:37,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt... +13: [2023-05-11 00:16:37,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt... +13: [2023-05-11 00:16:37,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt... + 2: [2023-05-11 00:16:37,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +13: [2023-05-11 00:16:37,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +13: [2023-05-11 00:16:37,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 2: [2023-05-11 00:16:37,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 0: [2023-05-11 00:16:37,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +26: [2023-05-11 00:16:37,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +19: [2023-05-11 00:16:37,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +26: [2023-05-11 00:16:37,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +26: [2023-05-11 00:16:37,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 2: [2023-05-11 00:16:37,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +19: [2023-05-11 00:16:37,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +26: [2023-05-11 00:16:37,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +19: [2023-05-11 00:16:37,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 2: [2023-05-11 00:16:37,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +19: [2023-05-11 00:16:37,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +13: [2023-05-11 00:16:37,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +13: [2023-05-11 00:16:37,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +20: [2023-05-11 00:16:37,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +20: [2023-05-11 00:16:37,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +16: [2023-05-11 00:16:37,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +16: [2023-05-11 00:16:37,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +16: [2023-05-11 00:16:37,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +16: [2023-05-11 00:16:37,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +16: [2023-05-11 00:16:37,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +16: [2023-05-11 00:16:37,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +20: [2023-05-11 00:16:37,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt... +29: [2023-05-11 00:16:37,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +20: [2023-05-11 00:16:37,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt... +20: [2023-05-11 00:16:37,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt... +20: [2023-05-11 00:16:37,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt... +29: [2023-05-11 00:16:37,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 8: [2023-05-11 00:16:37,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +12: [2023-05-11 00:16:37,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt... +12: [2023-05-11 00:16:37,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt... +12: [2023-05-11 00:16:37,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt... +12: [2023-05-11 00:16:37,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt... +29: [2023-05-11 00:16:37,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +25: [2023-05-11 00:16:37,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +25: [2023-05-11 00:16:37,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +31: [2023-05-11 00:16:37,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +31: [2023-05-11 00:16:37,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +29: [2023-05-11 00:16:37,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +31: [2023-05-11 00:16:37,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +31: [2023-05-11 00:16:37,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 7: [2023-05-11 00:16:37,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +16: [2023-05-11 00:16:37,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +19: [2023-05-11 00:16:37,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +19: [2023-05-11 00:16:37,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 7: [2023-05-11 00:16:37,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 8: [2023-05-11 00:16:37,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 8: [2023-05-11 00:16:37,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 6: [2023-05-11 00:16:37,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt... + 2: [2023-05-11 00:16:37,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 6: [2023-05-11 00:16:37,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt... + 6: [2023-05-11 00:16:37,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt... + 6: [2023-05-11 00:16:37,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt... +19: [2023-05-11 00:16:37,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +19: [2023-05-11 00:16:37,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +28: [2023-05-11 00:16:37,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +19: [2023-05-11 00:16:37,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 7: [2023-05-11 00:16:37,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 7: [2023-05-11 00:16:37,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +16: [2023-05-11 00:16:37,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +25: [2023-05-11 00:16:37,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +25: [2023-05-11 00:16:37,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +28: [2023-05-11 00:16:37,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 8: [2023-05-11 00:16:37,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +19: [2023-05-11 00:16:37,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +11: [2023-05-11 00:16:37,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 3: [2023-05-11 00:16:37,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +11: [2023-05-11 00:16:37,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +19: [2023-05-11 00:16:37,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +19: [2023-05-11 00:16:37,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 3: [2023-05-11 00:16:37,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 0: [2023-05-11 00:16:37,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +11: [2023-05-11 00:16:37,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +25: [2023-05-11 00:16:37,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt... +25: [2023-05-11 00:16:37,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt... +25: [2023-05-11 00:16:37,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt... +25: [2023-05-11 00:16:37,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt... +11: [2023-05-11 00:16:37,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +28: [2023-05-11 00:16:37,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +28: [2023-05-11 00:16:37,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +28: [2023-05-11 00:16:37,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +28: [2023-05-11 00:16:37,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 3: [2023-05-11 00:16:37,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 3: [2023-05-11 00:16:37,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 7: [2023-05-11 00:16:37,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 7: [2023-05-11 00:16:37,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 7: [2023-05-11 00:16:37,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 7: [2023-05-11 00:16:37,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +31: [2023-05-11 00:16:37,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. +31: [2023-05-11 00:16:37,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt... +31: [2023-05-11 00:16:37,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt... +31: [2023-05-11 00:16:37,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt... +31: [2023-05-11 00:16:37,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt... + 2: [2023-05-11 00:16:37,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 0: [2023-05-11 00:16:37,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +31: [2023-05-11 00:16:37,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 2: [2023-05-11 00:16:37,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +11: [2023-05-11 00:16:37,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +11: [2023-05-11 00:16:37,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 2: [2023-05-11 00:16:37,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +11: [2023-05-11 00:16:37,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +11: [2023-05-11 00:16:37,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 3: [2023-05-11 00:16:37,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 3: [2023-05-11 00:16:37,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 2: [2023-05-11 00:16:37,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +11: [2023-05-11 00:16:37,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +11: [2023-05-11 00:16:37,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... + 2: [2023-05-11 00:16:37,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +16: [2023-05-11 00:16:37,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +11: [2023-05-11 00:16:37,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +11: [2023-05-11 00:16:37,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +11: [2023-05-11 00:16:37,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +11: [2023-05-11 00:16:37,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +29: [2023-05-11 00:16:37,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +29: [2023-05-11 00:16:37,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +29: [2023-05-11 00:16:37,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +29: [2023-05-11 00:16:37,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +16: [2023-05-11 00:16:37,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +11: [2023-05-11 00:16:37,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +11: [2023-05-11 00:16:37,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +11: [2023-05-11 00:16:37,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +11: [2023-05-11 00:16:37,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +29: [2023-05-11 00:16:37,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 8: [2023-05-11 00:16:37,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 8: [2023-05-11 00:16:37,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 8: [2023-05-11 00:16:37,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 8: [2023-05-11 00:16:37,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +26: [2023-05-11 00:16:37,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt... +26: [2023-05-11 00:16:37,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt... +26: [2023-05-11 00:16:37,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt... +28: [2023-05-11 00:16:37,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +26: [2023-05-11 00:16:37,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt... +29: [2023-05-11 00:16:37,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 3: [2023-05-11 00:16:37,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt. + 3: [2023-05-11 00:16:37,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt... +28: [2023-05-11 00:16:37,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +29: [2023-05-11 00:16:37,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +29: [2023-05-11 00:16:37,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 8: [2023-05-11 00:16:37,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt... + 8: [2023-05-11 00:16:37,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt... + 8: [2023-05-11 00:16:37,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt... + 8: [2023-05-11 00:16:37,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt... + 9: [2023-05-11 00:16:37,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 9: [2023-05-11 00:16:37,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 9: [2023-05-11 00:16:37,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +19: [2023-05-11 00:16:37,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +19: [2023-05-11 00:16:37,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 9: [2023-05-11 00:16:37,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +19: [2023-05-11 00:16:37,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +16: [2023-05-11 00:16:37,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +19: [2023-05-11 00:16:37,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 0: [2023-05-11 00:16:37,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 0: [2023-05-11 00:16:37,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 0: [2023-05-11 00:16:37,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +16: [2023-05-11 00:16:37,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 4: [2023-05-11 00:16:37,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 4: [2023-05-11 00:16:37,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 4: [2023-05-11 00:16:37,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 4: [2023-05-11 00:16:37,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +16: [2023-05-11 00:16:37,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 0: [2023-05-11 00:16:37,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 0: [2023-05-11 00:16:37,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 0: [2023-05-11 00:16:37,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 0: [2023-05-11 00:16:37,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 0: [2023-05-11 00:16:37,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +16: [2023-05-11 00:16:37,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +16: [2023-05-11 00:16:37,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 0: [2023-05-11 00:16:37,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +16: [2023-05-11 00:16:37,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +16: [2023-05-11 00:16:37,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 0: [2023-05-11 00:16:37,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +16: [2023-05-11 00:16:37,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +16: [2023-05-11 00:16:37,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt... + 2: [2023-05-11 00:16:37,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt... + 2: [2023-05-11 00:16:37,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt... + 2: [2023-05-11 00:16:37,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt... + 8: [2023-05-11 00:16:37,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 8: [2023-05-11 00:16:37,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 0: > overriding learning rate value to 0.0002 + 0: > overriding minimum learning rate value to 2e-05 + 0: > overriding warmup iterations value to 0 + 0: > overriding total number of iterations value to 1 + 0: > overriding decay style value to cosine +16: [2023-05-11 00:16:37,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +16: [2023-05-11 00:16:37,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +16: [2023-05-11 00:16:37,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +16: [2023-05-11 00:16:37,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +16: [2023-05-11 00:16:37,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 4: [2023-05-11 00:16:37,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 8: [2023-05-11 00:16:37,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 8: [2023-05-11 00:16:37,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 4: [2023-05-11 00:16:37,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 4: [2023-05-11 00:16:37,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 4: [2023-05-11 00:16:37,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +16: [2023-05-11 00:16:37,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +16: [2023-05-11 00:16:37,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +16: [2023-05-11 00:16:37,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +11: [2023-05-11 00:16:37,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt... +11: [2023-05-11 00:16:37,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt... +16: [2023-05-11 00:16:37,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +11: [2023-05-11 00:16:37,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt... +11: [2023-05-11 00:16:37,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt... + 7: [2023-05-11 00:16:37,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt. + 7: [2023-05-11 00:16:37,205] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 56 +11: [2023-05-11 00:16:37,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +11: [2023-05-11 00:16:37,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +11: [2023-05-11 00:16:37,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +11: [2023-05-11 00:16:37,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 7: [2023-05-11 00:16:37,210] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 56 + 4: [2023-05-11 00:16:37,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 4: [2023-05-11 00:16:37,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +16: [2023-05-11 00:16:37,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt... +16: [2023-05-11 00:16:37,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt... +16: [2023-05-11 00:16:37,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt... +16: [2023-05-11 00:16:37,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt... + 0: [2023-05-11 00:16:37,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... + 0: [2023-05-11 00:16:37,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... + 0: [2023-05-11 00:16:37,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... + 0: [2023-05-11 00:16:37,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... + 4: [2023-05-11 00:16:37,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 4: [2023-05-11 00:16:37,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +11: [2023-05-11 00:16:37,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 4: [2023-05-11 00:16:37,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +11: [2023-05-11 00:16:37,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +11: [2023-05-11 00:16:37,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 4: [2023-05-11 00:16:37,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +11: [2023-05-11 00:16:37,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 4: [2023-05-11 00:16:37,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 4: [2023-05-11 00:16:37,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 9: [2023-05-11 00:16:37,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt... + 9: [2023-05-11 00:16:37,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt... + 9: [2023-05-11 00:16:37,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt... + 9: [2023-05-11 00:16:37,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt... +19: [2023-05-11 00:16:37,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt... +19: [2023-05-11 00:16:37,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt... +19: [2023-05-11 00:16:37,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt... +19: [2023-05-11 00:16:37,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt... +11: [2023-05-11 00:16:37,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +11: [2023-05-11 00:16:37,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 7: [2023-05-11 00:16:37,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt. +11: [2023-05-11 00:16:37,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +11: [2023-05-11 00:16:37,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 7: [2023-05-11 00:16:37,241] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 62 + 7: [2023-05-11 00:16:37,246] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 62 +11: [2023-05-11 00:16:37,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +11: [2023-05-11 00:16:37,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 6: [2023-05-11 00:16:37,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 6: [2023-05-11 00:16:37,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 6: [2023-05-11 00:16:37,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 6: [2023-05-11 00:16:37,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +11: [2023-05-11 00:16:37,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +11: [2023-05-11 00:16:37,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 6: [2023-05-11 00:16:37,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 6: [2023-05-11 00:16:37,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 6: [2023-05-11 00:16:37,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 6: [2023-05-11 00:16:37,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +24: [2023-05-11 00:16:37,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt. +24: [2023-05-11 00:16:37,265] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 194 + 5: [2023-05-11 00:16:37,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 5: [2023-05-11 00:16:37,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 5: [2023-05-11 00:16:37,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 5: [2023-05-11 00:16:37,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +24: [2023-05-11 00:16:37,271] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 194 +22: [2023-05-11 00:16:37,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt. +22: [2023-05-11 00:16:37,275] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 176 +24: [2023-05-11 00:16:37,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt. +24: [2023-05-11 00:16:37,276] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 192 + 6: [2023-05-11 00:16:37,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 6: [2023-05-11 00:16:37,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 5: [2023-05-11 00:16:37,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 5: [2023-05-11 00:16:37,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +22: [2023-05-11 00:16:37,280] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 176 + 5: [2023-05-11 00:16:37,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +24: [2023-05-11 00:16:37,281] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 192 + 5: [2023-05-11 00:16:37,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 5: [2023-05-11 00:16:37,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 5: [2023-05-11 00:16:37,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 5: [2023-05-11 00:16:37,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 5: [2023-05-11 00:16:37,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 5: [2023-05-11 00:16:37,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 5: [2023-05-11 00:16:37,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 5: [2023-05-11 00:16:37,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 5: [2023-05-11 00:16:37,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 5: [2023-05-11 00:16:37,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 5: [2023-05-11 00:16:37,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 5: [2023-05-11 00:16:37,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 5: [2023-05-11 00:16:37,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 6: [2023-05-11 00:16:37,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 6: [2023-05-11 00:16:37,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +15: [2023-05-11 00:16:37,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt. +15: [2023-05-11 00:16:37,298] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 120 + 6: [2023-05-11 00:16:37,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 6: [2023-05-11 00:16:37,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +24: [2023-05-11 00:16:37,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt. +24: [2023-05-11 00:16:37,300] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 196 + 6: [2023-05-11 00:16:37,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 6: [2023-05-11 00:16:37,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +15: [2023-05-11 00:16:37,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +15: [2023-05-11 00:16:37,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +15: [2023-05-11 00:16:37,305] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 120 +24: [2023-05-11 00:16:37,306] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 196 +15: [2023-05-11 00:16:37,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +15: [2023-05-11 00:16:37,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 1: [2023-05-11 00:16:37,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 1: [2023-05-11 00:16:37,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 1: [2023-05-11 00:16:37,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +14: [2023-05-11 00:16:37,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +14: [2023-05-11 00:16:37,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +14: [2023-05-11 00:16:37,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +14: [2023-05-11 00:16:37,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 1: [2023-05-11 00:16:37,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +22: [2023-05-11 00:16:37,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt. +22: [2023-05-11 00:16:37,315] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 178 +15: [2023-05-11 00:16:37,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +15: [2023-05-11 00:16:37,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +15: [2023-05-11 00:16:37,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +22: [2023-05-11 00:16:37,320] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 178 +15: [2023-05-11 00:16:37,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +15: [2023-05-11 00:16:37,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +15: [2023-05-11 00:16:37,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +15: [2023-05-11 00:16:37,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +15: [2023-05-11 00:16:37,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +15: [2023-05-11 00:16:37,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +15: [2023-05-11 00:16:37,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +15: [2023-05-11 00:16:37,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +15: [2023-05-11 00:16:37,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +14: [2023-05-11 00:16:37,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +15: [2023-05-11 00:16:37,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +15: [2023-05-11 00:16:37,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +15: [2023-05-11 00:16:37,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +14: [2023-05-11 00:16:37,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 1: [2023-05-11 00:16:37,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +14: [2023-05-11 00:16:37,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +15: [2023-05-11 00:16:37,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +14: [2023-05-11 00:16:37,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 1: [2023-05-11 00:16:37,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 1: [2023-05-11 00:16:37,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 1: [2023-05-11 00:16:37,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +10: [2023-05-11 00:16:37,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +10: [2023-05-11 00:16:37,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +10: [2023-05-11 00:16:37,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 8: [2023-05-11 00:16:37,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 8: [2023-05-11 00:16:37,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 8: [2023-05-11 00:16:37,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 8: [2023-05-11 00:16:37,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +14: [2023-05-11 00:16:37,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +10: [2023-05-11 00:16:37,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 5: [2023-05-11 00:16:37,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt... + 5: [2023-05-11 00:16:37,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt... + 5: [2023-05-11 00:16:37,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt... + 5: [2023-05-11 00:16:37,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt... +14: [2023-05-11 00:16:37,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 1: [2023-05-11 00:16:37,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +10: [2023-05-11 00:16:37,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +14: [2023-05-11 00:16:37,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +14: [2023-05-11 00:16:37,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 1: [2023-05-11 00:16:37,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +10: [2023-05-11 00:16:37,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +30: [2023-05-11 00:16:37,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +30: [2023-05-11 00:16:37,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +30: [2023-05-11 00:16:37,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +10: [2023-05-11 00:16:37,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +30: [2023-05-11 00:16:37,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +10: [2023-05-11 00:16:37,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +14: [2023-05-11 00:16:37,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +14: [2023-05-11 00:16:37,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +10: [2023-05-11 00:16:37,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +10: [2023-05-11 00:16:37,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +21: [2023-05-11 00:16:37,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt. +21: [2023-05-11 00:16:37,345] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 172 +10: [2023-05-11 00:16:37,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +10: [2023-05-11 00:16:37,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +10: [2023-05-11 00:16:37,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 8: [2023-05-11 00:16:37,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 8: [2023-05-11 00:16:37,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 8: [2023-05-11 00:16:37,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 8: [2023-05-11 00:16:37,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 8: [2023-05-11 00:16:37,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +10: [2023-05-11 00:16:37,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 8: [2023-05-11 00:16:37,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 8: [2023-05-11 00:16:37,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 8: [2023-05-11 00:16:37,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +10: [2023-05-11 00:16:37,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +10: [2023-05-11 00:16:37,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +21: [2023-05-11 00:16:37,350] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 172 +10: [2023-05-11 00:16:37,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +10: [2023-05-11 00:16:37,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +10: [2023-05-11 00:16:37,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +10: [2023-05-11 00:16:37,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +30: [2023-05-11 00:16:37,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +24: [2023-05-11 00:16:37,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +24: [2023-05-11 00:16:37,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +24: [2023-05-11 00:16:37,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 1: [2023-05-11 00:16:37,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt. + 8: [2023-05-11 00:16:37,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 8: [2023-05-11 00:16:37,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 8: [2023-05-11 00:16:37,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 8: [2023-05-11 00:16:37,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +24: [2023-05-11 00:16:37,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 1: [2023-05-11 00:16:37,355] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 12 +23: [2023-05-11 00:16:37,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +23: [2023-05-11 00:16:37,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +23: [2023-05-11 00:16:37,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +30: [2023-05-11 00:16:37,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +23: [2023-05-11 00:16:37,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +15: [2023-05-11 00:16:37,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt. +30: [2023-05-11 00:16:37,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +30: [2023-05-11 00:16:37,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +15: [2023-05-11 00:16:37,356] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 124 + 8: [2023-05-11 00:16:37,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 8: [2023-05-11 00:16:37,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 8: [2023-05-11 00:16:37,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 1: [2023-05-11 00:16:37,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 8: [2023-05-11 00:16:37,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +14: [2023-05-11 00:16:37,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 1: [2023-05-11 00:16:37,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +14: [2023-05-11 00:16:37,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 1: [2023-05-11 00:16:37,362] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 12 +15: [2023-05-11 00:16:37,363] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 124 + 1: [2023-05-11 00:16:37,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +24: [2023-05-11 00:16:37,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +24: [2023-05-11 00:16:37,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 1: [2023-05-11 00:16:37,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 1: [2023-05-11 00:16:37,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +23: [2023-05-11 00:16:37,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +24: [2023-05-11 00:16:37,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +24: [2023-05-11 00:16:37,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +24: [2023-05-11 00:16:37,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +24: [2023-05-11 00:16:37,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +29: [2023-05-11 00:16:37,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +29: [2023-05-11 00:16:37,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +29: [2023-05-11 00:16:37,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +24: [2023-05-11 00:16:37,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +23: [2023-05-11 00:16:37,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +15: [2023-05-11 00:16:37,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt. +23: [2023-05-11 00:16:37,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +24: [2023-05-11 00:16:37,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +15: [2023-05-11 00:16:37,370] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 122 +23: [2023-05-11 00:16:37,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +24: [2023-05-11 00:16:37,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +24: [2023-05-11 00:16:37,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +24: [2023-05-11 00:16:37,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +24: [2023-05-11 00:16:37,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 1: [2023-05-11 00:16:37,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +24: [2023-05-11 00:16:37,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 3: [2023-05-11 00:16:37,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt. + 3: [2023-05-11 00:16:37,371] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 26 +24: [2023-05-11 00:16:37,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +24: [2023-05-11 00:16:37,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +29: [2023-05-11 00:16:37,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +24: [2023-05-11 00:16:37,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 9: [2023-05-11 00:16:37,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt. + 9: [2023-05-11 00:16:37,373] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 76 + 1: [2023-05-11 00:16:37,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt. + 1: [2023-05-11 00:16:37,376] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 10 + 3: [2023-05-11 00:16:37,376] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 26 +15: [2023-05-11 00:16:37,376] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 122 +18: [2023-05-11 00:16:37,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +22: [2023-05-11 00:16:37,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +22: [2023-05-11 00:16:37,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +22: [2023-05-11 00:16:37,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +22: [2023-05-11 00:16:37,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 9: [2023-05-11 00:16:37,379] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 76 +18: [2023-05-11 00:16:37,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +29: [2023-05-11 00:16:37,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +29: [2023-05-11 00:16:37,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +29: [2023-05-11 00:16:37,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +18: [2023-05-11 00:16:37,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +29: [2023-05-11 00:16:37,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +15: [2023-05-11 00:16:37,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt... +15: [2023-05-11 00:16:37,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt... +15: [2023-05-11 00:16:37,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt... +15: [2023-05-11 00:16:37,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt... +18: [2023-05-11 00:16:37,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +12: [2023-05-11 00:16:37,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +29: [2023-05-11 00:16:37,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 1: [2023-05-11 00:16:37,381] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 10 +29: [2023-05-11 00:16:37,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 4: [2023-05-11 00:16:37,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 4: [2023-05-11 00:16:37,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +29: [2023-05-11 00:16:37,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +29: [2023-05-11 00:16:37,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +29: [2023-05-11 00:16:37,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +12: [2023-05-11 00:16:37,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +12: [2023-05-11 00:16:37,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +12: [2023-05-11 00:16:37,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +21: [2023-05-11 00:16:37,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt. +21: [2023-05-11 00:16:37,383] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 174 +29: [2023-05-11 00:16:37,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +29: [2023-05-11 00:16:37,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +29: [2023-05-11 00:16:37,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +29: [2023-05-11 00:16:37,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +29: [2023-05-11 00:16:37,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +29: [2023-05-11 00:16:37,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +29: [2023-05-11 00:16:37,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +23: [2023-05-11 00:16:37,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +30: [2023-05-11 00:16:37,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +30: [2023-05-11 00:16:37,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +30: [2023-05-11 00:16:37,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +23: [2023-05-11 00:16:37,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +30: [2023-05-11 00:16:37,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +30: [2023-05-11 00:16:37,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +30: [2023-05-11 00:16:37,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +19: [2023-05-11 00:16:37,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt. +19: [2023-05-11 00:16:37,386] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 156 + 4: [2023-05-11 00:16:37,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 4: [2023-05-11 00:16:37,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +21: [2023-05-11 00:16:37,388] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 174 +22: [2023-05-11 00:16:37,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +30: [2023-05-11 00:16:37,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +30: [2023-05-11 00:16:37,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +18: [2023-05-11 00:16:37,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +22: [2023-05-11 00:16:37,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +22: [2023-05-11 00:16:37,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +12: [2023-05-11 00:16:37,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +18: [2023-05-11 00:16:37,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +19: [2023-05-11 00:16:37,392] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 156 +18: [2023-05-11 00:16:37,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +18: [2023-05-11 00:16:37,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +22: [2023-05-11 00:16:37,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +12: [2023-05-11 00:16:37,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 4: [2023-05-11 00:16:37,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +12: [2023-05-11 00:16:37,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 4: [2023-05-11 00:16:37,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +12: [2023-05-11 00:16:37,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +28: [2023-05-11 00:16:37,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +28: [2023-05-11 00:16:37,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +28: [2023-05-11 00:16:37,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +28: [2023-05-11 00:16:37,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 4: [2023-05-11 00:16:37,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +15: [2023-05-11 00:16:37,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt. + 3: [2023-05-11 00:16:37,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt. +15: [2023-05-11 00:16:37,397] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 126 + 4: [2023-05-11 00:16:37,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 3: [2023-05-11 00:16:37,397] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 28 +23: [2023-05-11 00:16:37,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 4: [2023-05-11 00:16:37,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 4: [2023-05-11 00:16:37,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +23: [2023-05-11 00:16:37,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 4: [2023-05-11 00:16:37,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +23: [2023-05-11 00:16:37,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 4: [2023-05-11 00:16:37,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 4: [2023-05-11 00:16:37,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 4: [2023-05-11 00:16:37,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 4: [2023-05-11 00:16:37,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 4: [2023-05-11 00:16:37,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +23: [2023-05-11 00:16:37,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +15: [2023-05-11 00:16:37,402] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 126 +23: [2023-05-11 00:16:37,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 3: [2023-05-11 00:16:37,403] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 28 + 4: [2023-05-11 00:16:37,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 4: [2023-05-11 00:16:37,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 4: [2023-05-11 00:16:37,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 4: [2023-05-11 00:16:37,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +22: [2023-05-11 00:16:37,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +12: [2023-05-11 00:16:37,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +22: [2023-05-11 00:16:37,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +23: [2023-05-11 00:16:37,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +12: [2023-05-11 00:16:37,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +22: [2023-05-11 00:16:37,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +22: [2023-05-11 00:16:37,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +28: [2023-05-11 00:16:37,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +28: [2023-05-11 00:16:37,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +28: [2023-05-11 00:16:37,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +28: [2023-05-11 00:16:37,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +28: [2023-05-11 00:16:37,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +28: [2023-05-11 00:16:37,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +18: [2023-05-11 00:16:37,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +18: [2023-05-11 00:16:37,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +18: [2023-05-11 00:16:37,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +18: [2023-05-11 00:16:37,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +18: [2023-05-11 00:16:37,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +18: [2023-05-11 00:16:37,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 8: [2023-05-11 00:16:37,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt... + 8: [2023-05-11 00:16:37,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt... + 8: [2023-05-11 00:16:37,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt... + 8: [2023-05-11 00:16:37,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt... +28: [2023-05-11 00:16:37,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +28: [2023-05-11 00:16:37,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +28: [2023-05-11 00:16:37,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +28: [2023-05-11 00:16:37,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +28: [2023-05-11 00:16:37,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +28: [2023-05-11 00:16:37,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +24: [2023-05-11 00:16:37,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt... +24: [2023-05-11 00:16:37,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt... +24: [2023-05-11 00:16:37,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt... +24: [2023-05-11 00:16:37,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt... +11: [2023-05-11 00:16:37,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +11: [2023-05-11 00:16:37,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +14: [2023-05-11 00:16:37,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt. +14: [2023-05-11 00:16:37,413] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 112 +11: [2023-05-11 00:16:37,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +11: [2023-05-11 00:16:37,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +28: [2023-05-11 00:16:37,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +28: [2023-05-11 00:16:37,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +28: [2023-05-11 00:16:37,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +28: [2023-05-11 00:16:37,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +12: [2023-05-11 00:16:37,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +12: [2023-05-11 00:16:37,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +12: [2023-05-11 00:16:37,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +12: [2023-05-11 00:16:37,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 2: [2023-05-11 00:16:37,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 2: [2023-05-11 00:16:37,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 2: [2023-05-11 00:16:37,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 2: [2023-05-11 00:16:37,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +22: [2023-05-11 00:16:37,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +14: [2023-05-11 00:16:37,418] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 112 +22: [2023-05-11 00:16:37,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +26: [2023-05-11 00:16:37,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +26: [2023-05-11 00:16:37,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +26: [2023-05-11 00:16:37,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +26: [2023-05-11 00:16:37,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +18: [2023-05-11 00:16:37,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +12: [2023-05-11 00:16:37,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +12: [2023-05-11 00:16:37,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +18: [2023-05-11 00:16:37,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 7: [2023-05-11 00:16:37,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 7: [2023-05-11 00:16:37,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 3: [2023-05-11 00:16:37,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt. + 3: [2023-05-11 00:16:37,425] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 24 +11: [2023-05-11 00:16:37,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +22: [2023-05-11 00:16:37,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt. +22: [2023-05-11 00:16:37,425] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 182 +11: [2023-05-11 00:16:37,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +11: [2023-05-11 00:16:37,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +11: [2023-05-11 00:16:37,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +11: [2023-05-11 00:16:37,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +11: [2023-05-11 00:16:37,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +16: [2023-05-11 00:16:37,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +16: [2023-05-11 00:16:37,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +16: [2023-05-11 00:16:37,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +22: [2023-05-11 00:16:37,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +21: [2023-05-11 00:16:37,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +21: [2023-05-11 00:16:37,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +21: [2023-05-11 00:16:37,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 7: [2023-05-11 00:16:37,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 7: [2023-05-11 00:16:37,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt. + 7: [2023-05-11 00:16:37,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 0: [2023-05-11 00:16:37,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 2: [2023-05-11 00:16:37,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 0: [2023-05-11 00:16:37,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 0: [2023-05-11 00:16:37,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +11: [2023-05-11 00:16:37,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 7: [2023-05-11 00:16:37,429] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 60 +11: [2023-05-11 00:16:37,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +22: [2023-05-11 00:16:37,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 2: [2023-05-11 00:16:37,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +21: [2023-05-11 00:16:37,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 3: [2023-05-11 00:16:37,430] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 24 +14: [2023-05-11 00:16:37,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt. +22: [2023-05-11 00:16:37,430] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 182 +26: [2023-05-11 00:16:37,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +14: [2023-05-11 00:16:37,430] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 116 + 2: [2023-05-11 00:16:37,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +11: [2023-05-11 00:16:37,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +11: [2023-05-11 00:16:37,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +11: [2023-05-11 00:16:37,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +11: [2023-05-11 00:16:37,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +16: [2023-05-11 00:16:37,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 2: [2023-05-11 00:16:37,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +10: [2023-05-11 00:16:37,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt... +10: [2023-05-11 00:16:37,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt... +10: [2023-05-11 00:16:37,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt... +10: [2023-05-11 00:16:37,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt... + 0: [2023-05-11 00:16:37,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +26: [2023-05-11 00:16:37,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +28: [2023-05-11 00:16:37,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt. +28: [2023-05-11 00:16:37,432] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 228 +26: [2023-05-11 00:16:37,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +26: [2023-05-11 00:16:37,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 7: [2023-05-11 00:16:37,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 7: [2023-05-11 00:16:37,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 7: [2023-05-11 00:16:37,435] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 60 +17: [2023-05-11 00:16:37,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +17: [2023-05-11 00:16:37,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 7: [2023-05-11 00:16:37,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +29: [2023-05-11 00:16:37,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt... +29: [2023-05-11 00:16:37,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt... +29: [2023-05-11 00:16:37,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt... +29: [2023-05-11 00:16:37,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt... + 7: [2023-05-11 00:16:37,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +20: [2023-05-11 00:16:37,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +20: [2023-05-11 00:16:37,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +20: [2023-05-11 00:16:37,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +20: [2023-05-11 00:16:37,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +14: [2023-05-11 00:16:37,437] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 116 +11: [2023-05-11 00:16:37,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +11: [2023-05-11 00:16:37,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +28: [2023-05-11 00:16:37,438] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 228 +11: [2023-05-11 00:16:37,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +17: [2023-05-11 00:16:37,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +21: [2023-05-11 00:16:37,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 7: [2023-05-11 00:16:37,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +21: [2023-05-11 00:16:37,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 7: [2023-05-11 00:16:37,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +21: [2023-05-11 00:16:37,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +11: [2023-05-11 00:16:37,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 5: [2023-05-11 00:16:37,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt. + 0: [2023-05-11 00:16:37,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +27: [2023-05-11 00:16:37,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt. + 5: [2023-05-11 00:16:37,439] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 46 +27: [2023-05-11 00:16:37,439] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 218 + 0: [2023-05-11 00:16:37,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 0: [2023-05-11 00:16:37,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +16: [2023-05-11 00:16:37,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +16: [2023-05-11 00:16:37,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +16: [2023-05-11 00:16:37,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +21: [2023-05-11 00:16:37,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 7: [2023-05-11 00:16:37,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 7: [2023-05-11 00:16:37,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 7: [2023-05-11 00:16:37,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 7: [2023-05-11 00:16:37,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 7: [2023-05-11 00:16:37,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 7: [2023-05-11 00:16:37,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 7: [2023-05-11 00:16:37,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 7: [2023-05-11 00:16:37,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +13: [2023-05-11 00:16:37,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 7: [2023-05-11 00:16:37,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +13: [2023-05-11 00:16:37,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +13: [2023-05-11 00:16:37,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +27: [2023-05-11 00:16:37,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 7: [2023-05-11 00:16:37,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 0: [2023-05-11 00:16:37,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +27: [2023-05-11 00:16:37,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +27: [2023-05-11 00:16:37,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +17: [2023-05-11 00:16:37,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +27: [2023-05-11 00:16:37,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 7: [2023-05-11 00:16:37,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt. + 7: [2023-05-11 00:16:37,444] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 58 + 5: [2023-05-11 00:16:37,444] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 46 +24: [2023-05-11 00:16:37,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt. +24: [2023-05-11 00:16:37,445] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 198 +27: [2023-05-11 00:16:37,445] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 218 +16: [2023-05-11 00:16:37,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +20: [2023-05-11 00:16:37,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +17: [2023-05-11 00:16:37,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +13: [2023-05-11 00:16:37,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +20: [2023-05-11 00:16:37,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +20: [2023-05-11 00:16:37,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +20: [2023-05-11 00:16:37,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 7: [2023-05-11 00:16:37,450] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 58 +24: [2023-05-11 00:16:37,451] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 198 +26: [2023-05-11 00:16:37,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +26: [2023-05-11 00:16:37,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +26: [2023-05-11 00:16:37,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +26: [2023-05-11 00:16:37,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +26: [2023-05-11 00:16:37,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +26: [2023-05-11 00:16:37,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +27: [2023-05-11 00:16:37,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +13: [2023-05-11 00:16:37,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +26: [2023-05-11 00:16:37,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +26: [2023-05-11 00:16:37,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +10: [2023-05-11 00:16:37,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt. +17: [2023-05-11 00:16:37,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +10: [2023-05-11 00:16:37,454] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 82 +21: [2023-05-11 00:16:37,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +13: [2023-05-11 00:16:37,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +31: [2023-05-11 00:16:37,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +31: [2023-05-11 00:16:37,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +31: [2023-05-11 00:16:37,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +21: [2023-05-11 00:16:37,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +13: [2023-05-11 00:16:37,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +27: [2023-05-11 00:16:37,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +27: [2023-05-11 00:16:37,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +17: [2023-05-11 00:16:37,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +21: [2023-05-11 00:16:37,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +21: [2023-05-11 00:16:37,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 0: [2023-05-11 00:16:37,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 5: [2023-05-11 00:16:37,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt. + 5: [2023-05-11 00:16:37,458] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 44 + 9: [2023-05-11 00:16:37,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt. +21: [2023-05-11 00:16:37,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +21: [2023-05-11 00:16:37,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 9: [2023-05-11 00:16:37,458] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 78 +17: [2023-05-11 00:16:37,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 2: [2023-05-11 00:16:37,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +27: [2023-05-11 00:16:37,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +13: [2023-05-11 00:16:37,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +31: [2023-05-11 00:16:37,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 2: [2023-05-11 00:16:37,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 2: [2023-05-11 00:16:37,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +21: [2023-05-11 00:16:37,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +28: [2023-05-11 00:16:37,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt. + 2: [2023-05-11 00:16:37,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +21: [2023-05-11 00:16:37,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +28: [2023-05-11 00:16:37,459] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 230 +10: [2023-05-11 00:16:37,460] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 82 +14: [2023-05-11 00:16:37,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt. +14: [2023-05-11 00:16:37,461] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 118 +16: [2023-05-11 00:16:37,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 4: [2023-05-11 00:16:37,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt... + 4: [2023-05-11 00:16:37,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt... + 4: [2023-05-11 00:16:37,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt... + 4: [2023-05-11 00:16:37,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt... +16: [2023-05-11 00:16:37,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 9: [2023-05-11 00:16:37,463] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 78 + 5: [2023-05-11 00:16:37,464] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 44 + 2: [2023-05-11 00:16:37,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 0: [2023-05-11 00:16:37,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 2: [2023-05-11 00:16:37,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 2: [2023-05-11 00:16:37,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +16: [2023-05-11 00:16:37,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +17: [2023-05-11 00:16:37,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +16: [2023-05-11 00:16:37,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +31: [2023-05-11 00:16:37,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 0: [2023-05-11 00:16:37,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +28: [2023-05-11 00:16:37,466] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 230 +17: [2023-05-11 00:16:37,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +20: [2023-05-11 00:16:37,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +14: [2023-05-11 00:16:37,467] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 118 +20: [2023-05-11 00:16:37,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +13: [2023-05-11 00:16:37,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +31: [2023-05-11 00:16:37,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +31: [2023-05-11 00:16:37,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +31: [2023-05-11 00:16:37,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +14: [2023-05-11 00:16:37,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt. +14: [2023-05-11 00:16:37,470] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 114 +13: [2023-05-11 00:16:37,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 3: [2023-05-11 00:16:37,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 3: [2023-05-11 00:16:37,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. + 3: [2023-05-11 00:16:37,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +27: [2023-05-11 00:16:37,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt. +16: [2023-05-11 00:16:37,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +27: [2023-05-11 00:16:37,472] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 220 +16: [2023-05-11 00:16:37,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +17: [2023-05-11 00:16:37,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +17: [2023-05-11 00:16:37,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 3: [2023-05-11 00:16:37,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +20: [2023-05-11 00:16:37,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +20: [2023-05-11 00:16:37,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +20: [2023-05-11 00:16:37,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +20: [2023-05-11 00:16:37,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +20: [2023-05-11 00:16:37,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +20: [2023-05-11 00:16:37,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +16: [2023-05-11 00:16:37,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +13: [2023-05-11 00:16:37,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +13: [2023-05-11 00:16:37,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +16: [2023-05-11 00:16:37,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +17: [2023-05-11 00:16:37,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +17: [2023-05-11 00:16:37,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +14: [2023-05-11 00:16:37,476] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 114 +25: [2023-05-11 00:16:37,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +25: [2023-05-11 00:16:37,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +13: [2023-05-11 00:16:37,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +27: [2023-05-11 00:16:37,479] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 220 +25: [2023-05-11 00:16:37,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +25: [2023-05-11 00:16:37,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt. +13: [2023-05-11 00:16:37,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +13: [2023-05-11 00:16:37,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +13: [2023-05-11 00:16:37,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 7: [2023-05-11 00:16:37,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt... + 7: [2023-05-11 00:16:37,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt... + 7: [2023-05-11 00:16:37,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt... + 7: [2023-05-11 00:16:37,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt... +27: [2023-05-11 00:16:37,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +31: [2023-05-11 00:16:37,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +27: [2023-05-11 00:16:37,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 1: [2023-05-11 00:16:37,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 3: [2023-05-11 00:16:37,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 3: [2023-05-11 00:16:37,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +28: [2023-05-11 00:16:37,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt... +31: [2023-05-11 00:16:37,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +28: [2023-05-11 00:16:37,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt... +28: [2023-05-11 00:16:37,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt... +28: [2023-05-11 00:16:37,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt... +27: [2023-05-11 00:16:37,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +27: [2023-05-11 00:16:37,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +25: [2023-05-11 00:16:37,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 3: [2023-05-11 00:16:37,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 1: [2023-05-11 00:16:37,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 3: [2023-05-11 00:16:37,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 6: [2023-05-11 00:16:37,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +11: [2023-05-11 00:16:37,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt... +11: [2023-05-11 00:16:37,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt... +11: [2023-05-11 00:16:37,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt... +11: [2023-05-11 00:16:37,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt... +25: [2023-05-11 00:16:37,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... + 6: [2023-05-11 00:16:37,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +21: [2023-05-11 00:16:37,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt. +21: [2023-05-11 00:16:37,488] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 170 +10: [2023-05-11 00:16:37,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt. +10: [2023-05-11 00:16:37,489] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 84 +27: [2023-05-11 00:16:37,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 1: [2023-05-11 00:16:37,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +27: [2023-05-11 00:16:37,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 1: [2023-05-11 00:16:37,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt. + 1: [2023-05-11 00:16:37,491] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 8 +17: [2023-05-11 00:16:37,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +27: [2023-05-11 00:16:37,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 1: [2023-05-11 00:16:37,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +27: [2023-05-11 00:16:37,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 6: [2023-05-11 00:16:37,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +31: [2023-05-11 00:16:37,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +31: [2023-05-11 00:16:37,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +31: [2023-05-11 00:16:37,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +31: [2023-05-11 00:16:37,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +25: [2023-05-11 00:16:37,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +25: [2023-05-11 00:16:37,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt... +21: [2023-05-11 00:16:37,493] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 170 + 3: [2023-05-11 00:16:37,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt. + 3: [2023-05-11 00:16:37,494] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 30 +17: [2023-05-11 00:16:37,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +10: [2023-05-11 00:16:37,495] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 84 + 6: [2023-05-11 00:16:37,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 1: [2023-05-11 00:16:37,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 1: [2023-05-11 00:16:37,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 6: [2023-05-11 00:16:37,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 6: [2023-05-11 00:16:37,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 6: [2023-05-11 00:16:37,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +31: [2023-05-11 00:16:37,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +31: [2023-05-11 00:16:37,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 6: [2023-05-11 00:16:37,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 6: [2023-05-11 00:16:37,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 6: [2023-05-11 00:16:37,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 1: [2023-05-11 00:16:37,500] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 8 + 1: [2023-05-11 00:16:37,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 1: [2023-05-11 00:16:37,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 1: [2023-05-11 00:16:37,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 1: [2023-05-11 00:16:37,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 3: [2023-05-11 00:16:37,501] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 30 +25: [2023-05-11 00:16:37,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 3: [2023-05-11 00:16:37,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 1: [2023-05-11 00:16:37,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 6: [2023-05-11 00:16:37,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 1: [2023-05-11 00:16:37,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 6: [2023-05-11 00:16:37,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 1: [2023-05-11 00:16:37,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 3: [2023-05-11 00:16:37,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +25: [2023-05-11 00:16:37,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 3: [2023-05-11 00:16:37,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 3: [2023-05-11 00:16:37,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +25: [2023-05-11 00:16:37,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +25: [2023-05-11 00:16:37,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 1: [2023-05-11 00:16:37,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 6: [2023-05-11 00:16:37,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 1: [2023-05-11 00:16:37,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 3: [2023-05-11 00:16:37,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 1: [2023-05-11 00:16:37,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 3: [2023-05-11 00:16:37,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... + 6: [2023-05-11 00:16:37,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 6: [2023-05-11 00:16:37,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 6: [2023-05-11 00:16:37,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 6: [2023-05-11 00:16:37,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 1: [2023-05-11 00:16:37,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 1: [2023-05-11 00:16:37,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 1: [2023-05-11 00:16:37,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 6: [2023-05-11 00:16:37,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 6: [2023-05-11 00:16:37,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 6: [2023-05-11 00:16:37,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 1: [2023-05-11 00:16:37,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +19: [2023-05-11 00:16:37,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt. +19: [2023-05-11 00:16:37,509] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 152 +25: [2023-05-11 00:16:37,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +25: [2023-05-11 00:16:37,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +19: [2023-05-11 00:16:37,515] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 152 + 3: [2023-05-11 00:16:37,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. + 3: [2023-05-11 00:16:37,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +25: [2023-05-11 00:16:37,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt. +25: [2023-05-11 00:16:37,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt... +14: [2023-05-11 00:16:37,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +14: [2023-05-11 00:16:37,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +14: [2023-05-11 00:16:37,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +14: [2023-05-11 00:16:37,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +19: [2023-05-11 00:16:37,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt. +19: [2023-05-11 00:16:37,526] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 154 +19: [2023-05-11 00:16:37,532] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 154 +14: [2023-05-11 00:16:37,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +14: [2023-05-11 00:16:37,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +14: [2023-05-11 00:16:37,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +14: [2023-05-11 00:16:37,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +14: [2023-05-11 00:16:37,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +30: [2023-05-11 00:16:37,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +30: [2023-05-11 00:16:37,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +14: [2023-05-11 00:16:37,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +14: [2023-05-11 00:16:37,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +14: [2023-05-11 00:16:37,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +14: [2023-05-11 00:16:37,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +30: [2023-05-11 00:16:37,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +30: [2023-05-11 00:16:37,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +14: [2023-05-11 00:16:37,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +14: [2023-05-11 00:16:37,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +14: [2023-05-11 00:16:37,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +14: [2023-05-11 00:16:37,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +14: [2023-05-11 00:16:37,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +14: [2023-05-11 00:16:37,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +14: [2023-05-11 00:16:37,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 9: [2023-05-11 00:16:37,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt. + 9: [2023-05-11 00:16:37,542] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 72 + 1: [2023-05-11 00:16:37,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt... + 1: [2023-05-11 00:16:37,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt... + 1: [2023-05-11 00:16:37,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt... + 1: [2023-05-11 00:16:37,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt... +30: [2023-05-11 00:16:37,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 9: [2023-05-11 00:16:37,548] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 72 +30: [2023-05-11 00:16:37,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +30: [2023-05-11 00:16:37,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +21: [2023-05-11 00:16:37,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt. +21: [2023-05-11 00:16:37,550] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 168 +30: [2023-05-11 00:16:37,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +30: [2023-05-11 00:16:37,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +30: [2023-05-11 00:16:37,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +30: [2023-05-11 00:16:37,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +30: [2023-05-11 00:16:37,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +30: [2023-05-11 00:16:37,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +30: [2023-05-11 00:16:37,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +30: [2023-05-11 00:16:37,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +30: [2023-05-11 00:16:37,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 1: [2023-05-11 00:16:37,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt. +13: [2023-05-11 00:16:37,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt. +13: [2023-05-11 00:16:37,554] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 104 + 1: [2023-05-11 00:16:37,554] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 14 +20: [2023-05-11 00:16:37,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt. +20: [2023-05-11 00:16:37,555] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 166 +21: [2023-05-11 00:16:37,555] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 168 +23: [2023-05-11 00:16:37,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +30: [2023-05-11 00:16:37,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +30: [2023-05-11 00:16:37,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +30: [2023-05-11 00:16:37,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +30: [2023-05-11 00:16:37,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 4: [2023-05-11 00:16:37,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt. + 4: [2023-05-11 00:16:37,558] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 32 +23: [2023-05-11 00:16:37,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +13: [2023-05-11 00:16:37,560] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 104 +20: [2023-05-11 00:16:37,560] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 166 + 1: [2023-05-11 00:16:37,561] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 14 +19: [2023-05-11 00:16:37,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt. +19: [2023-05-11 00:16:37,562] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 158 + 9: [2023-05-11 00:16:37,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt. + 9: [2023-05-11 00:16:37,562] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 74 + 4: [2023-05-11 00:16:37,563] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 32 +28: [2023-05-11 00:16:37,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt. +28: [2023-05-11 00:16:37,563] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 224 +23: [2023-05-11 00:16:37,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 9: [2023-05-11 00:16:37,567] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 74 +22: [2023-05-11 00:16:37,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +22: [2023-05-11 00:16:37,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +19: [2023-05-11 00:16:37,568] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 158 +10: [2023-05-11 00:16:37,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt. +10: [2023-05-11 00:16:37,568] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 86 +28: [2023-05-11 00:16:37,569] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 224 +12: [2023-05-11 00:16:37,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt. +12: [2023-05-11 00:16:37,570] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 98 +23: [2023-05-11 00:16:37,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +23: [2023-05-11 00:16:37,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +23: [2023-05-11 00:16:37,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +28: [2023-05-11 00:16:37,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt. +23: [2023-05-11 00:16:37,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +28: [2023-05-11 00:16:37,571] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 226 +22: [2023-05-11 00:16:37,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt. +22: [2023-05-11 00:16:37,571] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 180 +23: [2023-05-11 00:16:37,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +22: [2023-05-11 00:16:37,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +22: [2023-05-11 00:16:37,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +23: [2023-05-11 00:16:37,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +23: [2023-05-11 00:16:37,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +14: [2023-05-11 00:16:37,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt... +14: [2023-05-11 00:16:37,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt... +14: [2023-05-11 00:16:37,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt... +14: [2023-05-11 00:16:37,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt... +10: [2023-05-11 00:16:37,575] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 86 +12: [2023-05-11 00:16:37,575] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 98 +23: [2023-05-11 00:16:37,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +23: [2023-05-11 00:16:37,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +23: [2023-05-11 00:16:37,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +22: [2023-05-11 00:16:37,577] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 180 +23: [2023-05-11 00:16:37,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +23: [2023-05-11 00:16:37,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +23: [2023-05-11 00:16:37,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +18: [2023-05-11 00:16:37,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt. +18: [2023-05-11 00:16:37,578] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 146 +28: [2023-05-11 00:16:37,579] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 226 +22: [2023-05-11 00:16:37,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +22: [2023-05-11 00:16:37,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +22: [2023-05-11 00:16:37,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +22: [2023-05-11 00:16:37,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +22: [2023-05-11 00:16:37,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +13: [2023-05-11 00:16:37,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt. +22: [2023-05-11 00:16:37,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +13: [2023-05-11 00:16:37,581] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 110 +23: [2023-05-11 00:16:37,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +18: [2023-05-11 00:16:37,583] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 146 +23: [2023-05-11 00:16:37,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +23: [2023-05-11 00:16:37,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +22: [2023-05-11 00:16:37,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +22: [2023-05-11 00:16:37,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +23: [2023-05-11 00:16:37,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +22: [2023-05-11 00:16:37,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +22: [2023-05-11 00:16:37,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +22: [2023-05-11 00:16:37,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +22: [2023-05-11 00:16:37,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +22: [2023-05-11 00:16:37,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +22: [2023-05-11 00:16:37,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +22: [2023-05-11 00:16:37,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +22: [2023-05-11 00:16:37,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +13: [2023-05-11 00:16:37,586] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 110 +29: [2023-05-11 00:16:37,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt. +29: [2023-05-11 00:16:37,587] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 238 + 6: [2023-05-11 00:16:37,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt... + 6: [2023-05-11 00:16:37,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt... + 6: [2023-05-11 00:16:37,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt... + 6: [2023-05-11 00:16:37,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt... +12: [2023-05-11 00:16:37,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +12: [2023-05-11 00:16:37,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +12: [2023-05-11 00:16:37,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +12: [2023-05-11 00:16:37,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +29: [2023-05-11 00:16:37,592] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 238 + 5: [2023-05-11 00:16:37,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt. + 5: [2023-05-11 00:16:37,595] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 42 + 5: [2023-05-11 00:16:37,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt. + 5: [2023-05-11 00:16:37,597] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 40 + 5: [2023-05-11 00:16:37,600] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 42 +12: [2023-05-11 00:16:37,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +30: [2023-05-11 00:16:37,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt. +30: [2023-05-11 00:16:37,602] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 242 +12: [2023-05-11 00:16:37,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +12: [2023-05-11 00:16:37,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 5: [2023-05-11 00:16:37,603] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 40 +12: [2023-05-11 00:16:37,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +12: [2023-05-11 00:16:37,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +12: [2023-05-11 00:16:37,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +31: [2023-05-11 00:16:37,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt. +31: [2023-05-11 00:16:37,605] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 252 +12: [2023-05-11 00:16:37,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +12: [2023-05-11 00:16:37,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +25: [2023-05-11 00:16:37,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt. +25: [2023-05-11 00:16:37,606] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 202 +30: [2023-05-11 00:16:37,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt... +30: [2023-05-11 00:16:37,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt... +30: [2023-05-11 00:16:37,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt... +30: [2023-05-11 00:16:37,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt... +12: [2023-05-11 00:16:37,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +12: [2023-05-11 00:16:37,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +12: [2023-05-11 00:16:37,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +12: [2023-05-11 00:16:37,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +30: [2023-05-11 00:16:37,609] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 242 +25: [2023-05-11 00:16:37,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt. +25: [2023-05-11 00:16:37,609] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 204 +21: [2023-05-11 00:16:37,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +21: [2023-05-11 00:16:37,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +31: [2023-05-11 00:16:37,610] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 252 +21: [2023-05-11 00:16:37,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +21: [2023-05-11 00:16:37,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +25: [2023-05-11 00:16:37,611] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 202 +12: [2023-05-11 00:16:37,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +12: [2023-05-11 00:16:37,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +12: [2023-05-11 00:16:37,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +12: [2023-05-11 00:16:37,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +25: [2023-05-11 00:16:37,616] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 204 +22: [2023-05-11 00:16:37,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt... +22: [2023-05-11 00:16:37,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt... +22: [2023-05-11 00:16:37,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt... +22: [2023-05-11 00:16:37,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt... +29: [2023-05-11 00:16:37,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt. +29: [2023-05-11 00:16:37,620] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 232 +29: [2023-05-11 00:16:37,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt. +29: [2023-05-11 00:16:37,620] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 234 +26: [2023-05-11 00:16:37,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt. +26: [2023-05-11 00:16:37,620] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 212 +21: [2023-05-11 00:16:37,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +18: [2023-05-11 00:16:37,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt. +21: [2023-05-11 00:16:37,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +18: [2023-05-11 00:16:37,623] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 144 +21: [2023-05-11 00:16:37,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +21: [2023-05-11 00:16:37,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +21: [2023-05-11 00:16:37,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +21: [2023-05-11 00:16:37,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +21: [2023-05-11 00:16:37,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +21: [2023-05-11 00:16:37,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +21: [2023-05-11 00:16:37,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +31: [2023-05-11 00:16:37,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt. +31: [2023-05-11 00:16:37,624] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 250 +21: [2023-05-11 00:16:37,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +26: [2023-05-11 00:16:37,625] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 212 +21: [2023-05-11 00:16:37,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +21: [2023-05-11 00:16:37,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +21: [2023-05-11 00:16:37,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +21: [2023-05-11 00:16:37,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +21: [2023-05-11 00:16:37,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +29: [2023-05-11 00:16:37,626] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 234 +29: [2023-05-11 00:16:37,626] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 232 +21: [2023-05-11 00:16:37,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +18: [2023-05-11 00:16:37,628] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 144 +23: [2023-05-11 00:16:37,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt. +31: [2023-05-11 00:16:37,629] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 250 +23: [2023-05-11 00:16:37,629] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 184 +31: [2023-05-11 00:16:37,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt. +31: [2023-05-11 00:16:37,632] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 254 + 2: [2023-05-11 00:16:37,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 2: [2023-05-11 00:16:37,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 2: [2023-05-11 00:16:37,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 2: [2023-05-11 00:16:37,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +23: [2023-05-11 00:16:37,634] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 184 +31: [2023-05-11 00:16:37,637] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 254 +23: [2023-05-11 00:16:37,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt. +23: [2023-05-11 00:16:37,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt. +23: [2023-05-11 00:16:37,640] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 186 +23: [2023-05-11 00:16:37,641] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 188 + 2: [2023-05-11 00:16:37,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +23: [2023-05-11 00:16:37,646] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 188 + 2: [2023-05-11 00:16:37,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +23: [2023-05-11 00:16:37,646] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 186 + 2: [2023-05-11 00:16:37,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 2: [2023-05-11 00:16:37,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 2: [2023-05-11 00:16:37,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 2: [2023-05-11 00:16:37,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 2: [2023-05-11 00:16:37,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 8: [2023-05-11 00:16:37,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt. + 8: [2023-05-11 00:16:37,650] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 64 + 8: [2023-05-11 00:16:37,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt. + 8: [2023-05-11 00:16:37,651] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 70 + 2: [2023-05-11 00:16:37,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 2: [2023-05-11 00:16:37,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 2: [2023-05-11 00:16:37,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 2: [2023-05-11 00:16:37,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +26: [2023-05-11 00:16:37,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt. +26: [2023-05-11 00:16:37,652] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 210 +20: [2023-05-11 00:16:37,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt. +20: [2023-05-11 00:16:37,654] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 164 + 0: [2023-05-11 00:16:37,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 0: [2023-05-11 00:16:37,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 0: [2023-05-11 00:16:37,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 0: [2023-05-11 00:16:37,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +21: [2023-05-11 00:16:37,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt... +21: [2023-05-11 00:16:37,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt... +21: [2023-05-11 00:16:37,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt... +21: [2023-05-11 00:16:37,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt... +16: [2023-05-11 00:16:37,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +23: [2023-05-11 00:16:37,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt... +23: [2023-05-11 00:16:37,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt... +23: [2023-05-11 00:16:37,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt... +23: [2023-05-11 00:16:37,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt... +18: [2023-05-11 00:16:37,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +18: [2023-05-11 00:16:37,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +18: [2023-05-11 00:16:37,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 8: [2023-05-11 00:16:37,656] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 64 +26: [2023-05-11 00:16:37,658] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 210 + 8: [2023-05-11 00:16:37,658] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 70 +18: [2023-05-11 00:16:37,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +16: [2023-05-11 00:16:37,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +20: [2023-05-11 00:16:37,659] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 164 + 2: [2023-05-11 00:16:37,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 2: [2023-05-11 00:16:37,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 2: [2023-05-11 00:16:37,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +17: [2023-05-11 00:16:37,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt. +17: [2023-05-11 00:16:37,662] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 140 + 2: [2023-05-11 00:16:37,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +12: [2023-05-11 00:16:37,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt... +12: [2023-05-11 00:16:37,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt... +12: [2023-05-11 00:16:37,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt... +12: [2023-05-11 00:16:37,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt... +16: [2023-05-11 00:16:37,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +30: [2023-05-11 00:16:37,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt. +30: [2023-05-11 00:16:37,665] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 244 + 0: [2023-05-11 00:16:37,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +17: [2023-05-11 00:16:37,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt. + 0: [2023-05-11 00:16:37,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +17: [2023-05-11 00:16:37,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +17: [2023-05-11 00:16:37,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +17: [2023-05-11 00:16:37,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +16: [2023-05-11 00:16:37,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 0: [2023-05-11 00:16:37,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +17: [2023-05-11 00:16:37,666] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 142 + 0: [2023-05-11 00:16:37,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +16: [2023-05-11 00:16:37,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +17: [2023-05-11 00:16:37,668] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 140 + 0: [2023-05-11 00:16:37,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +16: [2023-05-11 00:16:37,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +18: [2023-05-11 00:16:37,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +18: [2023-05-11 00:16:37,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +18: [2023-05-11 00:16:37,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +18: [2023-05-11 00:16:37,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 0: [2023-05-11 00:16:37,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +30: [2023-05-11 00:16:37,670] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 244 +18: [2023-05-11 00:16:37,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +18: [2023-05-11 00:16:37,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +18: [2023-05-11 00:16:37,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +16: [2023-05-11 00:16:37,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +18: [2023-05-11 00:16:37,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +18: [2023-05-11 00:16:37,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +16: [2023-05-11 00:16:37,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +27: [2023-05-11 00:16:37,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt. +27: [2023-05-11 00:16:37,671] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 216 +26: [2023-05-11 00:16:37,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt. +26: [2023-05-11 00:16:37,672] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 214 +18: [2023-05-11 00:16:37,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +18: [2023-05-11 00:16:37,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +18: [2023-05-11 00:16:37,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +18: [2023-05-11 00:16:37,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +18: [2023-05-11 00:16:37,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +18: [2023-05-11 00:16:37,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +17: [2023-05-11 00:16:37,673] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 142 +16: [2023-05-11 00:16:37,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +16: [2023-05-11 00:16:37,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +18: [2023-05-11 00:16:37,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 0: [2023-05-11 00:16:37,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 0: [2023-05-11 00:16:37,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 0: [2023-05-11 00:16:37,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 0: [2023-05-11 00:16:37,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 6: [2023-05-11 00:16:37,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt. + 6: [2023-05-11 00:16:37,674] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 52 +23: [2023-05-11 00:16:37,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt. +17: [2023-05-11 00:16:37,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +16: [2023-05-11 00:16:37,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +23: [2023-05-11 00:16:37,675] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 190 +26: [2023-05-11 00:16:37,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +26: [2023-05-11 00:16:37,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +26: [2023-05-11 00:16:37,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +16: [2023-05-11 00:16:37,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +16: [2023-05-11 00:16:37,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 8: [2023-05-11 00:16:37,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt. +26: [2023-05-11 00:16:37,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 8: [2023-05-11 00:16:37,676] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 66 + 0: [2023-05-11 00:16:37,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 0: [2023-05-11 00:16:37,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 0: [2023-05-11 00:16:37,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 0: [2023-05-11 00:16:37,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +27: [2023-05-11 00:16:37,677] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 216 +17: [2023-05-11 00:16:37,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +18: [2023-05-11 00:16:37,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt. +17: [2023-05-11 00:16:37,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +26: [2023-05-11 00:16:37,678] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 214 +18: [2023-05-11 00:16:37,678] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 148 +17: [2023-05-11 00:16:37,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +16: [2023-05-11 00:16:37,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +17: [2023-05-11 00:16:37,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +16: [2023-05-11 00:16:37,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +16: [2023-05-11 00:16:37,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +16: [2023-05-11 00:16:37,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +16: [2023-05-11 00:16:37,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +16: [2023-05-11 00:16:37,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +16: [2023-05-11 00:16:37,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +17: [2023-05-11 00:16:37,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +17: [2023-05-11 00:16:37,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +17: [2023-05-11 00:16:37,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +17: [2023-05-11 00:16:37,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +31: [2023-05-11 00:16:37,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +17: [2023-05-11 00:16:37,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 9: [2023-05-11 00:16:37,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt. +13: [2023-05-11 00:16:37,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +13: [2023-05-11 00:16:37,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 9: [2023-05-11 00:16:37,680] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 75 +23: [2023-05-11 00:16:37,680] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 190 +13: [2023-05-11 00:16:37,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +31: [2023-05-11 00:16:37,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +31: [2023-05-11 00:16:37,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +31: [2023-05-11 00:16:37,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 6: [2023-05-11 00:16:37,680] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 52 + 8: [2023-05-11 00:16:37,681] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 66 + 3: [2023-05-11 00:16:37,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 3: [2023-05-11 00:16:37,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 3: [2023-05-11 00:16:37,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. + 3: [2023-05-11 00:16:37,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +13: [2023-05-11 00:16:37,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +17: [2023-05-11 00:16:37,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +17: [2023-05-11 00:16:37,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +18: [2023-05-11 00:16:37,683] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 148 + 0: [2023-05-11 00:16:37,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. +17: [2023-05-11 00:16:37,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 0: [2023-05-11 00:16:37,683] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 6 +12: [2023-05-11 00:16:37,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt. +12: [2023-05-11 00:16:37,684] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 96 + 8: [2023-05-11 00:16:37,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt. + 8: [2023-05-11 00:16:37,684] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 68 +26: [2023-05-11 00:16:37,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +20: [2023-05-11 00:16:37,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +20: [2023-05-11 00:16:37,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +20: [2023-05-11 00:16:37,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +26: [2023-05-11 00:16:37,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +26: [2023-05-11 00:16:37,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 9: [2023-05-11 00:16:37,686] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 75 +17: [2023-05-11 00:16:37,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +26: [2023-05-11 00:16:37,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +17: [2023-05-11 00:16:37,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +17: [2023-05-11 00:16:37,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +26: [2023-05-11 00:16:37,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +26: [2023-05-11 00:16:37,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +17: [2023-05-11 00:16:37,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +26: [2023-05-11 00:16:37,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +26: [2023-05-11 00:16:37,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +27: [2023-05-11 00:16:37,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt. +27: [2023-05-11 00:16:37,688] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 222 +26: [2023-05-11 00:16:37,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +26: [2023-05-11 00:16:37,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +26: [2023-05-11 00:16:37,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +26: [2023-05-11 00:16:37,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 0: [2023-05-11 00:16:37,688] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 6 +26: [2023-05-11 00:16:37,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +26: [2023-05-11 00:16:37,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +26: [2023-05-11 00:16:37,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +26: [2023-05-11 00:16:37,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +27: [2023-05-11 00:16:37,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +20: [2023-05-11 00:16:37,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +12: [2023-05-11 00:16:37,689] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 96 +29: [2023-05-11 00:16:37,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt. +18: [2023-05-11 00:16:37,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt. +29: [2023-05-11 00:16:37,690] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 236 +18: [2023-05-11 00:16:37,690] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 150 +13: [2023-05-11 00:16:37,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 8: [2023-05-11 00:16:37,690] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 68 +13: [2023-05-11 00:16:37,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +13: [2023-05-11 00:16:37,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +13: [2023-05-11 00:16:37,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +27: [2023-05-11 00:16:37,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +27: [2023-05-11 00:16:37,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +13: [2023-05-11 00:16:37,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +13: [2023-05-11 00:16:37,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +27: [2023-05-11 00:16:37,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +31: [2023-05-11 00:16:37,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +31: [2023-05-11 00:16:37,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +31: [2023-05-11 00:16:37,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +31: [2023-05-11 00:16:37,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 3: [2023-05-11 00:16:37,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +16: [2023-05-11 00:16:37,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt. +31: [2023-05-11 00:16:37,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 3: [2023-05-11 00:16:37,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +31: [2023-05-11 00:16:37,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +16: [2023-05-11 00:16:37,692] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 128 +31: [2023-05-11 00:16:37,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +31: [2023-05-11 00:16:37,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 3: [2023-05-11 00:16:37,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +31: [2023-05-11 00:16:37,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 3: [2023-05-11 00:16:37,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +31: [2023-05-11 00:16:37,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +31: [2023-05-11 00:16:37,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +31: [2023-05-11 00:16:37,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +27: [2023-05-11 00:16:37,693] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 222 + 3: [2023-05-11 00:16:37,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 3: [2023-05-11 00:16:37,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +13: [2023-05-11 00:16:37,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +13: [2023-05-11 00:16:37,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +13: [2023-05-11 00:16:37,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +13: [2023-05-11 00:16:37,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 3: [2023-05-11 00:16:37,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +29: [2023-05-11 00:16:37,694] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 236 +13: [2023-05-11 00:16:37,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +13: [2023-05-11 00:16:37,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +13: [2023-05-11 00:16:37,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 4: [2023-05-11 00:16:37,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt. +13: [2023-05-11 00:16:37,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +13: [2023-05-11 00:16:37,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 3: [2023-05-11 00:16:37,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 3: [2023-05-11 00:16:37,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +31: [2023-05-11 00:16:37,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +31: [2023-05-11 00:16:37,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +31: [2023-05-11 00:16:37,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 3: [2023-05-11 00:16:37,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 4: [2023-05-11 00:16:37,695] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 34 +31: [2023-05-11 00:16:37,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +13: [2023-05-11 00:16:37,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 3: [2023-05-11 00:16:37,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 3: [2023-05-11 00:16:37,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 3: [2023-05-11 00:16:37,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 3: [2023-05-11 00:16:37,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 3: [2023-05-11 00:16:37,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +18: [2023-05-11 00:16:37,696] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 150 +20: [2023-05-11 00:16:37,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 3: [2023-05-11 00:16:37,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 6: [2023-05-11 00:16:37,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt. +20: [2023-05-11 00:16:37,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 6: [2023-05-11 00:16:37,696] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 48 +11: [2023-05-11 00:16:37,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt. +11: [2023-05-11 00:16:37,697] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 88 +17: [2023-05-11 00:16:37,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt. +20: [2023-05-11 00:16:37,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +20: [2023-05-11 00:16:37,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +17: [2023-05-11 00:16:37,697] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 138 +20: [2023-05-11 00:16:37,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +16: [2023-05-11 00:16:37,697] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 128 +20: [2023-05-11 00:16:37,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +20: [2023-05-11 00:16:37,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +20: [2023-05-11 00:16:37,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +20: [2023-05-11 00:16:37,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +20: [2023-05-11 00:16:37,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 4: [2023-05-11 00:16:37,700] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 34 +20: [2023-05-11 00:16:37,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +27: [2023-05-11 00:16:37,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +20: [2023-05-11 00:16:37,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +27: [2023-05-11 00:16:37,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +20: [2023-05-11 00:16:37,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +20: [2023-05-11 00:16:37,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +20: [2023-05-11 00:16:37,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +20: [2023-05-11 00:16:37,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +27: [2023-05-11 00:16:37,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 6: [2023-05-11 00:16:37,702] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 48 +11: [2023-05-11 00:16:37,702] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 88 +27: [2023-05-11 00:16:37,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +17: [2023-05-11 00:16:37,703] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 138 +13: [2023-05-11 00:16:37,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt. +27: [2023-05-11 00:16:37,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +27: [2023-05-11 00:16:37,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +13: [2023-05-11 00:16:37,703] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 106 +27: [2023-05-11 00:16:37,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +27: [2023-05-11 00:16:37,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +17: [2023-05-11 00:16:37,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt. +17: [2023-05-11 00:16:37,705] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 136 +27: [2023-05-11 00:16:37,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +27: [2023-05-11 00:16:37,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +27: [2023-05-11 00:16:37,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... + 2: [2023-05-11 00:16:37,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt. +27: [2023-05-11 00:16:37,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 2: [2023-05-11 00:16:37,705] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 18 +27: [2023-05-11 00:16:37,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +19: [2023-05-11 00:16:37,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt. +19: [2023-05-11 00:16:37,707] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 159 +27: [2023-05-11 00:16:37,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +27: [2023-05-11 00:16:37,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +27: [2023-05-11 00:16:37,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. + 2: [2023-05-11 00:16:37,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt. + 2: [2023-05-11 00:16:37,709] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 16 +13: [2023-05-11 00:16:37,709] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 106 +12: [2023-05-11 00:16:37,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt. +12: [2023-05-11 00:16:37,710] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 100 +17: [2023-05-11 00:16:37,710] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 136 + 2: [2023-05-11 00:16:37,711] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 18 +18: [2023-05-11 00:16:37,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt... +18: [2023-05-11 00:16:37,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt... +18: [2023-05-11 00:16:37,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt... +18: [2023-05-11 00:16:37,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt... +31: [2023-05-11 00:16:37,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt. +31: [2023-05-11 00:16:37,713] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 248 +19: [2023-05-11 00:16:37,714] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 159 + 2: [2023-05-11 00:16:37,715] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 16 +12: [2023-05-11 00:16:37,715] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 100 +20: [2023-05-11 00:16:37,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt. +20: [2023-05-11 00:16:37,717] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 160 +31: [2023-05-11 00:16:37,719] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 248 + 4: [2023-05-11 00:16:37,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt. + 4: [2023-05-11 00:16:37,719] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 36 +20: [2023-05-11 00:16:37,723] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 160 + 4: [2023-05-11 00:16:37,726] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 36 +17: [2023-05-11 00:16:37,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt... +17: [2023-05-11 00:16:37,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt... +17: [2023-05-11 00:16:37,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt... +17: [2023-05-11 00:16:37,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt... + 6: [2023-05-11 00:16:37,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt. +30: [2023-05-11 00:16:37,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt. +30: [2023-05-11 00:16:37,728] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 246 + 6: [2023-05-11 00:16:37,728] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 54 +25: [2023-05-11 00:16:37,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt. + 0: [2023-05-11 00:16:37,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt... + 0: [2023-05-11 00:16:37,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt... + 0: [2023-05-11 00:16:37,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt... + 0: [2023-05-11 00:16:37,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt... +25: [2023-05-11 00:16:37,729] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 206 + 6: [2023-05-11 00:16:37,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt. + 6: [2023-05-11 00:16:37,730] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 50 + 2: [2023-05-11 00:16:37,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt... + 2: [2023-05-11 00:16:37,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt... + 2: [2023-05-11 00:16:37,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt... + 2: [2023-05-11 00:16:37,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt... +31: [2023-05-11 00:16:37,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt... +31: [2023-05-11 00:16:37,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt... +31: [2023-05-11 00:16:37,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt... +31: [2023-05-11 00:16:37,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt... +30: [2023-05-11 00:16:37,733] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 246 +25: [2023-05-11 00:16:37,734] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 206 + 6: [2023-05-11 00:16:37,734] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 54 + 3: [2023-05-11 00:16:37,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt... + 3: [2023-05-11 00:16:37,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt... + 3: [2023-05-11 00:16:37,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt... + 3: [2023-05-11 00:16:37,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt... +16: [2023-05-11 00:16:37,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt... +16: [2023-05-11 00:16:37,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt... +16: [2023-05-11 00:16:37,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt... +16: [2023-05-11 00:16:37,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt... + 6: [2023-05-11 00:16:37,736] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 50 +26: [2023-05-11 00:16:37,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt... +26: [2023-05-11 00:16:37,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt... +26: [2023-05-11 00:16:37,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt... +26: [2023-05-11 00:16:37,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt... +27: [2023-05-11 00:16:37,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt... +27: [2023-05-11 00:16:37,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt... +27: [2023-05-11 00:16:37,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt... +27: [2023-05-11 00:16:37,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt... + 9: [2023-05-11 00:16:37,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt. + 9: [2023-05-11 00:16:37,743] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 73 +13: [2023-05-11 00:16:37,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt... +13: [2023-05-11 00:16:37,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt... +13: [2023-05-11 00:16:37,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt... +13: [2023-05-11 00:16:37,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt... + 2: [2023-05-11 00:16:37,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt. + 2: [2023-05-11 00:16:37,744] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 22 +20: [2023-05-11 00:16:37,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt. +20: [2023-05-11 00:16:37,746] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 162 + 4: [2023-05-11 00:16:37,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt. + 4: [2023-05-11 00:16:37,747] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 38 + 2: [2023-05-11 00:16:37,750] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 22 +20: [2023-05-11 00:16:37,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt... +20: [2023-05-11 00:16:37,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt... +20: [2023-05-11 00:16:37,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt... +20: [2023-05-11 00:16:37,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt... + 9: [2023-05-11 00:16:37,751] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 73 +20: [2023-05-11 00:16:37,752] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 162 + 4: [2023-05-11 00:16:37,753] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 38 +19: [2023-05-11 00:16:37,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt. +19: [2023-05-11 00:16:37,757] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 153 +19: [2023-05-11 00:16:37,763] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 153 + 9: [2023-05-11 00:16:37,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt. + 9: [2023-05-11 00:16:37,766] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 77 +16: [2023-05-11 00:16:37,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt. +16: [2023-05-11 00:16:37,770] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 134 + 9: [2023-05-11 00:16:37,771] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 77 +16: [2023-05-11 00:16:37,776] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 134 +30: [2023-05-11 00:16:37,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt. +30: [2023-05-11 00:16:37,776] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 240 +25: [2023-05-11 00:16:37,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +25: [2023-05-11 00:16:37,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +25: [2023-05-11 00:16:37,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +25: [2023-05-11 00:16:37,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt. +25: [2023-05-11 00:16:37,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt. +25: [2023-05-11 00:16:37,777] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 200 +13: [2023-05-11 00:16:37,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt. +13: [2023-05-11 00:16:37,777] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 108 +11: [2023-05-11 00:16:37,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt. +11: [2023-05-11 00:16:37,780] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 94 +30: [2023-05-11 00:16:37,782] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 240 +25: [2023-05-11 00:16:37,784] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 200 +13: [2023-05-11 00:16:37,784] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 108 +11: [2023-05-11 00:16:37,785] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 94 +25: [2023-05-11 00:16:37,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +19: [2023-05-11 00:16:37,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt. +25: [2023-05-11 00:16:37,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +19: [2023-05-11 00:16:37,787] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 155 +25: [2023-05-11 00:16:37,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +25: [2023-05-11 00:16:37,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +25: [2023-05-11 00:16:37,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +25: [2023-05-11 00:16:37,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +25: [2023-05-11 00:16:37,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... + 5: [2023-05-11 00:16:37,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt. +25: [2023-05-11 00:16:37,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt... +25: [2023-05-11 00:16:37,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. + 5: [2023-05-11 00:16:37,791] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 43 +25: [2023-05-11 00:16:37,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +25: [2023-05-11 00:16:37,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt. +25: [2023-05-11 00:16:37,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt... +25: [2023-05-11 00:16:37,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +25: [2023-05-11 00:16:37,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +25: [2023-05-11 00:16:37,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +25: [2023-05-11 00:16:37,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt. +19: [2023-05-11 00:16:37,793] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 155 +15: [2023-05-11 00:16:37,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt. +15: [2023-05-11 00:16:37,795] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 125 + 5: [2023-05-11 00:16:37,797] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 43 +16: [2023-05-11 00:16:37,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt. +16: [2023-05-11 00:16:37,799] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 132 + 2: [2023-05-11 00:16:37,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt. + 2: [2023-05-11 00:16:37,800] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 20 +15: [2023-05-11 00:16:37,801] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 125 +11: [2023-05-11 00:16:37,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt. +11: [2023-05-11 00:16:37,802] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 90 + 5: [2023-05-11 00:16:37,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt. + 5: [2023-05-11 00:16:37,805] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 47 + 2: [2023-05-11 00:16:37,805] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 20 + 5: [2023-05-11 00:16:37,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt. +11: [2023-05-11 00:16:37,807] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 90 +19: [2023-05-11 00:16:37,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt. + 5: [2023-05-11 00:16:37,807] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 41 +19: [2023-05-11 00:16:37,807] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 157 +16: [2023-05-11 00:16:37,808] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 132 + 5: [2023-05-11 00:16:37,810] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 47 +26: [2023-05-11 00:16:37,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt. +12: [2023-05-11 00:16:37,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt. +26: [2023-05-11 00:16:37,811] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 208 +12: [2023-05-11 00:16:37,811] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 102 + 5: [2023-05-11 00:16:37,813] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 41 +19: [2023-05-11 00:16:37,813] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 157 + 9: [2023-05-11 00:16:37,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt. + 9: [2023-05-11 00:16:37,815] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 79 +26: [2023-05-11 00:16:37,816] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 208 +12: [2023-05-11 00:16:37,817] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 102 + 0: [2023-05-11 00:16:37,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. + 0: [2023-05-11 00:16:37,819] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 4 + 9: [2023-05-11 00:16:37,821] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 79 +10: [2023-05-11 00:16:37,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt. +10: [2023-05-11 00:16:37,824] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 80 + 0: [2023-05-11 00:16:37,825] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 4 +25: [2023-05-11 00:16:37,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt... +25: [2023-05-11 00:16:37,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt... +25: [2023-05-11 00:16:37,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt... +25: [2023-05-11 00:16:37,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt... +10: [2023-05-11 00:16:37,831] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 80 + 7: [2023-05-11 00:16:37,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt. + 7: [2023-05-11 00:16:37,848] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 63 +11: [2023-05-11 00:16:37,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt. +11: [2023-05-11 00:16:37,849] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 92 +15: [2023-05-11 00:16:37,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt. +15: [2023-05-11 00:16:37,851] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 121 + 7: [2023-05-11 00:16:37,853] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 63 + 0: [2023-05-11 00:16:37,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. + 0: [2023-05-11 00:16:37,853] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 2 +11: [2023-05-11 00:16:37,855] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 92 +15: [2023-05-11 00:16:37,856] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 121 + 0: [2023-05-11 00:16:37,859] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 2 +16: [2023-05-11 00:16:37,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt. +16: [2023-05-11 00:16:37,863] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 130 + 1: [2023-05-11 00:16:37,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt. + 1: [2023-05-11 00:16:37,867] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 15 +16: [2023-05-11 00:16:37,869] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 130 + 0: [2023-05-11 00:16:37,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. + 0: [2023-05-11 00:16:37,869] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 0 +10: [2023-05-11 00:16:37,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt. + 8: [2023-05-11 00:16:37,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt. +10: [2023-05-11 00:16:37,870] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 83 + 8: [2023-05-11 00:16:37,870] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 71 + 5: [2023-05-11 00:16:37,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt. + 5: [2023-05-11 00:16:37,873] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 45 + 0: [2023-05-11 00:16:37,875] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 0 + 1: [2023-05-11 00:16:37,876] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 15 +10: [2023-05-11 00:16:37,876] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 83 + 5: [2023-05-11 00:16:37,878] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 45 + 8: [2023-05-11 00:16:37,879] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 71 + 0: could not find arguments in the checkpoint ... + 0: checkpoint version 3.0 +10: [2023-05-11 00:16:37,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt. +10: [2023-05-11 00:16:37,883] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 87 +24: [2023-05-11 00:16:37,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt. +24: [2023-05-11 00:16:37,885] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 195 +10: [2023-05-11 00:16:37,888] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 87 +15: [2023-05-11 00:16:37,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt. +15: [2023-05-11 00:16:37,889] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 127 +24: [2023-05-11 00:16:37,890] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 195 + 7: [2023-05-11 00:16:37,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt. + 7: [2023-05-11 00:16:37,891] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 57 +15: [2023-05-11 00:16:37,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt. +15: [2023-05-11 00:16:37,892] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 123 +15: [2023-05-11 00:16:37,894] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 127 + 7: [2023-05-11 00:16:37,896] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 57 +15: [2023-05-11 00:16:37,898] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 123 + 8: [2023-05-11 00:16:37,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt. + 8: [2023-05-11 00:16:37,906] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 67 +11: [2023-05-11 00:16:37,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt. +11: [2023-05-11 00:16:37,910] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 95 + 8: [2023-05-11 00:16:37,911] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 67 +11: [2023-05-11 00:16:37,916] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 95 +29: [2023-05-11 00:16:37,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt. +29: [2023-05-11 00:16:37,919] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 239 +24: [2023-05-11 00:16:37,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt. +24: [2023-05-11 00:16:37,924] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 197 +29: [2023-05-11 00:16:37,925] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 239 +24: [2023-05-11 00:16:37,930] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 197 + 8: [2023-05-11 00:16:37,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt. + 8: [2023-05-11 00:16:37,931] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 65 + 8: [2023-05-11 00:16:37,937] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 65 + 1: [2023-05-11 00:16:37,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt. + 1: [2023-05-11 00:16:37,947] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 13 +11: [2023-05-11 00:16:37,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt. +11: [2023-05-11 00:16:37,950] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 89 + 1: [2023-05-11 00:16:37,953] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 13 + 7: [2023-05-11 00:16:37,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt. + 7: [2023-05-11 00:16:37,955] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 59 +11: [2023-05-11 00:16:37,955] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 89 +24: [2023-05-11 00:16:37,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt. +24: [2023-05-11 00:16:37,957] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 199 + 6: [2023-05-11 00:16:37,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt. + 6: [2023-05-11 00:16:37,958] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 55 +22: [2023-05-11 00:16:37,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt. +22: [2023-05-11 00:16:37,959] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 181 + 1: [2023-05-11 00:16:37,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt. + 1: [2023-05-11 00:16:37,959] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 11 +24: [2023-05-11 00:16:37,962] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 199 + 6: [2023-05-11 00:16:37,963] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 55 +10: [2023-05-11 00:16:37,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt. +28: [2023-05-11 00:16:37,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt. +10: [2023-05-11 00:16:37,963] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 81 +28: [2023-05-11 00:16:37,963] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 229 + 7: [2023-05-11 00:16:37,963] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 59 +22: [2023-05-11 00:16:37,964] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 181 + 1: [2023-05-11 00:16:37,965] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 11 +29: [2023-05-11 00:16:37,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt. +29: [2023-05-11 00:16:37,966] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 237 +28: [2023-05-11 00:16:37,968] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 229 +10: [2023-05-11 00:16:37,969] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 81 +28: [2023-05-11 00:16:37,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt. +28: [2023-05-11 00:16:37,969] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 227 +29: [2023-05-11 00:16:37,971] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 237 +28: [2023-05-11 00:16:37,974] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 227 +14: [2023-05-11 00:16:37,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt. +14: [2023-05-11 00:16:37,986] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 117 +29: [2023-05-11 00:16:37,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt. +29: [2023-05-11 00:16:37,990] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 235 +14: [2023-05-11 00:16:37,991] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 117 +29: [2023-05-11 00:16:37,996] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 235 +11: [2023-05-11 00:16:37,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt. +11: [2023-05-11 00:16:37,998] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 91 +24: [2023-05-11 00:16:37,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt. +24: [2023-05-11 00:16:38,000] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 193 + 6: [2023-05-11 00:16:38,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt. + 6: [2023-05-11 00:16:38,001] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 53 +11: [2023-05-11 00:16:38,003] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 91 +29: [2023-05-11 00:16:38,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt. +29: [2023-05-11 00:16:38,005] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 233 +24: [2023-05-11 00:16:38,005] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 193 + 6: [2023-05-11 00:16:38,007] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 53 +29: [2023-05-11 00:16:38,010] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 233 + 4: [2023-05-11 00:16:38,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt. + 4: [2023-05-11 00:16:38,015] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 37 + 8: [2023-05-11 00:16:38,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt. + 8: [2023-05-11 00:16:38,016] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 69 + 4: [2023-05-11 00:16:38,021] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 37 + 4: [2023-05-11 00:16:38,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt. + 8: [2023-05-11 00:16:38,022] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 69 + 4: [2023-05-11 00:16:38,022] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 39 +21: [2023-05-11 00:16:38,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt. +21: [2023-05-11 00:16:38,024] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 175 + 4: [2023-05-11 00:16:38,027] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 39 +30: [2023-05-11 00:16:38,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt. +30: [2023-05-11 00:16:38,027] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 241 +18: [2023-05-11 00:16:38,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt. +18: [2023-05-11 00:16:38,030] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 145 +28: [2023-05-11 00:16:38,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt. +28: [2023-05-11 00:16:38,030] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 225 +21: [2023-05-11 00:16:38,031] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 175 +30: [2023-05-11 00:16:38,033] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 241 +18: [2023-05-11 00:16:38,035] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 145 +28: [2023-05-11 00:16:38,035] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 225 + 4: [2023-05-11 00:16:38,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt. + 4: [2023-05-11 00:16:38,036] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 33 + 4: [2023-05-11 00:16:38,042] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 33 +30: [2023-05-11 00:16:38,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt. +30: [2023-05-11 00:16:38,045] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 243 +30: [2023-05-11 00:16:38,050] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 243 +23: [2023-05-11 00:16:38,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt. +23: [2023-05-11 00:16:38,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt. +23: [2023-05-11 00:16:38,053] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 189 +23: [2023-05-11 00:16:38,053] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 191 +23: [2023-05-11 00:16:38,059] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 191 +23: [2023-05-11 00:16:38,059] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 189 +17: [2023-05-11 00:16:38,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt. +17: [2023-05-11 00:16:38,061] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 141 +31: [2023-05-11 00:16:38,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt. +31: [2023-05-11 00:16:38,062] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 249 +30: [2023-05-11 00:16:38,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt. +30: [2023-05-11 00:16:38,065] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 247 +28: [2023-05-11 00:16:38,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt. +28: [2023-05-11 00:16:38,066] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 231 +17: [2023-05-11 00:16:38,067] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 141 +14: [2023-05-11 00:16:38,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt. +14: [2023-05-11 00:16:38,067] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 113 +23: [2023-05-11 00:16:38,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt. +23: [2023-05-11 00:16:38,067] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 185 + 4: [2023-05-11 00:16:38,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt. +31: [2023-05-11 00:16:38,068] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 249 + 4: [2023-05-11 00:16:38,068] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 35 +14: [2023-05-11 00:16:38,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt. +14: [2023-05-11 00:16:38,069] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 119 +22: [2023-05-11 00:16:38,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt. +22: [2023-05-11 00:16:38,070] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 179 +28: [2023-05-11 00:16:38,071] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 231 +14: [2023-05-11 00:16:38,073] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 113 +23: [2023-05-11 00:16:38,073] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 185 +30: [2023-05-11 00:16:38,073] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 247 + 4: [2023-05-11 00:16:38,073] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 35 +14: [2023-05-11 00:16:38,074] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 119 +22: [2023-05-11 00:16:38,077] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 179 +16: [2023-05-11 00:16:38,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt. +16: [2023-05-11 00:16:38,079] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 133 + 2: [2023-05-11 00:16:38,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt. + 2: [2023-05-11 00:16:38,082] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 17 +10: [2023-05-11 00:16:38,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt. +10: [2023-05-11 00:16:38,083] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 85 +12: [2023-05-11 00:16:38,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt. +12: [2023-05-11 00:16:38,084] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 101 +16: [2023-05-11 00:16:38,085] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 133 + 2: [2023-05-11 00:16:38,088] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 17 +10: [2023-05-11 00:16:38,088] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 85 +12: [2023-05-11 00:16:38,089] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 101 +21: [2023-05-11 00:16:38,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt. +21: [2023-05-11 00:16:38,091] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 173 +21: [2023-05-11 00:16:38,096] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 173 + 6: [2023-05-11 00:16:38,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt. + 6: [2023-05-11 00:16:38,106] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 51 + 6: [2023-05-11 00:16:38,111] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 51 + 0: [2023-05-11 00:16:38,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt. + 0: [2023-05-11 00:16:38,115] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 7 +12: [2023-05-11 00:16:38,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt. +12: [2023-05-11 00:16:38,118] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 97 + 6: [2023-05-11 00:16:38,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt. + 6: [2023-05-11 00:16:38,119] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 49 +11: [2023-05-11 00:16:38,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt. +11: [2023-05-11 00:16:38,120] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 93 + 0: [2023-05-11 00:16:38,120] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 7 +14: [2023-05-11 00:16:38,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt. +14: [2023-05-11 00:16:38,123] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 115 + 6: [2023-05-11 00:16:38,125] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 49 +12: [2023-05-11 00:16:38,125] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 97 +11: [2023-05-11 00:16:38,126] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 93 +14: [2023-05-11 00:16:38,128] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 115 + 0: [2023-05-11 00:16:38,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt. + 0: [2023-05-11 00:16:38,130] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 1 + 0: [2023-05-11 00:16:38,136] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 1 +21: [2023-05-11 00:16:38,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt. +21: [2023-05-11 00:16:38,141] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 171 +22: [2023-05-11 00:16:38,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt. +22: [2023-05-11 00:16:38,146] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 177 +21: [2023-05-11 00:16:38,146] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 171 +13: [2023-05-11 00:16:38,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt. +13: [2023-05-11 00:16:38,149] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 109 +22: [2023-05-11 00:16:38,151] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 177 +13: [2023-05-11 00:16:38,154] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 109 +12: [2023-05-11 00:16:38,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt. +12: [2023-05-11 00:16:38,157] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 99 + 7: [2023-05-11 00:16:38,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt. + 7: [2023-05-11 00:16:38,158] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 61 +21: [2023-05-11 00:16:38,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt. +21: [2023-05-11 00:16:38,159] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 169 +23: [2023-05-11 00:16:38,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt. +23: [2023-05-11 00:16:38,163] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 187 + 7: [2023-05-11 00:16:38,163] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 61 +12: [2023-05-11 00:16:38,164] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 99 +21: [2023-05-11 00:16:38,167] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 169 +23: [2023-05-11 00:16:38,168] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 187 +18: [2023-05-11 00:16:38,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt. +18: [2023-05-11 00:16:38,169] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 151 + 2: [2023-05-11 00:16:38,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt. + 2: [2023-05-11 00:16:38,171] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 19 +22: [2023-05-11 00:16:38,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt. +22: [2023-05-11 00:16:38,171] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 183 +18: [2023-05-11 00:16:38,175] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 151 + 2: [2023-05-11 00:16:38,176] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 19 +22: [2023-05-11 00:16:38,177] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 183 +31: [2023-05-11 00:16:38,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt. +31: [2023-05-11 00:16:38,178] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 255 +17: [2023-05-11 00:16:38,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt. +31: [2023-05-11 00:16:38,183] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 255 +17: [2023-05-11 00:16:38,183] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 137 +17: [2023-05-11 00:16:38,188] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 137 +16: [2023-05-11 00:16:38,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt. +16: [2023-05-11 00:16:38,197] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 129 +12: [2023-05-11 00:16:38,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt. +12: [2023-05-11 00:16:38,201] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 103 +16: [2023-05-11 00:16:38,202] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 129 +17: [2023-05-11 00:16:38,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt. +17: [2023-05-11 00:16:38,202] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 139 +17: [2023-05-11 00:16:38,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt. +17: [2023-05-11 00:16:38,204] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 143 +12: [2023-05-11 00:16:38,206] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 103 +26: [2023-05-11 00:16:38,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt. +26: [2023-05-11 00:16:38,208] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 209 +17: [2023-05-11 00:16:38,210] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 139 +17: [2023-05-11 00:16:38,210] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 143 +26: [2023-05-11 00:16:38,213] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 209 +16: [2023-05-11 00:16:38,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt. +16: [2023-05-11 00:16:38,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt. +16: [2023-05-11 00:16:38,213] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 135 +16: [2023-05-11 00:16:38,213] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 131 +13: [2023-05-11 00:16:38,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt. +13: [2023-05-11 00:16:38,216] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 107 + 1: [2023-05-11 00:16:38,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt. + 1: [2023-05-11 00:16:38,217] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 9 +20: [2023-05-11 00:16:38,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt. +20: [2023-05-11 00:16:38,218] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 161 +16: [2023-05-11 00:16:38,221] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 135 +13: [2023-05-11 00:16:38,221] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 107 +16: [2023-05-11 00:16:38,221] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 131 +25: [2023-05-11 00:16:38,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt. +25: [2023-05-11 00:16:38,221] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 201 + 1: [2023-05-11 00:16:38,222] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 9 +20: [2023-05-11 00:16:38,223] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 161 +25: [2023-05-11 00:16:38,226] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 201 +30: [2023-05-11 00:16:38,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt. +30: [2023-05-11 00:16:38,228] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 245 + 3: [2023-05-11 00:16:38,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt. + 3: [2023-05-11 00:16:38,229] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 29 +20: [2023-05-11 00:16:38,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt. +20: [2023-05-11 00:16:38,229] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 167 +30: [2023-05-11 00:16:38,233] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 245 +26: [2023-05-11 00:16:38,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt. +26: [2023-05-11 00:16:38,234] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 213 + 3: [2023-05-11 00:16:38,234] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 29 +20: [2023-05-11 00:16:38,234] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 167 +26: [2023-05-11 00:16:38,240] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 213 +31: [2023-05-11 00:16:38,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt. +31: [2023-05-11 00:16:38,242] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 251 +27: [2023-05-11 00:16:38,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt. +27: [2023-05-11 00:16:38,244] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 219 +27: [2023-05-11 00:16:38,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt. +27: [2023-05-11 00:16:38,245] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 223 +31: [2023-05-11 00:16:38,248] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 251 +27: [2023-05-11 00:16:38,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt. +27: [2023-05-11 00:16:38,250] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 217 +27: [2023-05-11 00:16:38,250] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 223 +27: [2023-05-11 00:16:38,250] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 219 +27: [2023-05-11 00:16:38,255] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 217 +13: [2023-05-11 00:16:38,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt. +13: [2023-05-11 00:16:38,255] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 105 + 0: [2023-05-11 00:16:38,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt. + 2: [2023-05-11 00:16:38,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt. + 0: [2023-05-11 00:16:38,256] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 3 + 2: [2023-05-11 00:16:38,256] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 21 + 0: [2023-05-11 00:16:38,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt. + 0: [2023-05-11 00:16:38,259] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 5 +26: [2023-05-11 00:16:38,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt. +26: [2023-05-11 00:16:38,259] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 215 + 0: [2023-05-11 00:16:38,261] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 3 +13: [2023-05-11 00:16:38,261] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 105 + 2: [2023-05-11 00:16:38,263] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 21 + 3: [2023-05-11 00:16:38,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt. + 3: [2023-05-11 00:16:38,264] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 27 + 0: [2023-05-11 00:16:38,264] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 5 +26: [2023-05-11 00:16:38,265] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 215 + 3: [2023-05-11 00:16:38,269] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 27 +18: [2023-05-11 00:16:38,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt. +20: [2023-05-11 00:16:38,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt. +18: [2023-05-11 00:16:38,273] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 149 +20: [2023-05-11 00:16:38,273] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 165 +26: [2023-05-11 00:16:38,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt. +26: [2023-05-11 00:16:38,278] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 211 +18: [2023-05-11 00:16:38,279] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 149 +20: [2023-05-11 00:16:38,280] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 165 +26: [2023-05-11 00:16:38,283] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 211 +31: [2023-05-11 00:16:38,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt. +31: [2023-05-11 00:16:38,287] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 253 +18: [2023-05-11 00:16:38,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt. +18: [2023-05-11 00:16:38,288] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 147 +27: [2023-05-11 00:16:38,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt. +27: [2023-05-11 00:16:38,292] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 221 +31: [2023-05-11 00:16:38,292] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 253 +18: [2023-05-11 00:16:38,293] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 147 +27: [2023-05-11 00:16:38,297] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 221 + 2: [2023-05-11 00:16:38,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt. + 2: [2023-05-11 00:16:38,301] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 23 + 2: [2023-05-11 00:16:38,307] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 23 + 3: [2023-05-11 00:16:38,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt. + 3: [2023-05-11 00:16:38,321] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 31 + 3: [2023-05-11 00:16:38,326] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 31 + 3: [2023-05-11 00:16:38,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt. + 3: [2023-05-11 00:16:38,335] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 25 + 3: [2023-05-11 00:16:38,340] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 25 +25: [2023-05-11 00:16:38,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt. +25: [2023-05-11 00:16:38,349] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 203 +25: [2023-05-11 00:16:38,355] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 203 +25: [2023-05-11 00:16:38,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt. +25: [2023-05-11 00:16:38,360] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 205 +25: [2023-05-11 00:16:38,365] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 205 +25: [2023-05-11 00:16:38,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt. +25: [2023-05-11 00:16:38,366] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 207 +13: [2023-05-11 00:16:38,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt. +13: [2023-05-11 00:16:38,371] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 111 +25: [2023-05-11 00:16:38,372] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 207 +20: [2023-05-11 00:16:38,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt. +20: [2023-05-11 00:16:38,374] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 163 +13: [2023-05-11 00:16:38,377] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 111 +20: [2023-05-11 00:16:38,381] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 163 + 0: successfully loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4opt2 at iteration 0 +31: time (ms) | load-checkpoint: 14279.09 + 0: estimated model parameters: 3.581186048 + 0: estimated model parameters without embeddings: 3.4276352 + 0: [after model, optimizer, and learning rate scheduler are built] datetime: 2023-05-11 00:16:39 + 0: > building train, validation, and test datasets ... + 0: > datasets target sizes (minimum size): + 0: train: 1 + 0: validation: 102400 + 0: test: 102400 + 0: > building train, validation, and test datasets for GPT ... + 0: > building dataset index ... + 0: reading sizes... + 0: reading pointers... + 0: reading document index... + 0: creating numpy buffer of mmap... + 0: creating memory view of numpy buffer... + 0: > finished creating indexed dataset in 0.031277 seconds + 0: number of documents: 3133972 + 0: > dataset split: + 0: train: + 0: document indices in [0, 3133972) total of 3133972 documents + 0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_doc_idx.npy + 0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_sample_idx.npy + 0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_shuffle_idx.npy + 0: loaded indexed file in 0.114 seconds + 0: total number of samples: 731002 + 0: total number of epochs: 1 + 0: > building dataset index ... + 0: reading sizes... + 0: reading pointers... + 0: reading document index... + 0: creating numpy buffer of mmap... + 0: creating memory view of numpy buffer... + 0: > finished creating indexed dataset in 0.037184 seconds + 0: number of documents: 364608 + 0: > dataset split: + 0: validation: + 0: document indices in [0, 364608) total of 364608 documents + 0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_102400ns_2048sl_1234s_doc_idx.npy + 0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_102400ns_2048sl_1234s_sample_idx.npy + 0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_102400ns_2048sl_1234s_shuffle_idx.npy + 0: loaded indexed file in 0.076 seconds + 0: total number of samples: 169955 + 0: total number of epochs: 2 + 0: > finished creating GPT datasets ... + 0: [after dataloaders are built] datetime: 2023-05-11 00:16:46 + 0: done with setup ... + 0: training ... +31: time (ms) | model-and-optimizer-setup: 30296.56 | train/valid/test-data-iterators-setup: 3618.56 + 0: [after training is done] datetime: 2023-05-11 00:16:46 +31: ----------------------------------------------------------------------------------------------------------------- +31: validation loss at the end of training for val data | lm loss value: 2.525244E+00 | lm loss PPL: 1.249394E+01 | +31: ----------------------------------------------------------------------------------------------------------------- +END 3494406: Thu 11 May 2023 12:19:34 AM EEST diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a4df799e24b14c8ac90d15dd4afdac0c7b253352 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dac7aa4f3d6432ad3a7bc2870304a65f43727c4b048204f883d66c2e2022341e +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f009204dd63206b8539fecd5e68fd22cd116242 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d074446bb05b717cdbcbe0a0c0a30ada9b06206e1b28d5e7f54cd3e6f8d9eb93 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2fa1951f2ae3ce2b26e5bcd8ba0e9a2984a8421a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90bcb1c3701e9f932c0ae786b357763e693c553bac45c56e5245492dc3f3f1c9 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..28cd1f7907a95b8d378381498a000a83265da0e6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07e92075d1b87857dc556198f73296b179d19caafc93a5a050088937d2627a63 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..497a32d49f68268b5e313bb7b92369080ce4552b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c72ddd6370f1fa645a964530ade78c850b5ad0f5a27ebb262bddac48d6b5a23b +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1bfefb1ae98783dd26f1e6c9dea0043adccf57c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f4bd617aeccd7aa34fe0d1406f46838ba1b5af4549c424ad92157e59625029c +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1e1bd0b3fef31b9f6b61f13d8d6d39a1a55f07c7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d4980397af88d0ec1f6beba788ab6900b9eec55d53303e5a02f51e026f58a41 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..22e4ad334265117aa53ced784cf1fc941066c369 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e89fd8f2c57d3c92b18f409d63389236225e9bea889e5358100ab1f58157f2b +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ad56977241895d8043f951697649475ffb1dbce --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c1d42c840b5e1e97f1dd55db3b2b2288f7770ba96e26522fc353c3926a55307 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..994470d48bbc9658832f10425d8706f271bee819 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e2ea36d715ee034bd69b96cf9fde77d30d3c6e9cf3828461fd0441dc5b33597 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a90ae4220f1d3a056e2672c098545479ac0d16d1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a9d73da460bbf5b1c707cfc232fb059998ea25cf4f481b72a9dda3a0c2758df +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c89dee4f7a63b336f8e0631a7854a86690a76c36 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e58802504982a67646b86b29c1382a2e94cf47d700a4b00cfc9379342bc5853d +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..78fb40da55ed7e14f8333b96e5b4e1ed37baa514 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0da09dc278dd371efccb4f4441b49f1e3b02f84a00842b1d359ed62c2c1788b3 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..be428a4e4f89665e164648d7d7a162bf6a0fd59c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fde27e0478c1fa927590162cf47502c625ecd206d1a0531ad559157c552d5a3a +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b9b89d315b11492bd0acf06223334dea1c07064 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4704fd7cee6206d7655a6558f404cc86f2a1f3d7e1a6b7a87c603993268f3697 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..37ff6c37043856d2a2671a2746283297a5d667ca --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5b600e9bb7b3824ecf95466a0db11bc741e1c754ab24444661d5619d0560d4c +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..887fc36a633cf7c8803c2ad1145e5d2ec5983034 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa13f4a5c7200b24d3968f22278ee7b4a43c362baec8ffaa273ba3e9e9e33f05 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6fa3d6998db8e17b469dd8eb7fe06ffb1bb012a1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b7419bcfed3078170f94a71e772baefc46b329f16e609c674445c291c56d2c0 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..25b4ebd2fafa30737cdaeb35eddc1987e91c1cb5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b29e6ae1548f2c90c00074218bdb2497147a40994da541c9163696e6bea9a702 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f332ed5a11451361d242b69ea5d91dc63e0eb71a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b27fbdcbaf1c2f25191ec97f2df755dc7a64157fff0c9b977bf10869d5fe8b4 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..012adcfba3fa8d77cbbc667ee0031466ece277bd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf1448685546f178fa70d0e755f1ef32fe3008cffd01be386a017012374fd193 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a82acd169622b8ec2acef91edb77ed545e6c051a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44abecd47a2eeab35298cade46821e0fb57247c4a2f9029a33a462b1afad40e8 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2edc78ea032d6634320dd7c816f73e4de2e4ec46 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00eecddf25585d3784364881159b50c3057d7a0ec4a78494b08bf4982cae027e +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0ce0a4fe2dde38b1b37975508fd4661bc58ef49b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fff96319477c70bf49b20523f0a4c8c270a1b7671ce273284ba56a03ed0d1d6 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a57b2568e7c7a889e7e89546398b614a614ce96 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7ef9650f74f9c0f64f8bc1b1c4dcf45117a9c416411c8fe8ead96e43635cb49 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ab4c2a9f2c609ad118070b535e049de3dd080a5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2df34fd00462001a4282842c579ce2377ac051dca9cfa0f123d8c3c5d4df588c +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8eef5b383547cb2576d790c9f27c7bfd76cfff89 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8054e2f63fe1f746cca134d5f779f15f1731f882d333b20da3652acbbe2ed459 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..593d38f29f3f29390bd55586801a9c7e004ea7da --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a547d3153bf85336932960c278021f13173a9a7c40f5d10d8efb2af4a3aef425 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f5ca2d543da6da9f7d1ca93ca06e3e783320ccd3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c483c6fa16a5a2b677f334fbb8618aa2c995bc34d3a0df65f36f9b90cf49efa +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1db7063f9926287b3756327e5bb6342b2eda0736 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af2a4331409d8966d8773c7abf674e0e5e084c06fa2ac0c504418eba696a391a +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..193aa6dc26139127d881a36e7c1247292d07fc8b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:551882c105d0c183b9a6e11eac4554b4c4c0b1a7baf1e3cbd0e91036b58931e4 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5936d132b0f2ee74c3994cafe48942523f024361 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6783190d2e634c6f08bd40445e232fb45b86e1d57685294b4fb408273412aec6 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..61e3d48cab6471f56de6da06b3cd318a310ff9ff --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f91e96eaaba3df1e02754cd71b61198b3d625c00bdfa27eca04271a6f47fcf5 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cce2304f058d595a5089d7fe96d8722b46d6c9fc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82385732271e65de0734bd9671f4a9c6bdb3cc242f3619bd4cadbc98699af6c3 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d28bf83f745e722e9d07a138df0324da0befd545 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1f9ec19a59d77fbee5b3fcbe02227392b76db351d009b1073f1a242ad750892 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..812c1ac2833874fe53cb1cc54e3c6663524fafdb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70ea591f4b6c65640b41cd9e69c5b2399457e19a1786936e3c76fc1428bc2bf6 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..475816d0861d9a568f0ca42c3c993fff48115950 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa34686456c45d7b11c6f46897788e176495c8e9c918918163db7ee715295076 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bc2b26a20abe29efbeeb81495c8b6301877f32f3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65f116cb5f9f72bc448069da436ce5747530c9ee476392ef6cdb1ba191631ac1 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2b37402975dcee90ba5828333305621cfe3859fc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b027561deac16c7cc10720f51b058e50bf789916a58ea64ead811e922028bad3 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a3183f5a8a0e87d39b220d1e27f1ce961a0b44ad --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc40861a376ed2762c5ef4a5eff71720f6751eec3d69be471b9845ada4b0fb1b +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..affbc217e18b4a6071115256677431fcd628bba0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a4b854f51b74f19fe285422e8e7ff96c249f6388fcc69eb781b5bc856fe3224 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..566113da7eeb85524b7b37b0e9afd78ca94fe8b8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f37bc11677d4e587e315d551dfd457204de5dae3bc2e850f5c8534af108787ac +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..11cf1be53227741e55462f6eecec8bb4c62d59b9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54471bb3e3bbb3e5a0ccb98461bb6acb49c9c74598cfaba4b393fc13e5839791 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..044f0cd1c8303c70cb075cfce4d48a9c5da135b3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc58d8d7068b81602d484e35d8b34ef6f7823a046ae517078a17eef072422dfd +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c9d4bed421fd4a8f68de19543ca8bcc81e8a0293 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b225d938f2fc1a184e31b30df5097b9e165e25fd588dda1baf9304bcf978053e +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..770e02aeec247013aa2e3af91e07588702838a44 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7725b9f0c8636266995fce79211da5b22f519fb5fc1bc672d08e8ffc7148a05 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0e7ae4678e3fb62b8db47f3d8477dcfd6f950ee3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35b4f4e869035e647220453ae8734d3db4faaea3dc5672687c17601b2fbf7330 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c62147510b9286288689875bf702cef02e10f8b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:827f62ff055450ce54e28ee6be1af184d0b8766afdf06b29833f1a1617067387 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..433277838514f5e0cfa04870151798197cdb7b9f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f878f7e843c37f05d7216df6c5c79b8faa431d351f19aa7f36573bd1169332a +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3e55a38f551660b7d023846b8d1d887da92cc615 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fd04f60157167e934844925393123c34a1dde929644983df2c99bb4849c49e2 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..72762b7393cb60cd4d620cdb9602e8a83d350013 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9aa5ade9b7b7417eac38cc3bf766afac8717397ca340ba1125583accced7dba +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c8c175c8149a14b85d218a6c72f5bfa73d0da45 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fb2bdd1a79189cc3222cd0f919222e1390e8500c7a02a0e00e6b46d07dce2d0 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7aaa8072758ef97798776f8c30f0f3b4a568f2f5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d5bc272a208ba3c04146150535236775cd07d5a1fff283752137c314fcffa12 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..df78fa7ed288c1193a4b1876f58216cc01820bc2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6719b0522cfc27d982e2dfdc559b5e5dcd27601b0192b2830f6ea9f4be328ee +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..31d5e5caa5827c89e808e9cff3458b48b1f78673 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87253dd10234a7a0b9e8d6e0e274f0ad9fa61c27b334eac8d126c7b06b419aa1 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e7c6455fb5050806b0fe2da3dedeafee58827263 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b129b7096ec816e9562055e406f712a5605e9f4ea327db21a64c7f924a85129 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0310e69fcd7867744060d0bdfde76a26437ea38d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8344ce6f7c67eaf3213f16e53278abe3a548f6ab78b40a14e3472ecdc4f1c050 +size 167872941 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d1f13f32268d10e50f11dcd528b9e08a6af5c4f0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8141792a2f795385a7a2830c14d60ef69c2053be374271c393d81140810c728e +size 167872941 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4e81031bfd5fc2b3d9fcc146a536f497be1932a5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f0d54ac292d9fcc27f5efe24b7e144c3f6dcf752a772856082e8078e9650586 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d6e5c029dc22ec5304473414d43d2f46cceffe1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a356b9eeb586dc658cd5378ae4d7471b51120758b054fe8a5ee64682e3e7f1e9 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c21eda013fa220751b5f9157b019f079f8399052 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1572afced3b0954ccdcad7598b3f048d4613d1c83855f15bbbde61c82aca6226 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0454feea50ec2c021ca9eac4cb661105a5ed5ae7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccfdd3fe16db8101fce3ea42df4c491813b6c1fa82dce79e56eab6007349f555 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..673554b0223dcf22344f71db6550563006cd8fa1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c16bbbe5fd07543fc69302e5fcff641815bdac7fb932bd34e56b95a91d00ebc6 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0b8165a9c89752c7d876daf5f87125102e1e0ca0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b9484b58fa0cb70c8fc8c798331285fc159894f860c3d61bf5352011e4ea488 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c8be3634e625073800293b406cc486b0f9611454 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18d84ecaa2a39a15077de12774b8003e89dd150dd278429234707e82e4311fbc +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2b09bdf5d0f3ef122e3a18e5b177d0595f7e2088 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d8bb6ef58eb6f6538d2a6350aaae10e0fd062626fb1a79f19173869cad361b9 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4636e4b83158536343154b8e9acadd14605d45ca --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ae8205641802d4ffc33c995979d9b376b9040487778c3294485d623ff0731ed +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1f1c0502914f5b4c9880256e7164cc56629a5db6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:931a98abd4f7df8e5858aa35a38bf9c417135a90551ceb5c148ae36acbb1a217 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..12b0114544f326004d08907a621a8057d2ab0fd1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39bec50900521439aa2dda84750442154f1c78dda9d00eb0defa720da0fb8b79 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f4c0a7e43e932bd554e45cc5b50dcfd32058d1b4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c8ef7451884e65b1694b50860e0e3c9751c53b20ed9ff88f892e2cc962c4f83 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7e985469cd7eed8f411e8d38cd5f07fb8fcccff5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef17109e33be8c6cb1a1890e3f70c2626247907caea03c4af1953bffe7a955c4 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cfd278bc70bdc56acbd8921161ee8c9c26578b13 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42144e5da9b54af02191b5a653305a38a78e24f5325671be44f2aee9c189abfe +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3745111f6a2e26f4ec189fd6b9f4755ed2341408 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d2775ecb2a42a6a7f6763cc4168fcff48965a392f0eff76ae478b614b38cff4 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f274d2900352a5b148cf7044c2c67a5582fdad4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14a272636ab11ea98e7ee673f8ace10aa9e304e8ef3d3875f378178aaf3b1248 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b7d2a627fee028b9bdf98e185e01ccd14b980c20 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:120a52a5a2df1faa07bcb50a5dfd20876016a7f33a58b40200186453c97e8752 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..36aa81f3ad3f15f74315eed63ed0f4ae27e996f0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:369e18591439db403484496b6dbe1ea3cb4b2bd2e9f694fe6a38972c9557b9c5 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8bbe5a3e336d9ad214b0c6d2fa8aeefdfb21dd87 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba0002f3d7dabe0221e4a3bd2e9e26e4292025c5acfbf26df5376d642173f990 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3887ef5b2effa4f4bca5ff3a53a2b5d437fa9d4e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d8817de15cd26bfabb95c28a38c936b6a8544b372dc150b18b329d5cf8d08f0 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a510e1397009afc4e6bcb4dc1e98b9cb729ab0e6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59762c04887890c21e464f19bb54a657372b13ff092f6e10a04de5b0ef4eb53d +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..397be79c35841c0b319178c8c39fc6db7de21d06 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02797d6c692cda1731757cf6032e2ff8329b485aa154f01e25ce22fc7b7e2b6b +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7b57516acde556bb0d118733822cdbdef38146ec --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ace1f1c45444fa5ce6107a84160afb649faa6e5d29c672af1aa8f4cdd372871a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ac593a4a54d3d7fe8e9cc8155f960cdd08eaadf9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bdbac90ee5d1eab59f50545644848e9218aa4192142573740d96fc6ff1f8ce2 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e94c696e3ac74b30e3bd6c82d77b9ab441b9b1ce --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d5624d1c8f0ef28d50a8aedf6665b798a180db0bb16222bb546778961aa9ec9 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4b69534fc2cf292fe5aa97538e74534167e79c0e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2713f8f392352996f37ce5c6b314d1a9e609639fa7fe6b9086146b695bae143 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3e7dea38faa7f57c97126d555190dcfebb6fa914 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f72f224ffa01a308290c6b22d5ef75812514cef02ef5ac06ecf62736c3e4d69 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f3a54806fbb543c75f4163556f005112d8457c4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0ce400776ac898280ef3f8e54a0da80b0ab4c9f22188d481a631ab7dfb8a15e +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..124f1a80bab3b93e540b106108e63e40a425e9ff --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56c3a7b3fae6f16c20a5ac163a08c8009606005f0b126ad9833bd3c568ae2519 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d7c75415ed9d8304b0a4f21044f60bedb614199c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83b79348c528137ffbc234443b998cda9854a860608c1905b3b8501ed3359a73 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d16795e261f8f68d3c294854fc354a3a7ac2dc1a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:691bd9f1b1611f9de20b51cdb93940307e7ebaff9f335ef70718200be12493eb +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6572a3e72acde385c74c87eba10b7f73d81ca740 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba6d505f90aade9b0b8b4de864fbcd0a60d5bf5f585f623bf27a0b8ba191f4f3 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6dfc7a035c38f83c06ec1617776eea7c553eea38 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:700ddafb5c96cddc0c761cbaa44802d77f520b0506234eb2f22a750d3728c369 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3e342550d3e0dc14bd1e0f356e53a0c1e934d645 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21f0e9b96cbdaa1ca896d9363a5e73a2348e83e00aed43e44ecd7635b8255860 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d933d7dc0925cdbf3f7c10d4c9115d3daf9c4c7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24a0787f23d722f58e20159262b4ef4bb32f9683385e37fe54edcf31df7a44be +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff7ea27ef1b91bdb0ccd6750532e0e4fde6e7c8e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db121ea94b9d78df2d694af24019bb9bbd9a545699016a7c860ac3654da27556 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0a3dd9a6bbdfabb5437bf99f9d7d8755faf3cabc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84e6330c6906ef9e2c12e1422c682ea2835f22a19f2fbb90e87d6f8627ec3487 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..46854bc16871f62b34d558f12b96cc4eb7a81ca8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f7b089ec299a5926bc7fdefe3d30d48d7d818f9032cc0e5193f3bb9191ca76b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..20a05879c6c50cd69258a97500e28c6ed5d7a0f6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71a0a500658d020b0b9393a2d4877187802f3c915640489041f4c19c457e30db +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f20efa577f5da394fb024b1c3c979cac299d9ff9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:181cfd02045a89b095733053b141b9426fda96ea2720a18178886e4aced1ed31 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..42059c606ac14e829257de374d70bb970eaaf407 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22776e1e1774d94203f9d0267220bd0d3820ceade8627a9af8ee0440d6270d83 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e929b9756a1537b46722d72ed88eb1eebcfd00cd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:278bee3ca5f6b5992e0547af0c002f4e6fac8b4fb90493468ba0d7d4abbfe482 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4a3b6134b7f592ac3f0d465748f6d1a21153d57b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:860a456a0a549095300e49590b72f23d0469d6570bcd59d3775c457af3e6e78e +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0b1eb88e0275b2d4aa31b9ef442401afb5f2b38e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6bebc68a6a1a57209aedc0678df8c6f6c98a4284896bf2048c1af3df6fec608 +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dae5cb2b1fe2937a0061fe7fb6c3cf170e536408 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11dd3b9e73f9b9363a8168e6c17479b4b72e5c72108db9e35568b2f641c88acf +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1a6d89ac06afd3d11f4fddd85c78aef011fbf92b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a21d9a39f92074b76f71010aeab297b79a7becb6cc7c6d02fcee25d7c4c9bca +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e093414a8d8c4e9234306aa640ff2f3e62bb0115 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4158bff5771a120e269326e0e7523ccb6be3e819b4857acd886a9897ca0c8ebb +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..84392d54a872d8b06e327595110b9ac35e0a166e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82e8fce8e7338d789c3a7e3b3dde0883cec1315948ee60423b095b47a77df72c +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b87b75ac118c3ff00a9c33d31cae87f29a92be4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c4418c3a5c657cb4d9df4322eb6df0823b0733c4eeac01996e7bbff63eab25d +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a73dcd021095a37564f1a0b7d62cbf785cd05f4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e78532656d6ea28e58abcf544adfb6b9c3c6f2eb8a333f3a18b5050771001ef6 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d05691864582f1ba560cd14a879b72e7893ee768 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:137e3e8de818a6f95d9bb348e6d8f9a95ea74e7644f9b8ca3cd195ef07e5e9ac +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7cf5fb406b6fdc8d39a808a4bb72e4f2ef4e7f81 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c469a7bb30e26d791a5b73046948abcabb521765c7b9aadad55ee7e8a700de47 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe5c2802b5c0c7735376eeeb6f28034d829f5d10 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:601a455fba1b5ae3c1891a94f3ee2f2f61232144cf7ad031d4ba814aacd77269 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..651a23e3624612fd9f9a9bf41e1842be3135e708 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:742b63ad7a2b640a894b778ebb1376d81bce819240317c4f2ec99a38154df877 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b427f391adb5f62c220e532f6d5f271497e29ac6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c338d9de68f84e68bad14b29de3f76498a9623319f7e382c074d3c33f61ee614 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d389182cdd5e46bc9bdaf3aa19939cb0273e6ab8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47cdb11e2a99487f02a86ddfe75d8ebdc4c48c562e1d36268c1d02bc137e80d4 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2b4ff96d53e0c8d6aa48736038a6f2094779112e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48ba886386f9f35cac3e849768f0418ebe0b2fe5a558de9e8ac0863d5c1f80f0 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..513e7716e947886b0552d945a99e86b7f85eef44 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2b09251cc8011a9de5f2ce3c94df7be05db9d45a55e5fa99180fbb0149a94a2 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..32cd8aca20eb06213852d2c8187573d961a8f347 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b7652189559ce00e55b4011c5cefbd770a401a43a343b06300ffb40608756a0 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..36adda1a6c760794a52c8b20ba5eaf79cf923eba --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dda67a91ddc83075daab53c42b5cb169c8b3957bf3ceab3c9e7b85fdc796f48 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9bb9141a4cd8347e5431a264e6b54a27be7e937e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:952822e3076f96d896e56222f1921af214d8fa4deb19c6fc90267d4cd07fc2d3 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..134c75de7c98e3751f3c308c0b07862891e0f29f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:833a30959ee896200d0deb343339d3405e3ec027783e7be62b28aefaadc81085 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1e5e0b6a23a3e93b7345eaa7b075cf8c55a303f4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:382ca993539267b72b14271654991e1ae4db9f3f9345e97e7a8f069df55b0d44 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..118668100ea724e39412ac023a3bdc95df67a240 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6e13c6caa072e1f2eea5f23557b89707ad9cf83ed72e10b5c192f901f167d87 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..91926068c1a2f398fb4599d0b1f6d50cc2ad86ec --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:944ceaf360d012f3c831340433e4d8bdedf86d51514d450aeaab1a4f512cdac1 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0296dc95dada47c2e5c35044fe415bca0d4bfb96 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f040b46504962155f57561b5865671b5b526bf0eaa73a15bb1e5ed4cafa8e4ce +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a0106826681798d2fb4ac8cb93bc441e029be0cf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bcb636c5184d6decc96eb3d34c65eaab177282682bbd91f97fa45e43d6839ef +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9578188d096f619aa85ce82bb99387568c94dc77 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edf6cf56f1485ac5bdc48c3216fb49eede09a55f7e73cb5fa22f260f366de49d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..232c6ad335c010dbfd6f0337b354b1d0d786f372 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57f34249dae2f66a10981f5f4b0e214b8eb02c3b85efe52973b8e52e8c039605 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..41f0c814775963008fd20ff681b37f33d4cd3f23 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1afbdf52e68874e13580a69808390f4cb1657a9936676dcd53eb4358cfcd7ba4 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e4760d98fd266ef689a74c24e2d865409c56493c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca4a0333f4342b48d6a3d2316650204e6d001fa82b0e7527012c52487271e15c +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cad8e38d11445de2314ba80c9b37b23d89f7b407 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a635860096f2876dc6dc0a586670eca9cec6507725acf12b8ca92943048462e3 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6d99fb9862d6c759a84f39ef662081e3a8e386d9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26ae4d8a1009bf89b8f5c00d91c21dec6409ecaed22bb191a57b4c24f0d276df +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4055f40fe3c08d391a19096c3a32a8a1951a5515 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4701d14b2037e59b31004bbc2f032b1c1e9d8241f8267088c895148c71b44072 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4b956a58a0c3c25531690209beaa90e899a60f6a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab6717cf0dfd94f8c12f5140dfa998ae0c79c91aa4e098aa85661b1d410a13b5 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d595e1d9806a3cc575466c45f62529aa3b2ca920 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1d7486db01b0fec94138c9fd1b12e894d9a3ed5b61b143b47205b7cd858d940 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e0fb6f3395324625fde3769035b311c3d90a37cc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5972a3e9c1da14a2180b1b07d65d0ec8c30e7405a21cd4d407fca21ce0d00ff +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c080b26fe9a361a255acdfbd77d3bb4b635e3a9e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb1324b2264ef1e1e62a71295bc65aa8e26a46474d118a83dc4bb208ff5e3576 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f1cea23199809ed1e9437bbfb8dd54f1cd4b3e0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8018df076aa868e1996241205e60a1e5fe378db8f10ec96736c2a3517be4c358 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c9e7cac91e410eb37b85e2a4ec78fec5093e376 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00de0cccf0bc1d9d8ca415acec85b6f36b0d5734bfec2a84764f7dc2dbc9efe7 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..afd1858fb1b429e707f4ae6aa575e235ac2e71e9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fdd5aeab1137207429386b3fcf2151ec0a5abda87de52eb8f05a2b180914b05 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e048ce662d54d13a9fdb245fa9eaf117cf901f18 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88d038bb9a0092d19f12122594f7673ad242737d7f3c071ef7ba65746821155d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0f7eaad27e6acd09972b7193c1e1797d8735fba3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56a6a96ecfb8f10d779e0dcf27a9ea5ba8b61aa54f839ac6ad69dd064cd6b3c9 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c22ec4f559d4566770a961189d743e387e2644b6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d852097dbf791c334987df294a78e59a80001c4bba03c285010093e737af0bfe +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..08970296839ab0cd426a858240dadc6b9c312e83 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cededc1b6926140583c145d9f4b00be68d9006a5064bed527c29c19daaa06630 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f5d765958e15aa61ee18f09b535a2b1bbbbad99b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:937f4c0ffef5fca8b2bbd4c9582d2705fd5fa6d8c2c3ba57c9ad008017e80ca5 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b001fbabbfe5f9a9b46799862325c338f9a46d8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb09fb0ea70e9bd3668a6ea37d6fa9799c4546f63d818b3702a35727898bb9d2 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f038c7865cb8dbd0be63824b0664fcbfa37a612 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92d2bfb2842af27c9b4b9ed01d17f6613ab5b2f676b236b99d29009c407a4020 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6ba1eab57943927dcb868e519e39bb0dabde4a3b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:611a9f1e5d51bdd263b0bd0162b2d275e61a9b1257718dfef9d722621cf2d3a0 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..40aeb9ef9e34033b0d0fee4af98aa8cc9f54bf61 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c102a1cdb4847ad8a057f8f46cb1ed7fcb2df4b590e7d2798d4269f7825aa89 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b69a7443f4c0fffe7ad63c8b3ca588e1cf064d46 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:801a8e53e8101a837cbbf5b408fc5a94762818c670bb826072e957b0a3594d62 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a25bf6dde0b0a1341c314dd9ab9fdd6335bec61 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daf3d4332777a5b4599795cfe56337590a26686f2158afb7d3f89743bb92c35d +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..387130c774a4c871d0f4a36c66a7dcd450a8b585 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12ef0090a749345ef1e2178db4a5b30feb235b07dd1fa2a4626dfbab8c9a9d1e +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f3f2ead2752d53fee3e86504ef1b055ddde77bd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:431e38b1fef482f433c7855e153ac30777245912067ef7eed34eca94200fded0 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4d438a546e7716aad9fe69dd22ef7cb5a2a588e7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f8944641d4462443b279091a9013f7b6e5b23a38e38ad82fb146618176d431a +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c93d8924248603173133728e6f8e9af9c0ab0a83 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a79fc09e946bd28322855096847eec77135075fca7ddfac102be80f52cc2f76d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..434a306d937039a6c8259896cbba4064b626f2b4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92acedc99db884f4b2c4c514b413dd413b6a05eaea0f22a1dbb72497256745ab +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bfcb32850090330b6e8a9c00de47ee1d5c0dea4c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42d9a4becd25ec1c73ee9b028bfbef4bf4b93b0d6a9d85fdf22406fc019ba79a +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2e035acc42c74be557561444ea780103cecd214 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f677be383ef2fca075db4e1259d215ee5e1468dbe67c04136fdd53cdab298d3 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ebbf9360a9c1322fe0959735ebe3e39231bea69 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae6e908de0269c28253ceddfde98e8c3ded206adc289300b823e9d376ecc9ecc +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..18f4078987781d4612ce36d9a397b1faf4d56ee5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:179e9dd6a0c4064b0d126525e37989b8e82b9612359de7f3eeb11fa35109ab23 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd86c010e28006f1dbfe94b02a1bf2f69cd83f8b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42bf37d363ee26f71917dcf509bb942a963fbc0c6960e95b13c0363d5a55eaad +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7d2cd7f8d741c459d370a7481aa29ee27b140a63 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb5c8ce8ba244ca959f055d235f0375fc44b684f3da14a2643a3289e61307f30 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..127268df4ee2cf330e657df1d3b8145a35f0d729 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19f479883083561ca053cdacd20f9845650f20610574530b1e15565edc67b0a3 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7a0a6fc434040afd0ca385b2ecc50ec2d7e9c806 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:723d1a15684a3d86039970e9f5b62d01f9478acf783a2860297775f4505f6ee6 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..290ce4ba90fd208b07907f7b0e64cce061bb3148 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95ef666a9744e8c4e213df2265b7ec4b4c529405bb60bd4e26827b1db564d8c5 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..37b723538436c0bba68db3c354915921fb27729e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fafb0aaf58b4dec07db64059b7c8f635e0bf04965e84398384b6f33919117ad9 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe903876e8aa94f2b5d0871a823668398b471afe --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56a7b71424686749ec51bbb6a6cbc2c19eea5c4a9ddcbb8c35486bbb4aa4458d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c024913d115b4ba0d1907f57c1d5320adae2b94 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18a853d72bf571b610a9159fde869a921900a661ae291a8859c7f2de5ed38163 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b562c5ecd25519e611af1ef625c51cba4d8d00d8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97728a31ffe372aad99056e60826c9296389b35161e80380b419a403f57f5b99 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..41ef6a513ba3dec6c4f70b741666c6d069edd450 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4216af59b80ea5e4ed1f0ded4239dc9c0bacae850b609418c6548f11e589e675 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d0c6ab433ee11b3281655db8022c54d0815152d5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:def830f5a9fd728e799ed9a83fa915f216ba2ffc053561b2f3bd84a84f7443b3 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb9e451cf4b91186b8dc8baa220da911ffb654c6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb370b3a3ddf47ae8a8e30668b57454cd8832b7b64cdcb5496c71038eb264218 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c1fc72366a3471947957045a97d9614630dffae --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22a234f16f92bf73a5a59c729baaaea322abd404e862ae8453918474bbbc69cf +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c7f824feafe40b393a986f0b86d74411d55f0607 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec2d15fbf450370e425450b05735d7acd65a00f83d877e6cb0d6fcaddce7bc9e +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b684c688031a1fb771c6e27b848414e73b92104 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da08216b205d11b7012b7e8166db2ad050ef06c7130bb39aa171f0ee81c92256 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4081112c8d60657ced41c38e8f68abda5505f5ac --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b2fbaaea326955d6426cdb55ac90e5918878e20d1fc88d9bec191943f69b8ed +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2bfef0bd4c5ea67978b69a4279763d7904fb414b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66c931894adad6d18025e924ca270665c0a6a53ea9825214b13efdcfb63f03f5 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..08639a0e16ee9d20d27ee2d9131ff81ec1d72564 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e19e209927c792b1401e5de529a1fe7c5574f8d412d588840aceae13ce2a7953 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e0354f64453ae7c5e51efcfdce4b192d01b82032 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa9d58a5fbefe3ef5acd4682ab6417706bf5494cd13466733387b4206762f7b9 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ac60ae4f0a755e624b2f6c7229df6ff601a5788e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6bc1ca0e1d4a017d29465ebeaedf5db50aeee02d0d5d6fb56e4e63e7bcfdba6 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9566a2d5c587817e7be9a631f1f28314e1a8bf20 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fa5add2a8d757cd174fef8e0e7721ada03c90003f88791eb24b1d680b8e84dc +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..82fb83c7a786230f0524404adc035029aece58ce --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:579fe1a670190d0c2442ee19962cf553ab6c8af6f9a2604b095f163bff0adfcb +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0c5c200e7b7332c1483126557f5e7c44d44b4c29 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2f61d0508ed723f9c9944a59f843f41eecc9973b0cf2cabcd373dcecb9e7e93 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f31e96bafab46ceb46b937ab4e09bccd9d5fb7b8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:683e1bb09c7f0baa09e3f6c00724e6306658f5c9d9eb8c1bdba130566603ae6b +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c278250d09ea1b6d32dbc794fa8104ab3d9cab1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c853364cff15777f746d3d12a020f61a989a44d3282522301e512047b469978 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c09ce7bf3c42b96d266f49ede5bcf5aa505c6bc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db1d170a3017e0a4a5620521de87e129c227bbba4e58e5fa94655b613d66d0aa +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c89774f3249ae80d51d9de2487bb05283292e947 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:514ad8bb8919740d144549553a111419eb8d5d95e705a099f5138501ecfaa827 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4cc5314526c9663ec2bfa3e1537c8d711f3ff6b5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:044692fbdd0d2ec4bc7a78b9350de8aec8012955bf72fdd899f83cab0da5eeeb +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f83e1e50ada9790f04155276553f12308a3be91a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:636708d570ae9bbe7357c70c17a874613f0137689203a2e58b3e1918ed6ebcdd +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..72917cdcbd6b74aed991fcd5238e6405909d2ed8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:118c07a633048914c5aec796d9f3486a3e14cbcf6e338f9ec02aa41fd772e9b6 +size 167872599 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6566aa1afa9a11c636bdde4e7380d09d2dc51e66 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd57ae94c4d8f948e5c2e76d11fdb9ad2a48d97a247365998c4a8b02a8fd1baa +size 167872599 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bff4e7d0a6e17b218f637fc38c743aff34e684d2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0274898e59a57f4e7a1945835301bbaa198afa3b9d4c3e29a7531826ca4faa5 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f6741b730fbea071b2baf0f2ff177fd56dc1911b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b5961937aaefb47210ba6ed8d24e8ec54f5b9352a0b065930abf20422963a0f +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3006263819119fa68ef0c84ff603c2ac2cdb919e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:437721da5a4b945ca26fbec799bc5539b0b8437f959844c578f4c4a6a04231d7 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8e45368c2f538f26765db30c9250e9770b33dfbb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:565b8ba4fba379f298e426c2ae62dc3a19f450bee22b7184ec779d8a9d92f720 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f628a3584d7e3c35b0b50f43d4c28691b1e6db63 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c6e326d60015bd640f77c29a84438b54d985527000fa1130dfb7b102bffe09e +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ee40bcab3fdab2540f5d1c305d78f7bd449f5042 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07f1c023b9836ead9bfc95a4777a3bb9fd781cc603e0d0d594b2a2a22c6364c3 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..328b8d03c5a2507880d748fe7d80915ac535e8e0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0766975485cf34f96d6c3fdb1c688b87467808c8c210d3ed13dcfa2aa298852c +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..76b9b4174c8a47b0158d01ae06b9cdcd6808a585 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e417b5ba3838c3ba28007bd1edc327d51dfc02f94e66dec1d9e48fdb265c1bc +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9025f2c6b6685ff1f3d0aa65b871e50c5fab2451 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97d8f394aa82924f5bd0c53cfe60af61774c4e5ff131204ad8d7b38fa68fc0f2 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..de3c1ba0ddb7c0f92428573223dbc03c88269b2a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d4d1b4c1568960b2ea7a1d1ffc645ecd43d7c7c180c7aeb6160a91612e6b2d4 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8bfe0c56db1a47bb1b5a0d091e92c1e03b7a2976 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b8196d080f54d97d6386c4db41c3633d954faf89eabfc7b4c98c11ec6388d83 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf87e65cba9ccf99458cb0abca1cfa15ed41e98a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:182547ca5e813641015da2f97b94ac74d90b6835e9dcbd7b42d2740d4efd20fd +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..107e8ad5303631ed4f409f41ff27cfec83d37664 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8809772e2ab399322ccec3eca57aaaa0501e9b6259d9bb2724f25f1ce9db7d4d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4aa608021084d745cc84c57381bb7d68b042e44e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1995390195e80c814189ba6100d3eca4cb636e8bd15986b5b8e011d523895f4b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff2fd9d4fc7a53061184cb76c4c8da9ed0d4b55b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7e4c28a3c2f929ab3a0955748e85ba5fedd59327cfc06c487eeeb7410961f7d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a538b457701b1a32a6e6fb270479f4ffa4c6dfa9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41352b59b9bb025946e9a566bc49fecf93e365a8826fbb0d87b769205b171f26 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1efcb8d569483716eeb268a0dc7f68d4a38f44d8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adf84004b846f583217ebd9abaf97777fe25a20b3e7a8c54a8f6912ec6768751 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..36436ec8bd7e089ea38b19ffa89aa563bd292fea --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b061fdf2a3016af63fb783a730523d43c95521578280b1a37b7768fbbb21cc8 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c7c894463317cd3e71503f9d2214bbee6fad914 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cd589d610fdc192c5b336b61e2b6dde4a4106670f32dfde502d7cea173f7c13 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1768080e0adea075a87f37f08dd020a8ca398530 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f89e1f7b48a42c5e08bb95f944b94830e4156d9370680a9cf09c962ba9b144f7 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1fb3468cee1f044f554e239ed16764ad8c44c117 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbe1428e8af5adda9d922c566caee30b79645ba0495cd73aae0f774366cdeb7b +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..84da5f1e159827dbcafad909110ede87c4936b39 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de69818cc2704820e812e0e1a6a442c5cadd620143e4b1131ed364d868e4c4d3 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2498d419ddacf9ca5c63bbcce37474e19248f2ae --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70c04a3b4f79495b9a1c81216f6a01b7c821052789fa0c1451ac237fc68e29a1 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d781282e48abfb57e28e0e126d8a874c7c534869 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a86afd02b132fd20b7939ff066a38c53182e93af1023ea8df8039dd3b40bcb3 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc5fc2fa1b84e6f6b2312798a85a757ec1044023 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1770fb0a9aa1fbd77ee910901529f69dd2c3d91b915dfc7bf65a7fa259859721 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a574d9770498da97b986b102d066b7b978dde1d0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b4ad2d7210c9abda02a57b175e0c6b28118e8cc65c38234b6d3462580cbc23c +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eddd12e332f39894b88d30c4d090738f1a2f5aac --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5a973f9a44daa51a9930b105176d6a0aa262a9d7e6922cb038bd6a2d40319f8 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0e576f24b637ff3c9c70903b854ee05a32fa6049 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdbc471072c3f0de45b5b0f4d84b2b87e0b5a2c7f1ea0f81701d1006392614be +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a245e617f5d94083e6f751acbee535226ab198c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a4b894da492b13f1747ea216508e7612598385d289feaa077f34ac176755ddb +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb444635aff593ccf4539eb2d434cb06f29112f8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ef9f90db8548fcf2db8e90b9eade59aec50c401f138916a2e457086d3458229 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..644175e5ebe6f09b9ee12afceff854351b3825c4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5c4dbcc4e85dff65cc01b555f32d4bbeb5888c977164fba92322fb25910fc69 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dd4598c91db7ae661e35774c5cc4c98eab699ebc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98851b7ef4a1ae32eea8efa78f2fc199204cde2baf61928fcc0b3641b7d22e1b +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..24c8d91cba115753ed090d32884e621d9e9e4f8a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d24987327c46fb9de3418f1bed3de678fc1b820846cc1e05591622885cf9a63a +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1a9c589622e857ac0e95cfa3d0816d9600fa60e9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af887b3db7e044c91e78904f0f7ae51663d39a6032d434fd6ac0adf58ecc6fab +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eef83817fbc67e05cf6a212d3af6b24fde350d03 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77b75c942f7343d6c7776d599546367659f36c742b155de2837786095ccb2cae +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d842403ac1cc55ef92ecd8de8bd295f2e6a5c8a5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1696ce26439dec4c4befba0d32d8914d49047d4bac7adeb97c8ec16bd05aafb4 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a0bb6be784ac2e5d494a8ecd62075d6e1d291f0d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d563e43c2bd92044854941aca297cfe37fc97dae1bf60b1359f58fe88b7a763b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..71f3aa78849ad15c2997953998dadbb56f347c28 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdc9162ed7b36ff19b9597c5bc60b389a5cce692f8b9009910279e1a1feee594 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..abf27afc7871ed13a33c82059c6d72d11466f4af --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c25bbabbe406dbb18156be3920aaa827a6cd342898cf9410990be9e53b517ce +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..98151c4d6f82b2139908206bb67331032d16b5f9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dfabb57a581cd11d0d99eba82383bb1e72dab8b84813c2e1a578d1b4cf5a6fa +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e9ac53e336cc5dc72f055e44608a2d4193699ab5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1050ef067e9fbea0df92636dab5be1d6669fc7498b4f125968e50da2be356d7 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa1dada5d0ad74953b2c48892a47f68094d5ad4f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92ae74be0a3285b28cf656d32f53df6666ce3427eb323952f3eae205ac711e7e +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b9d5afd9fb42ca93d3fc14667b90b767c86eb18f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e22ce0ea95eb7f7cf8350986eaac985d7ff2730ca4d4a9af428418187078085c +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a405e22e9b8bcfd9255e21025493a12a2df629e9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a0f9a42a1f9a5d1f173d2997ba5d4f2a04e97f0e1a533016410e65a2c7f5655 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9557ac4112903aa1963e51a737c727ca180ea16f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abbe1d97b3663e26c7107a2890dd98fb245ee1bed5b079818b871f0f148550b3 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4d6701ba81538851cc9ec952381535363e91c0a7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b817a74b23cc604d10e4a43b0922cedd55357a28ba4b1e3869aeeb4681e696fb +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0386c4983c04acb4673ab6a71dd38ff42207da7b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc43ffff259146712a0b02f211ba4d6960d46e2a8700eac389c037fc3a7e2e91 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6c23a95e418a7597fd02626fdb64ec9bd9fe192 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9eebfb93dd2bcdae88887d78d8abab6f7d14f0b5b1ada31492157de599332075 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6c462c9a159f04157e1976843fdb73924ed63733 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58cf2b4847e9a7685817b69fa09e3db1008d6156c60757e04f9ebc513d2a3185 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c7c6355ef43d01a8ca39919675154aecf603c38 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0a9434c2a7bc1f6f87f26e84d6d2ade7f66f30243fe6f790547f18ee9cb30a4 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..964f976e4ceadeef09806ac213fad3ac5ed34929 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca2903c9dfaf90b3a54170bcbde8a40a6d2c4c40d576835451c6f6ec4888589e +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6c94395b921333f453a294fae3516c629e8b9b02 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1764e5edf6202043bdcf161353ebf457f47d79426084fec06d5f93f062860d2c +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..133476e40f209d684db16f33c10024d199ee31a3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b20d694d270f01234feb2726e9df88685b34785f0944110c99c26db621797541 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..54129621f0499c601452dfd197bbfcdde00ed355 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2f017de3ddfbd8b2483eee9b05e24b899085a077acd2474fb2ae2e1fdd4db8f +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c789b3a25c4345ca9a55d1b3b5c7779b063bcff --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4af3c3626aeafff04d0693447c39033d841abcbb672808dc628b8533223141d5 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e05d47f29ff05656b455d71367d4e269d41b84e7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc597b9dd035ec530f997ef4dd6601e57e712a089a1b6950b8e9797dd4286ceb +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a6a6c16040e69354cb1d4e4134f448db27bbc0e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d78d6d24913a69ab80225cb7ac36b14956bdb489c6d28c07e5c844e43189594d +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7652d065380b446320d65dad8c70e17f260a7cb7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f04d82a885fbd504c3a89ae809e9565e2e2bb56ef101b6406f4a07565905f92 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..46827479c2dfc616d406cccb237bb2d9db9ee397 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00eda5e9f1a5df974b4613cf6e972f58a073419259883d692bf28d525126291d +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..139019a7c1f50adb20f6a752d86b82e93b0346ed --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ede64ee9e999c83aea05f76e702f2f199c1395e16a13a18e25a3d9fb3a00f226 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e86973a1e0dc7e10296bd1bb6dd3f9004a35bbca --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:052b21179ce4e65a6e0f3614f7f38aa1f748493f751b9ccb2ff64060967496eb +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a14067fd5c33ec24a18967edf4bcc4569e29e88 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c587701d88b68d3d6bcb748e4b59e6266e60a092ace362b928416a32c81570d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d66d3ebfeacad977822f935cc3dddaed1e68b040 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:808c2accf494d1b38eef43b8e1d38eaccb06576ad8ee69026e548861aa64d64c +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7899dacb573be5fcba15ea077475dc1e46cd3087 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:048116111c7c75734ba0b1d0fe9fd0997396b87731730028dad2cdb764a1931b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4110cb1555ec6729dc9ba33413803209fb721fcb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19d936940c49f53b11d66def8a4d3643dae5d2e059a8f638ef4994c7849e5303 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1073a71de46284b6800b9f65c65da3d0a32d9e24 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c4b68113a35749badb81d753f4665ffebfe6326e2316bbbd72133afae83bc28 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_01-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ed76296d21985eedc202452ae3ea7e61c484ae73 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ecbf9634521787b22c5bf4d4b7b22dbc42d82331211c77a064b0ed5684a8820 +size 153552131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_01-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_01-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b2f96622128604fcc602429f8aeb627ce70bd9f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_01-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a2dd7becb83c5aaa1168f920e15dddb9527ecb14023e7e1bdaf889b81fd8497 +size 153552131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_03-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e186a78c6d8b170e073d504744ccfdf7301e7ba7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33cd0d76e1cdba17b998d2a0131cdcdce097a40d81a31ff2316aece0835a7a0c +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_03-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_03-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..649130b409a3179c708da320d46141ad0a5e2100 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_03-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd8b3ce27a871d8134e4f99c41cc51c8beaa70baafca1494d297f3e873b89edc +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_04-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..16d50a9d2a8f722f645fe58eb1f2d58e0b9c66a0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23d1da5d1e26d95f09ab8056e0743f44be782e943af5c8727264ab794e372340 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_04-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_04-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2fe8089b811abcea80c13faf9aaeeb433320f08b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_04-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d724209e5199b49d9e5747cff5259ef603dc9ba393e625bee4aba10132cea760 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_05-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..154e37d8baf506e44e3672bdc991ed3f630f7568 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa5a8974cc87781fa30c70f3d2854bf0d748049309f3ef741a8043e35f050b3f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_05-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_05-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bd9e663e2016eadd1609e16916150d4090b86824 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_05-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:802340c68980f836cd238ba10ab8bd178b8861745c13846e2774a435b99e2ebc +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_06-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..52bffeebb07a860690b312474d808e10a1a611e2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6ffe65d94b68d2a3d0ad49f981294161ec4ddbed6aac0bb69a03f32cda7c3c3 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_06-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_06-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..63905bd4510eccdeb76d50b219c73acf8c735e14 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_06-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40c10b58cd9676e663c5c658acbf4591b408e041fefec537e4c86f1469bcabb5 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_07-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..26f8f183ccad797ef398f5ded45da52638ed8b8f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50b84354003fc0cd736adf1a22a3d7d0070869346ca4d3b0ffc8075c260c9689 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_07-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_07-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2edaa21d096037f54c9bea8f7bf15964b3096b2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_07-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fee57595241b55a2c20f8d3c0a7d882978316dc882663fc08899083e657a781e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_08-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a0a1961a781cf680c81655fed0f511c7cef5833f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2945447bf1007307657b106dd32d858a2736a259d083c59ab40a409e94c38db3 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_08-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_08-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f8d8292b8c611c2e822704f5523176138f886fb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_08-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65287a5c911aecd82344d039b3d05dc5edc8c04dd7449e74dccdfaf22ab9fea3 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_09-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..83768fcc3326fe0fb145223a1f10026a47167237 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03088beb47bba81d88d2b1404b7dbf8449a91f832692bddec82ae738ec895e3f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_09-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_09-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7eaa7330903892a99a81998f8336177be6b96fd5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_09-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11bc17b7262f62fdf234483b6054887bdda67b56177cf9860470b98d5dd09211 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_10-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f727f23a021d121a8994f473d6eb025f201a59f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07600475fa22d23b74c4bda13d50bd6dec55e102f4258b55c1e7f8b3d0dffc32 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_10-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_10-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d3058137d1071ab4b8ef4dca8cb8f40e9cc90270 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_10-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d1729f97127f048c40290611fee24bd91d5c0ac71b1f8de1ab00ffe2356d37a +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_11-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..79cc0bf0acea42fceba85bc9861c48687e580180 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81e0708fdf431a9ac90c99aa881be6e81213adc05d44b088838ad872e2f214d5 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_11-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_11-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d4c545bad2feef7ef8ceee768e856ec9e76e9dc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_11-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:136c2c37418d74e9bff8b238cedc59b177d68fabd61309fb5723e9cccf8e3e70 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_12-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f03281c0d6551397dc1d4fc2afca3748d444d0ea --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0660b8b46c6f210f04d48d418b0aa024ba47fee33c65a1f6eabeda151d7fdab +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_12-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_12-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f5aa3cb81722f9b875a32737cfc34022e8cb81eb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_12-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1e1ed8cfcf21e53929156e6ab5cf152bdd834bb8d79141ccfce1e9536dab037 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_13-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0a055be0d7a5bee3d5f15b23cf3dd75010dd837b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26f02d20e2d80cdfe3bca1fb14efe88d8e67c9869000495ebde7c47159879649 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_13-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_13-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..04cd5a350c7a27c9ed4552a6027086fff79ee0ad --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_13-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8783869322bd478e7f1861777096b50024d5c698e1f8ab6ac6a1b9c66e152e52 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_14-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a69c78fc9142708dd88493ad8b481742f3aacf5b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe70d5afc93d6e60eae5b8cc408876db205a941537533737c1f39e9037ad5e4a +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_14-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_14-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7523ac013ee2aebf76b39276c2511c8622bc37f4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_14-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4530d45f90f3df048a05cae98b3b5b663133efb3ea7d4f502941b386535ef4d +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_15-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff1481061f358b988736686007588837dd52a0df --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c89c2e2bdd2176c7e771dfba76c7bcdd87b5da7dbcca964aaa86fd3adf3bbf8 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_15-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_15-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..91f386d69c4e327e4a62d51839628ae46edecf4f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_15-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ee71b9345c8ee91c2a3f24c7432245f78ad6c024088f110032ce9279319058f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_16-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..001024663086bcd6578c75f91e9c1e9ae468c319 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4eb4b01290edf3820f158f7d203287f4dc39f04e511dbee1d056d0f8beeb0f05 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_16-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_16-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d2e2b45bf43a03cb9a5af0b7b84c90e27bd51e38 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_16-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7122daac71bebfc094b453d8b9d0439d0cf7a46af761ae17b958ee57968d6bb9 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_17-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c74d1bf0093099d86a25c895f01e452081156f40 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39af6a940d7ace4684830227968ae4f824f25c52e91da3edadb740755e2effeb +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_17-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_17-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..861b0875e8d4a04c8061ab4af4235f319d2a321c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_17-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebe3a65eeeb59297abef14d3b13917e526e0d22a2463a337f366590fa3d5e7e2 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_18-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2897a0a2b0069f874c15776184e02f746bbfb93e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5aa125fbb03733c8c7d457687d4a9a155232e32c1c5452d4f88edc5a69aa191 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_18-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_18-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..492588369a9f2f04f56014b770a008cf6c6370c9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_18-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99c6e380d0386232e12b113dbad601d5ff06991fa9a000073355330c84460ed8 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_19-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..73f1c5022d994759aaadc6947cd25ab5d6ef8f68 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e7e27faed4ecc36771899048ca755e4db26730738c571db9ff8808f43148b22 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_19-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_19-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff4abe97a503d26520c6f27930eefdec8832d2e7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_19-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6dcc7acf7d474a9f4c6ddf569e9e582249ec9cc922ba8de54018da792d6e26ea +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_20-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..acd3315665b694777711b4d54d19f7d2fccf624b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e13fe8faa9916d905f0e5090b0f468e6b6f33f9aca68351a30ef70ad385c2c14 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_20-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_20-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..332f92c820af779580e9560b842262cc092b759e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_20-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c06e5d5626ca38038fcf3bc07db013aa11a604079c001325cbf30013a8efcb4f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_21-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ac7a3db326afa104b52c7eedd7b69c53ac3e509 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:325eed1af62afd99822ce3af8317d53e6f73a0b3b1caf3a4fee3a0b5994f2a02 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_21-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_21-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b80f85e87cf684a42d563c3a48209109087c72a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_21-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25a9eeca192fa2d8f78e8494cf0e5c5660bd4bba21f24334936dee66f8594bec +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_22-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_22-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..88d96b76c2848f0292c3a09dabce785dfb998d87 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_22-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73ffabec5f1833ee98d3479cee16a7ac8bcdb659933d96cf7071a34d87bf1146 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_22-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_22-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..30ba58e759259524453cfc281574e65002fd48f5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_22-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13d65fa90c53f947035bfd66d1338cdb94b73e2a05f2dc9dcc63bdcebaa3fe85 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_23-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0caece2da72aff98f388c3167b3a0b634c1a046c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0f215b8a2468e3af83068398135a9308cb83ccd26d1b89df15cb8fc28c3814e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_23-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_23-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..512ee03e5d5b8e031b8e9070fb0058521d436456 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_23-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4adba4f427e334f3549e6083c03084155cdd7b02522b7f1b4dd368512f65f223 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_24-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_24-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..79560c6e99e6eb1fd3a7ac17763a961f3359d1db --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_24-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c539d0fe11ec59ec7f95692d4d0143bdc69fd97c73c14791704a27d404e3db7a +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_24-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_24-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..399d2fd6f94da7b730a8063408ba9e6ba5d62ef1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_24-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17f3a71aa6f47e839e19565871b1ec5b6b264c98a86529d71a46e18ccef53815 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_25-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_25-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0f7a0b0f607d9f873f7d4f7993bf7303cb153531 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_25-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1931db3839cca73af74b8e2c0300d9db3f0a10702885c16c40e419263be5931a +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_25-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_25-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..42146706bcc7acba1b57deae654d9f29b2979928 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_25-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:271bb0c01b4aa8db890dfa3abc6071c9f3aed64960ec770bb53ae70fdee79b61 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_26-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_26-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c101a76c6c89f0d1f832c6efc1c6dbe7d33a1b83 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_26-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9ffb6a14606fe4a44eead249789e240ca665f1cb53c9e4bdbff8f73dac4c3a1 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_26-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_26-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8411ed55ff43bbf9ae01ece6b1a39c8a70ba17c5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_26-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0cf5598335d3fa3c0f3607c56e5659bcda66a7c16682da92e2f68fbeb5ac803 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_27-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_27-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d15224fabee98aa31167fc31bcc68a85a2cef5de --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_27-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0366927fd37727330180233a5ccced06323ab4e61130f186706af8d8ae2c4a5a +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_27-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_27-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9e0379509d638051044c9841b99ba7e3e8059fcf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_27-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c27d58302a9396bec55a9aa071058a4837aa16d3d954b6744620d65928cd9f18 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_28-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_28-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bcaf7b156b4961db5a7418333243eefc10a196df --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_28-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d08e17950845de5c71b28d6ca695a05bbd063946d76b5284db7578729bef31b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_28-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_28-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3de7be7850d28480ac8ae5599160a13f12d2133e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_28-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d9917bd53c52d2e7f78fb68fe9581a24c6e7de0cda055c07e13a3288d50dbb3 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_29-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_29-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fd6164085b035ce53e37c4c8f62ad71a287e32ef --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_29-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b60ddf0cc04188eb312bed5434b750f2d4a5519e1512e875f6d18cff844e70c +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_29-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_29-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fbf1ea0416361e10b167677d4c3d60f82367d206 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_29-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6faf5be24d764a1bfb174232678b048fc4c2994a4d62d631778c2633df6f3afe +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_30-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_30-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ffa354c57b8fb3cb8ace2f5f7fdc394e7beecbe2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_30-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfacd122db25fb03f9375ffacede444050679c8b3c46ff19863124e811a2164a +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_30-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_30-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..924f31350a16e3d83c8401ef6fec993c64097951 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_30-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15f8903789192fb467ca73d469de5e94d8791f168170b93d0ab505b6d63b571e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_31-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_31-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b55c7d11d85da9af823bf4bcc614b6bf8582ba41 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_31-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fef51766a016c5700b73d7d6b3ae7079ec75d09d43b701d3754160969194c6e9 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_31-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_31-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..67b476e976c9f4cadfae6a1abcfcde0e3d06ea0b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_31-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92655490bc96692ce7849f67b11d9a8e162d9cdc938ddf9a2b7d0d28f36b771d +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_32-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_32-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1afbb95a51401ddce77a805c3a204a1d4302355d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_32-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0e545ddc7ed839e67d0ce764df10d3e680699aad7fd0df974839c8384bcd9a7 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_32-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_32-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1e57d9d1991d20524fa462291a9a74e06a4d4b60 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_32-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb9bdf829c7ffd481baa6e3c5cb99713909809553d66230d1acfbec011b16164 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_33-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_33-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2961d80ca65ae24c9e3997e4635fed0ca337c391 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_33-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc736d00fdc0625a5b41582a26b657e2750478c303521ff2a4db9fd6c0a0e71e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_33-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_33-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..08e8fc3d72bdacc8b04710bd5339cc4d0145fc8a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_33-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92cd3601a0410e7814346b147a467a52fbd7140313e90757e021c3bf44711714 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_34-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_34-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e8e36538795b442401bf5a2b3d8c555014ea6a0e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_34-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53a5d72f73ea5921dee6e71e80031e023c82bc9cbb6a27f4ba87e6ee9c230ff3 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_34-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_34-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2b23f723a8a5a17a78d289837a66044f560afd49 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_34-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b03fff15f6b945b608b3985945a4aab76cf031bd0b6659251efca2eaa8c35eb +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_35-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_35-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b796119bae91854e0e567f6c4e74f7b5f61e13b4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_35-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cb23681b6d8de47376580719528e5304f876b06a5254724fd927f346bcc1e05 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_35-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_35-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8868a6e9fa6d935c645db1e4b6fa3c0ddcce58d4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_35-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:014378fc70a25b2606bb7ca1fc9e8fa36d8b78699fc51b924134035ef4d38f6c +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_36-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_36-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..531218b17488fbaebbfa49eccc1c78f04b518b17 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_36-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9706e74658292a14021b9d67e351289afb24c9129a2b6a9cd4e1b10e1000be0 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_36-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_36-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..691ba0b75eabc1ca7c0b00e46cb1c793261cdbef --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_36-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1756af3e0df15ed372f6f4913305018e04e0bf99e877e1c323ecf63911df8e9 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_37-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_37-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2c722a8938defba3b4d67e5677a5f889e69e303b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_37-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4d759e645145ab4d0a87664fe27f4aaf40287a100c967d07aa4abbd10329f8b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_37-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_37-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..03b5a7ea07e7339d8ea99e334af9c852126150f3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_37-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95cc013724a9c1c5bb4c560ccee159080db20958577a7df55fdbbf1dd80724b6 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_38-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_38-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a5b57d3ead9e90a44b9380a01d1fe0e852ef4de5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_38-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bc82bbb94a7f5279ab44955dccdb440b0154f5ab4ecf83a0ef21a0eb4fc06a7 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_38-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_38-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc238d80c7a8d54a39db51c871c95a1f4898353b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_38-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dbfed388ce74c0dcc89af499a96af6d4ca7a07337b782ba5320fdcb5c92e057 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_40-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_40-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..05ca3b3ce4fe0b160b47e1022c2b3a3f093f994d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_40-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9730192b38122267b54055b728346557d54b2c80e359fba9dbc0f79caf27350 +size 12483 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_40-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_40-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..35d70627616926468021ad1c7cddcf75517a32a1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/layer_40-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03f3e7141d577b855713e7f10d038028c06cc6c516c5d08177f8a34b5349dba1 +size 12483 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/mp_rank_00_model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..68fe020507200149b7e8574be2b782305fa133af --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ee035caffcb6399891ee40d7107a49e8a52d2148f7ca5bffd508e10e389d810 +size 51443 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/mp_rank_01_model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/mp_rank_01_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c9368e442958d8dd3549fe4f7ca1c977c06db98c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step10000/mp_rank_01_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf9b6440e08954eb65ee49548e7f60e433e7917b0e01bf7fd16ae880ccea26bc +size 51443 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..baf800af56d262c2a5afcdcce806989bb046a5d5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5129f5894c7ea0c8d5fb8cb2f7ae64fd2457eb5ce798564c42603433ea88b04 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ab9cec55268f1daee235f79032e957dfed2e9fa0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:873c9dcc0f19a8347da4179b18abfad978732e395bc8e076fad6f7f697ce5a74 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ba5c8731326b699bfe868391fae839ec26e7c342 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5c812daa6531aeaa413160f25cfd375cb3f8c413247927803f899b2fc87bb38 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5e609d047a2f267ee629ffaf9028e4e2f6a4afb9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f4137893c469a1b97029fd0a03108e22964a186d00483110dc072cb2ef0f42a +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a7a97ce5856b7dccd31eae19a6880941bc98ee2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f85fc09719970f57bd2de731e703223e4df024d349052bb5feb924fc9c9cc05 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a5fc66434122e55239605689709d8f1ccb45ea72 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a18323cf78060335f66bdc362bc6bb68c846a8f73625da3fdfce1ff56dadeb3 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..867f13b7999d540af3355c607b28357edac0b8da --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d525decf4f07b0af7a5542298d4655d735ba899b0f460367c4702156522cdb1 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0556d55d54721acf23bd7cf90244f4b7413646c4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d980522f507e7e29c5340d19fb42f667e7c5718864677fe4ed65410cf1a6995 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..69bfe8c6b1fa2944f99a92ef0a8f2e8a3ddac0d5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8eb3de06d74a64106b7d13ed33ebba37335034876da6e84e6179aa85db36b2e +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b28c9d3627e76aafe03621c83ae47c5fb612b1d3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da83bcc64a9c204935288b140cd6a928dd0739c0f753a81da545990c277986d3 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e06455c63ba2ca66f0f67e5246afc4554159745b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7af0806f337113030cfa8faa4df9814513b3e0ca015e13b08620341628b300a9 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1febc3773610976e03de70f762376c703fb4b20e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97d4e960309843372ffe5bdd3677ca2f7c816d25d14a2c45cfe109efd28a826e +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..af72b1f838533d1ee1915d17a78205cd92750ff6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a3cea6db2036ac7a698c82a38b63ed813c7032206742aa81a796871729d459f +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..30f65592877889bd87c61956e541f8f125d9acd3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9aad6c8ca722878a7089f5073c596583ea92c46ac9f936998410a72e66c496db +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1b79c7292529acc46d477b5f8dcab5975d86ff8d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d575c298ecaa54e9fde4a48371b21fa5eded9f11519a9dbcd90d4959787cee8b +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..98f412988c6265a354e149913d62bf90db9eed6d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac15784bbf78bc892f493ac06f0f373ed3193d3f043315d92fd726c6ec5296fb +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8e7e158f2dca2513546e4a18eaac707b1cfeca0c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d826b9751a664b397662f19010eb2717c42ea4660e468093812bccd792404a1a +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ee362643ffb91e832aab669686343d7de68c0d77 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b062de6af08641a8e1bfa2a0697e3c993b52273eb126af48dad771bfd408b65a +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6845fe0a0e608c8531367ffe28ff38ef104d456b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf37b8c0c6284053413fbc2465044f2c28966bc158587683f4ba7a073e2734ab +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8df8d0ec2c41033ae33fe96ce7b26495a98e8b79 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26e2b718ec9e1a75e752973e9c124e2feff947ee0af71fcc83e4310cf69d0f98 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b7987d1e16a098a4d5583375ae93b71db872f4aa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5350d1d867bf2eeee03cd989b80e53177a0147488a18d902abe51758f351a4a0 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..151be37e863e17003302112cff82d8ce9a8e8acb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bd1e2fa97aced0de2c42442c5e2101a52f77f2f6dec204316111d6583cca781 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..482c84ef62dc5e32e96eea753f5d9c38b325b1b5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a22ecaad4ec41e9c3ac3d0040af816923e7fad00c357a49520e426894a7d2c8 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..47a62fe7446ceec042053c4692672c6d2bf73737 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd1a2620c0772fadca1e85ecc536dcf2f1411d93e814a1a2c29172ac9b0bc56e +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0c93014863540ba4e473d2a69172550739c0b5ae --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2416e45ada7abd19d6bc9a3ef409ed1da2a0136c1783816340341d93f0c683f3 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aa33ec5e521c1bdee4b0b06fff4addf1b2caf148 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46d62858695803b43011c2574c0d5629f47b56f291c443c1234ec57bc9e3f727 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b3feb0cded8102b38637242262daf08b3718bb6a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf6509fc407ede437a4dae7498abdd74fbf5e1999f944970e8bd618a4fcc1862 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..841e81c85c94ac0708669aa81e18463fa66d0aa2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b0173d2f7528e83fe7f24c82c36bb2ae04e7a7d1e8cc4cdbe0702c0ac0866dd +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..05d6ce1d8513c74650dafa17d88d9f61895ff6e8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9c826f92ddff83b145f92492059686c719c3e6eeabb0d69ed43f0c82b4fba48 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3d846554f5acfc2ba0ea8639c91918b2e59a28f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:375adc07b89eff86af293f9c2863ffbd56300573426149d2bcbe5a2f883293b0 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..87592fe14a8995c7725f9d16de003494c17d76b7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbc0a005c176892afd76a56b76201901bba5676e316e4465ee107e5937cca25d +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..837d98bb81c1d92d9a4ad57623cf7dc074b7c455 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7167c27063fa2b86fea3e5e7cbca626f36642405ff79576ef6227547fda7b0f5 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..73b425e0fd891d6bf7d9ff4072aef5220abf4d49 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:475c95050b956b650968c985d30e1cfbf7b995a117f355fd9ffbda97c2908504 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8047f95030c19b239641fcef6247b8e197f9e016 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8beb6c3f1c6e3c0e34b1081d34e096c70a0fa649122f4099309a2dc2202938fb +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6fe4803ad3b8b3871b1b4a62618ea81b69d45d9f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25504b25209065230d2d2edee31c8f73eda65197fbb3aabf6132d9f486ca7ffe +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f146b840a743e544ad356dab338898493b64fbe --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68f709eaeb546cd5c0f92f07b25b900fd6a19419dacf3a585ac8137b177ed0d0 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7e8b56c1801af36a676d4b436cc1927f8a7d169b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eca97238289fdcb289f1002f8585a35c39e153a3189f3630df0efebab1417f5f +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..87fe56a58805a0aec41549700ae90cb0a49fdfb1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cca4ea495ff5cc6bc211e55fedf2c6eeb5cba621ffe055e21675a08912fe50a6 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9e2718ff46a1249b246b16bed8f5df61c133a3a1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ff97fe251ce02b555419685c30f6887c27b7c658c9f69bfdb69ca638e79e49d +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1a77dcdd971ad6e84e619a8384708cda3f7a634b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb6e7380712121e8b6d5e2d3fc673107670bdd4531fdc220c1935ef7295b0683 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e954d160d3cc829b94da9169f24e8153ac7eacd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2b3852f5613df137478d4614cf4350ca4dbdbd32f77f5b93c57601825b80b13 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a4751e17ac30f60a328f7337aba581d311cf207 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb441eaab2fcbe23c8e9f2e69ff007b52ffa0662a4cdc88d059c4cb455c4d897 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad5f3440d027947b670a1308f0f35d9592b2b470 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2def43d59cb3a5322c211f51ee3a4b2f680a34e94c3865ffdce2baa1e58ca15 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a9785fca04c75beffef9c7af784e317e56cd9783 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:387950321c8b6cc78c8fb0abab0609a2a240b7605f987e39c69bc383f9dcb642 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..df8a91674491e27ffc0e5d07cd8d07979118bce7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3b4e0f83392d39f4fc525215837bde4490b853ce1abd8c2ebf8eb32acd00c0c +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dac43970773e4c48538113ed68cd59645e8a822b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b93ce0f7a8c1abbf8907a215f65af7ec88a31a80b55a9082fcd27db311c72ea +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad5fe69b59499b9fad28d161ef7ff12f5c9a40a8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:009b8c8e9c5eb0ec3ac1924ea1d858f4a45bc340a8ae139f087815d65e03e471 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..29e7bcfb548f64f6b852ce35c8c8016c48b9faf7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f652ad6eac45b9d494c6c5f932c64a87fa607b723b6cde7f7c700c4af7b1390 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bc4294a47349a5c7217fdc1c486bc40d65ceb404 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26671e389301e6b78fc6d02a6cba23333873c5cd00ec99e3f36a928c7c3c7d2c +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cea70d975d1ec88511b40e60ffc962912af9ae00 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:471834fd020223c820d762ce530dd634b7a416f16374533def1c80ac0a7f8cbb +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..be53b7a6363d9fef3d345919d6a8f23930c1c8aa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dd50451e3206cda99301299c226b8ffa4eef51a01c4826edc3ddbdcc147d7bc +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cbb00f95639a107e58b880a47276efcebebcc754 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98dae4261bd0eaf132fb0c5639d6a04f52d57886f98b49c3753f81cb2277fb17 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..645448c5ae0c5da95f49c17ce026246d056e9f84 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:212f8e654574e4f7a6c3ef9a88f0f520b138d751ecb23f02568ff35ee578983c +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d9fb51a0db141f7d5e7da25d5c0aea5fb42a877 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0251db9651af8a7f2c745c1463c2dcf4d78f1abb1a79ac7856ea8e4b0ae29aff +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc74673d8ee17fa5fa34ea2b9c2409822dd04252 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8688a13fe3c0ff384952daa15ca596ba2a7bc51344e045bdffad0711346d1d64 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c4875152c476b1c3f8215e9cc291595d6bdd5086 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3873ca01ea654b60d6a21fc5c543c307dbd1a6228c74cc3d9aa1214c657a9f22 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..850f8e664ba45ca067568cf4f936abf07c1b0c3a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2ed5a16ca9e072fdcaafcae3a961efe2f95f0dd42b4120a130b07fcd00e34c5 +size 167872941 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..19fb7d125674527e1388050473701b6fe2ab563a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:917df77c02e28a8be62ec78934427efd784a89b31450cff7a94267736f45efc6 +size 167872941 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..82507ea10891c8c2d0c4ebfcca1a113bdff45891 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10ff797d59c563cce6547d3795b4d662f3a2026c0824983c59b8497d5d7568b8 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d9121fb41e2f109b44403640569674d6caf90d8f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0c43b785e3a5045ea589c2134f93df56128d5c54a3d0c298b5b67f2e3916af6 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..35c750ef2e74913fd04fae1db32af76019344a10 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f31751aca636c2bf0f0b9a3a06180a6ef6f8e915f3b67f4f6244a13c667c4ec +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..29bdc515c5a10d9365f07228a50b8553466c8c95 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c185623b029d9b211ca75b26121eb324929511b3f8149ad7b8d16029074093c +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..706c8087433db0ca2dc11c957531b2b0d87f9fd5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0403f3af3ea413b4aa06915f212792a2f3dd4da78e155d9dd7032cabfa82a4be +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4aa033528d12ff4892f2ee1591f8493472345430 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e6e6b6690ef3b6910941e1adc34c1466ab03634b37824ca677ffc46f3039c41 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..844af353b210d462ec722e51c7e3ef2e2f70a868 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:346217d1b509b7678c515cc208a3108de625497d3d83f511e7e7cdad7a5cc722 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ed87dec080c523bd4b4876d849833d3daddac16d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:791e3dbb3322e630d402d67a743a0132358ead84198c15aa73e2c435a06b3d3c +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1743d71a374a8c4c8279ca346c1099f7dbdc04a9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:389661c2599035b24878daa3e26f717696078517358e04d6eacb30c083024b81 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c9ffde5d39ed7ef2677bd6868d4713987a36a81e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:305751e6b0a93197c33ed0afc20c0c54429a227c7ffbeb61e27d42ba79b3492f +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6fc76c861bb8b73b1b8eee1740306343a33ddfd7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ee863a105753f2dd2212242546c4569c72777b1c393aa8c96f99d0af298dce +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b776d8aa1dd67d66d2add549e48b0d4e4a3b638d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac289e2994cb1754c53ca46356ee7e7bb3ea9aef78a1fcb4f0ffc18cb04cb02b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e31644202809e1bbb515dd20686d6f724e852c7e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59b19ca3eafa4679f384d22accbfbc4baadcb4bb0ba2d68e50e3771bbebc4daa +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6b50be440e07e33084a07cca35c93d3ed368e17 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38f8f05dc894fb6b56f3f1da6799cc6a3385cb280daeb44ef974583aeb701446 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c0c5144fe0561ff1e2390a38fb2720f35ce89b89 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5be0282b68a621a8fcbac36eb0f52cdfaf58b114cbeda700affa7c62c86bc080 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f3fe7e755346c489cff87123fbfd5eba874f551 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21e45ae0bc511e3c94c466d88dd1438a09b10d0a20fb4a18f0c3498519b30fe5 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aad377fc37524c1329c564a49fb7dc6b9a83bce4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4db872f86bcb9c1cc56dec34eae9314b38914232c396bb824755ffebd47b4cb +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..682a9bbf51afa38785c23585ae20be4c81fd3551 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26651b77545ebacca2654c59e68df59b2626f2c1c3536e95f6f2e2984308cd80 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e15ee18d4c0452765f36654e7689936804653b4b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a986fbfa63bb0504800b2949032b24c9b4e42b8ceff11399d9bbdd28e9efec0 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b073b6504f38339f0a5e8af1dff738001ff52e74 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93ed7b5fab272c779eb0a22bf9134d429fe91bf9a1accca64ca60df123598df9 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7230807e9e4cdad5487058f883a6b2373ebb95b5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5e2cc09bd1d227c836bbcd0ce74b108fa1905541ebfdd6e2f15b40b15c45340 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0c3dc2e3c639b21f188c9ec39f0b95e6026cb1fc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fe19e153a896c8af385e7a145e0b48d570d2b55ec67fd0dba35a9290942eade +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9630f34c0126583365a23710f513b15b82751b1c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa998380e10291545cabce3ff1c8edcc42b19484ae9ad0eafeb3de7672569206 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d2722eb55e848ff6c848d3e3c179bda0dc6c2d52 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9fb0fd0c5b5c4b9dca6477ca5c3ede9ba03172ea100a14e3200efd2cc98bed1 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eabe8e25c1c84f9a949440c073657b0bc8fa9fab --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e95ecf178a890bd9455ae33a37a41e65eff83e2940a905b40cfe3dfdbedad65f +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5850f3b19a4c618866919b1038208b847e0486f3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d71048f6fe77ec1c5c44b169838869e319722aa4465c7501b717db0f16aee880 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..953a8b8c75d604377368d1d1bbb9550edbe9f80e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0d7d76dcc5c84d4b98ed3101471b92d4f945d0ee6742d87e764707366b1b27c +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c0e2d957eeece310169ea92990d67bccc0ef3480 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78dcc0082d3c04c27992bba699414b29311121d55c7cbcf658d47ac018d1e052 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b62828f0578b7f5ec7025efcb80dcd8a2a33096d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:130dc16b0f54a2729e411ff28f1d352a3d42ed3b44be80a38a18940a07fd5668 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..251c2c3eb9c4f168efc74991b61d8a8ce43cc088 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fbc27bf57635d1fa4c9780cd2dcf8396ca7017e9c2b2fbdeb51326d027d430c +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dee3f26f3c1e304c6850a46f8c8ef81086a68e59 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efe34a6186f33fa59cab7ba632514b6dfd877b4b9c41ad9f02649ecb75ddaf2f +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b04075ba44416930ab8fb90b3f44f9d24f65751e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01eebf83cb3692f77528932ebafbcba0e40faa2daac18bb9da06759e60acfb70 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7d6907596833f3fcfe44d1f7e57c64e961198366 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5b999169a99e6a4503e9d03335db140e7705e054697a908cf42df8eab28fd43 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..62d307eee2119316b164a5d1c766d99c82e1a95d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7e0d487e62efd93767b43fb6a0b7ed36627c01110d08350943b16f151ebe55c +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c5da49c908e9cb835c9f51aa36e29bce2b636b66 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:940d1bb5f0e297707ffde66215c2f7876102882b4d3a329e3b314946532e3aab +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1275d4e91ac706742095d53fb439ba307e9d4f1b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbd32fa5ec116d9d6458613742b1bf9ae1fa5a077805cbbbb2ba2a38629be5b4 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a920a4cd10e5fa4e3ebaee82eb22e7a2ead72523 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:658854d13fad0c2d14e130de3d1c07d503a922a8d106bcb5a35b36329a59750d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5e7c5259fe04c8908605f8a1874ffa50c5d2dffb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd2d0c74f0c34e81f31f1a7f1217ab9b2f1aa0e3a0e51e8e6d0afbc6ab028e9c +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..db7bbc221d98308a291c3e768435124939400883 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48af1cf753b2dd13751a85dc7eec65b447301c71988d9c360888535eee28356b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..111e1395b9fd79d6ea9c12dcff2d5cfed02d4bfd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68fa67dec690976ef753750ed4fc633d286f9cfc3dec32ce523b1862dca6f65a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f68db42d7b7e9073843a3fb042066261bd929eb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1c7feac726835d89ac3a2d4f9a5aab9bf2c2d1be9091d70e459526a2f19a404 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a4d9759e5d1e097040512b50729fab2131921e93 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91ef333fdf91805236981ad39d2cd93edf409b2c98c38c76d49fa1907f925910 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..36d05c4238d6b9af3d835250c0d5eb3c03088dd4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a30dee4b2de103b8e5e3a961a8c14bf7e5554d9f6e28448661e94bf93eb76fd +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9365dde107c9a69889b5fcd1ef2f117d231608c5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:681ede67c10fa8ddc1a76bca53d6821f86e75625126f10da295680d7572fb443 +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..32cb1eb66e3879bd199ac8f7fa3b69b56872c914 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:348208c46e7836c77687911b258c8eb8133c8398eb04bc443f044486b91dab67 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0734a2e8821c41ff4cb3991efbea5b4ffaff2451 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab83c5258c8b6c60606f5df876f8290bd4d30baf683e6c318134a3ec44eea79a +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e83f61eaa8eba2cd692289f10287a96365d802eb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:699fbd797c559629cc12b30c0273c3d8fd7432a58e7580b0868fbee43fa01cfc +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c2ef5f62d473fb087046a7f54d50e3f4d8a6959 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57c476366c2a122444143b2c9c07c524de94aefb4e75675608b8d55048c9212c +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d7dfd1417d7b1cf7a3ab82c3af674672db0a4224 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:577cdd3ec3508a6557582d95b5036ce38deb2e5383008cd8853867bb51348e96 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b105c1c99182a28f87e1bf2708c66fc93407b131 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:336191a6a4c8635ea9d1216bdb8482e3e0aba991053013610833e3a71221ef6e +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e0d72b7bf9b0d38c3b67717e29e28e43f5d1d794 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fb28dce949a5698f0614116cffa6e7caf4be86c33842481e5083154778dbce2 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d2976a35d44fdfe6a43affe78acf386e374ea857 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:513831a0d3ebf8de8e1ab6c9afd569ea324a14843bb4109c176a51ccf50b061c +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aea1ab6b9bf34f411fdbd0784d037a999b6d55a1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ede180afe06564f57f699ddd619951d8edf3e03cce48a746ca48eb8f6893e7fd +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5565e9f3a46bc2517d26cc59b8a562654f15ff29 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f81f9a7198dd3689fbb6589ebc19a775a1a4e24b6b6532ff4da6f3615fd9e23 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fd761e14e0de8331f8c71e546d5687b01a6f54a2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c04cb77cd5e54e22b8ef5583ed343fcb8793dd9d8028fbc03f8c0a910ab7d3e +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6fa852ba80e13f06ecc04fda26e7f5dd225f212 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec9ab79dda49868e2c3b6125cb964965f7f9ce2542a09d0c73d8808a82c34596 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..da4fd83bf7c5c3124031d890d07ea8783190efde --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03feb5279b570041524384a30ea6bdc326a851c609b898d26978da71ebb1bb85 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a012a929e3f10dcfb131d31f25402e23cd0d9d2f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cc21f64191f5991253d8f50e0484821d49114b79e1b0393294fb83427e17966 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2565631958bc1323d7d4bfc8000c9f6128212048 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c6732f402fe5ac9fc52a04ddc41fbd49a6eebbac8b8a49c73a3c2f47f6206c7 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..39f7e162ad0293918c2724f25129ba421098b479 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7e27c9e5013acb0a4c398a6eb171f948c1ca53bbc71ffd20d950b71896057bd +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0beff4a55b9aefd230565a0584fd677001708fe1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c71c8282ec91a36b38d1c347905f8f5700031192b5dddb950a91045d54db28d +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6949eae6ebafc6888d248bfcfad125210d780b92 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e664a3269e2de690085e7f6fb02fd765fdb425e27f4048d2f14064e554a156c +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e030ee9d0c8528f45950ac1c8b9b565ef1068db --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cb89157f9d77c163603b4d771734f0110977c231d04de716184782c1babcc2b +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..96171ac50c07384bc68dea8ddf5e6ca32de33130 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be224d15c09c40437c2a5d53fa832f2934c2713ed1d03cea8671ab6eb1474c06 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..82eac57e1e0f928e20517285db9fc90ce59b6f2e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f5293a14ab78bc38d8abad81cc80dffe67ef3800d8167f808bb4b54b991a314 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c9d6f6f0448732298fc524aa2bf1d974cfc4974d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e61d7ae7f60eed6ee6be5679cbadca42456a3078622480ae7dc73523783b7f8 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6d661ee893d645fab50de5e3f7ea76852339a1fa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1533b6b3e82f8ead294c2a7b0ce48f51a97309aaba1163cc8d34777e75456c2 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4c19f16f7760a2957c331f70e7c0dd99659ae9ec --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24eba5b0b27db50762c2c24f3d66a4e95bb0e557a9f63c73001a90c68e9850ff +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..45300dc125c4d9dcfb27b799ffede4d7f7fc09b8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16443bd3b6729ec842b327567f70784d9b31d0cbf3782d84c20cbf37a28d46b9 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5778322923516796cf4eeed678c5ae23ff11f14d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cf767137c73c3d8f98bbcc465e2d5ec44089b9f55789f88574af75e9b108845 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..133ae974cbad033d632f4801b1d09fef4ebd0f6a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d084b3a1b354cb91ff7686dfd670aa45f04967807c91a5d68dc0ec212e799fd3 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0476c99e04af066177a6c75c0b09434b23b721e0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63c44dbeb903c162293c830cb7726e16430b87394f301211e872f4e58cc384c3 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6eb33cf75d27732db56c577e632ed4887afcccf6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbc1160e50dae95455be2d0c96cc65b8c3a1d83dbc7801ce81552848101af27a +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..24129ca3a2c2a5971030f2eca1e95b377e65b138 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cda4fcee66984656672ae07b62b85d680093b58bd6f411fc5e4238124580867f +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1b01e2dae817170be561ba47264ed50593a25137 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcd034c799514f42ab24a40a51da89e4e74ac7e48fe8c9516e65cc59b10faff3 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..29be1c94d2675a7db2c9bdb1302f9d14fb6e52f4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1644c7f1e7214acdc769760b6626dfcded947bb4c8304be3639c819798b58ff6 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bca6a2132eff40e93291b7257f3348b29c74512b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c166daafae36dbe0ab4005e787a0c67f019b095b05d19c0bc9231bd9e608eeb9 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..864895fa2db06a8bc564afebf32d41376b332e17 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cbe216a6bb7c287395a7257e1ebfa95ac3f204b96e60ff032d341e114cbdc85 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b48159073893ceacdfa14ff539a37b503856a592 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7247b085cd5ba95245092dd1845c5cbd10f90e7c07f7f96ac3ecf55bff033456 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..973c97ed7f9dd08ffb0f137297dd96011ae96b85 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eca8492d7ca4bed46cc9bda657f45756256d60a9fab7196ffcd2e9da8d4db897 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..716a8b172ee94daaba0f79e62a724b4441316cf7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8912e9c9e9efce53905e99de0253581a94fa0bf25a41719c242833785a8e7086 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..91fa764b89b1148c6ed1e4b9d4df5c7bc2a72192 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2210329c9525a48c0afaaec687f15d9f7492317bcf6c34e252786da3e87fad95 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fba4480c5cd74a9f7e4194e9e7c526da439b1a19 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56e1bee6b930165283abe4f5b516e25a1b0e4e4d4b73ad677f076a76accfac87 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..43fc500c844f0bb43f1ab36485e6a7d9c6d06a42 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:529c0c237a089b6d21584379e0d37e0dbbb9430648b287ced5dbe586e86322e5 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9050690c781b15ba1b311cde26ae977ee6981223 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bc4b6c4c10dba21a8596d7b190201ec609bf87b55c82e157c15698a1ee2e3d3 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b8ebc80185c83f66ef1bf85b75c9b3cdbff88466 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6ef94492d582f5b72028ba89a796afd9859345a3713358c11d79a829862e8e8 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cb1e9135e1d9a032b25ab3f9e428878a4225f580 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75c4e196117e46d77e4e6b1332a93369973d8b525c9b90c7f25c8bb63f93264a +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6290de430043294cf3cfed873687fa41eb32475 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52dc642d31aa643774686233115eb817629b2b07eea23b58ee8ac83c64051ce1 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..12812f456b1272200887c446351daac333d44499 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:825dd3ee8d6b0afee9f45e825e4668cc89db01065f9abc3577f7d24b962313fb +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3838bc1dacc35916c947b46e6d3c5814f2d771a0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d9d0faea3a9f38ca6649e04d3e5b5d87dabc1d7b6475f5c889f17f1467317ac +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..01b5a6d65f883c5615afe83a0580cb4b068bf4dc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a1807015bc30802458194e378d0974bfd55f123c8620f25c602f78345ff4c77 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..99f2083d957855daaac4e2ab12746f7e3592a4f0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f2ad5d2b063bc4c6b5019e987e3f6e918678cf98256f626f7d4ef6b7711a551 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..827fde010af45d707cdbeb7b254cc7011a5137c0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4170b0e67ef6f0a2b1659893c60639777d7999f2dcd0b6bf0ce1891f6800a13 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5129c548ec134ac3c3b2ce5aec15082a528ade1b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:634c12f33512388303851017669c016fcb5f9127acc1fe94af6d3b10f25ebdb5 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..699f36647dfce78ea56049430557d0f1eff94cf5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55f311be2c878885b21d45e9df9f4633f83244644fe0045b1d891f2888d94471 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ac09d3ed2c74fb3df1c49802206f144daac2ef9a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe4818908de22ecba3374e98f747a19a0514c4d64a425118fe65f708fb39938c +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a5d1f72aa994509277103111b7a30ba6341999c5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd8c89c46a9c630fc44830e47c5d07d70862c82f376c28249dc550061c07bd89 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..384cd362b15f73a463ba7e0538df8d2992633ddc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd04e016a285e14a4cfff0493eff27c96c7f85edff5304cf51a201e03e48b97f +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b527f15735a4191a6114b51d5988a55c8bfefd8e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e19ba0af491d7db6ea088af8e72bd7a6ecbf4de4cc9252eb4814f3ddd3dd313 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..12cfbb0b6339603155bc718c823b22a665cf85da --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56fa0eef1dd51e41d84fc727268416acc6395087effd9860fb7e6838fb5ab070 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..07752cc9975d473df9a746b939ddfe5e63f4ce15 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5220f760ce0a5267a90fb1ff8181e199e85a34e721a5a1459a40546959eaac60 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ea9b4a9197d2d4fcdd3c2090c9b62b90dd946256 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8eccc3012c04bdafaab0d7c8a01e0d832725435fd525dac75db43393ed06076 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..72c1a2b691395ea0f0a5e63f2c98ef34658a93b6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93b8586933587b62aef29228e8525df9021430cbcce964ec17c06f647029599f +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ed33e35d416748f8ab64051659bc93e25d03bfd7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c04be099b68a5d027c3058fc2331621e378856a320ff09d4240bc31aa6839972 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..81cbac9ad7062d395a10f7c2b5180b422fef4691 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91809bcc8da49bf1bf78610b688870c66585e7ff53b1ab040512bafb1943f68e +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c683d17718f222a443eb0ac0b9c2c2868d6cbbf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d56976524a1f26f20a5b65e0451fb7ae13c7f3d2c31a806f601d845c4bf36dd3 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..528c6090e2a14c0ef90ebb30f0f569ad8f1cc5e7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd57f3bb1848f9a8ea1fed72259325443d6e4fba01d80af6b8a8a2dfe4e7ef13 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0ef3a34231a6e34cb88421bcc0907facd20e41ff --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:009f5a17acee0b1fad84447d4593435101de053e503d9b0833087884d14a6b52 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b2eeaabd1f00eba4512b2ce6eb476bfc86be3f4e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb0fc4ad0f69c51feb2caafe18e764a34ca27e37b2489b4b7e4aadf645631ec6 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c0f5a9d77ce5f0b43cce3b6bf346227ef7036230 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a93c21f1887f24359f094f7d4142bba7795d4d54d7f25f87b3b3656e46b3c4d +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..12f8fe83de2aeb00f10da1b88a9cf1b9cfaa1d8f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:798ed2eaff2e6f224cb725b4a33b5a2f4e33bf4ab7981ac1c2ea69d86d9a7c80 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c81a6f28b9b93f7df71da6adf951abb8d5c5fdd0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed4e095a89063671bdff9ee353e66776d98d66cb62414e2c119cb5f4ab8726e1 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..392b7c6e63483a4eb9dc1bf6b6e10ce875070b76 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0724bbeb1b0c69d8dc340c7d28eef1ae0a00e4ec9f23551893d9e599df05dbc +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1fee2289af15e55da46f631287c786af4e011827 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed739e6efde5233eb2a0fa4bf0ae379af02793e9562e8d166d35d1e60a92827b +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..83b4df6ecb58ed5d7bc44e43816f3ec5d083ce1d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22a891b3b59a953108a24104f020d76d6182cb5e7f1bfa016d2347a96ee386aa +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..47d8a368fcc3c67c6dede32c81edb272d82d2953 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:368be71199584abb2ba4e0eb99234eff0afe2cc7dd016d64007e56751a7105b5 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..143da6afbf5e483d5ed9611b34e25c02a2fc45d2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a80e7f921bab49f1b127df6dd9ec13d47da59e79ecac9aefb7584522bec4de3d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2fcbe9d187660b360254de3272d2c6516d16c32 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12325e7556c94c22c32208dc47c2f50e9b54211e55634818eefefe92385c28ab +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8927bcda223aa42f8b02f7d8a5438b08ed132386 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ae45f8f40de06b1dd6edb4ccff87bfda6e59fd63d539f31ef9bd9946000fafd +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0b92daf51fe167432f0aad3fae7dcd9f417dfd39 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b2ba9e1b26841443cb7bf6fd79cc3b60a013c8bfa942670d123ac2ab47347db +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8e36fa7c06557123546f65746e7a41f7f4cab34b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38187d0251df0b1e376c92687b7768c15d062ea7774a26a0ddca151d13bc6e89 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f50d16e676ad12b8eff1835b827569f606b876c6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0621c313f1cf04910555dc850f1d530cb5c239273ee6ab24d2e24e970937ef8f +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9bcc9dd9eedb689e93cb3da58db744703d8b142b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cadbe46920471caaa9fa8215b3c0072a7510abe18f8750cdbaf954af1bf15f57 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a5ac9558ac29949aa1066ce0d22d87e614b1ff1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:025fba96763e6be021f1d9d0da506cdcd68a25a603075b59b070145605adff8a +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0ff6231ecdae1d5511ea9f32b5b593e8e73e2a82 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f55b2b28e4dc7c2846396c2d08e195d2bec76ffad0102d4e355277a23ac8cbac +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2a77cedbd2aa828574014a1cd950e1b78791b64d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7834dd5bc41751a06a2e77123740d9ba10a226e7302de807c31ce8230b74579 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..03af5b28f23fdbea02fe7767f76cb786619aa47e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30ffc82e2c4a0761dcaa8c319e25579dfe685e1170aeaba8dd31910b66482eae +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b70659d30abc82622ed734796720254f3627be9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cb3eeac36201e7157480d10188ce22cf662cb2a4f0ca8ab9c4b77b26c0370ea +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0182dd501e06e204cd0649ed6259de66f074693b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:560d1817d3ac3d0005e4d1095be1dd5310682119dd78a9dd214299cc5a1987f6 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0e7aaba1d97d511806f76c6152a664b50a2412de --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b211825a3ec2a0038825c47f515caa3da4db5d17a54736e80dd35e22b69d807 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1db454c02b15122716efb766a844629edb84f64b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b89507a58e49bcbd02065b72a5ce13c43f77161c0291cc7db885144e0acbfd22 +size 167872599 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..405f8534ffd13a1a377c2174a338172f1417a1bd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23cd40d1125de482a7fb5f660ccee6d2c15c17f796f94fd34d0cb96c84228ac1 +size 167872599 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d2f6bb91669945f9b94d148a6c08c9407ee1365 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce486d85eb6a0f196dc358b3a6ff3013cecc12fc9e33f6b893c49d89d16a53d6 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c030b367624c13753c51e87ace786e4797af8249 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17c1d22bba89b8447c8dc9df2d0698e4184e9780d1137888434fc24f77986c43 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..36ee6e19a064ece0e075c74ea8162ad36953bfce --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c544f77f2a784d55e252a46013dadb74a2db6c1eea372b8dca62fe40bcbeb10 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a063e348e8773e4c18f7b2c4221208b7baa285cf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8accea290b912be43561996587a166760cbc5035adb9d89b6aecc8fba432e384 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1b50ec38b4a85e1919b12ac9ddde9a16d46581d0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6dfa17aa2b92e60368ee4eaabde64d886e7c2c2ff8e32783f3dd429f8669b3b0 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e28a6a0dc0da87c585c628f2e556eb9d44257f23 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f056d2f75455b0c2dee0a0a6bd62253a283c2a70889a7cac67b0b2a9316844ac +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..323c91461d665bf825b6f13348d246a86d7ba9cf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc3b9be04f9a9aa669c78d1b0689ea6dfa36104a9e43b0920a6e10e016010dc9 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fcc03015454bcf53b758e7d3ee642ed0406f9d0b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d13c83a590099e608ee5eb6347d81c626b7e087d1054e0794c8a9a2cc3ec8d60 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9ec8ed9e4acd26ae2f742d9ebebb25a6c6d177ca --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9739a12d258d59350d1bbda427b0e5a5e770ce85494f697778248b3c75555cc +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d88fa7ace9f1f37c256c45c5e37139ae1972ded7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6facd960a085ae68e27e1f888d42e0ba571c2d12c0d12ee0d6783c4a8a25262b +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c5db4f7866282780c7d6b546e3640f59d1310583 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ed66b2f8df603a95d73f6be7ecebc116c12ec5c9eba1a4f7b00adb4197d16b3 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eeee6de31c2e5d84c5ed7a794147341322560bdb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd4a1c5461568389476448141ff25fa0cf7594f900964d5215c669db0620e21e +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..533387a732580ae9682a7cdf296b6d1b4caf6030 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8609cbbf1861620e124f00e76cbca9202e7170a4aabafa5e66e0866f1de03b0 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b460095734aab9c3caa307a303a16f35df416e1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:545fa80fc8b2b39208c0ffa52d0018bd3d4d42671c080f3b954e2af9af3bb993 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..948c452a240d4516d28f49b4fc60bcd4097772d1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:912f2ea0f2e53b02eefeb54ec7fe1bac8f9e3156ad01c138e211a3e665e524c2 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..83fd1cbc5bd95e910a9e60bdec7bb430da8aa41c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc65a3f3ad5ab07e55f54496586038d5cc166ebd5b64ca5d56d212b19eb860ec +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c5cd856573d59f8c1e3b9d592aa17107fa06842a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da0e07b08a478963f66b7a35ecacc41cea765c6df2e8184ad08e6b676007d71b +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6caf1b8a9275d12132dc9bc38a849ddfe5e0353a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a032a91c895e95ce7a309560c59fbac370b694224fb8e90de76e6d162fcf4dbb +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4076cf4c51c2a14f00effb5d3b897fd09d71e8f5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0a2544598414c270c56cda81f074d94d448d23ef502046184e0db6dff1fc4f0 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f455b77063e5496414d9cedf1ed1ae88993f0a3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7dc7c1646df418b079bca6a67e6830b13ebe086f48b2dbcfe8e81196f0c58ca +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..097ddb9c539623cf38d8ecc6dce48b570e9af1d5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5abfe3670595f0be224dea93a8716508ff2f2b39c650eaaca949061659e7aa03 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bc19dd9593fa028470edbd1dae092d915379e88a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd419b82e25ef09deae5893737687a6fb85ede5b20741f8b46b8d1b5d7944a12 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..50e09129119f00d19cf040eb2eadc0afcc02a619 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f02cfe193b4782463b1a9e9a51956f5253b81a58a848e14bdc3d0bc3a98dfaa +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..be228cd5de574c8f1368c1c62e2a8ea5f9fd04be --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ce17b785d6325df5c2312b22f03646cce8da5254056009be6c980271a791ad5 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f1bbc0640ebb571116c4342684e638c193c17edd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eca91ae4ba91971e9999256e83ff436c98793eaf3545fac043f8cfe0ee02a55d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..68b56b9a5d38079b83298f996255a923dbe97950 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58ee3fb25e7cd4decc1db4e04fe832a3cd85dbdbd3f30270e48a1506f39de3d0 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e557d2306040a4b48f2eb0cb069f8b8c0ee5d6e5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:611e59e5b1b42a04f5ae5421e405c1bb2bf741a6a8d9c89bf752a0c471cf7d55 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..41ab793b12af2955ec57a235efb9bdd55ef72a56 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fefc78c44e011e0876c03cd6b40083399a7a69b6081927e153eac5288cbb1810 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eeaf2ea60ae8a0c4ea79b271ff8600edc34c884c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b31ca5cb69a0fe268b82ccf6cbbb8ff0db00e09723eb74e41df2f49942a1357f +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..583ebf30d3a0080ef1bdd6ec7bd4fc9d8d090cb6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9790af7958b5b40e1b62bed6521eb58d2cfdf24e6a3c41e6b9323f11cb41f2bc +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3e158dcfc70072e2d6d393f4f500ea47cb4bdc4c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0b7281721069ea3799597a57f7e7c14a17bcfe268e813fb05247f4ad3267f29 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..35de1aee102a6876ad85050b5e3501914f7642ce --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c8175841f684b669be6a948d51814066d3c72d1f0e706836bd39e6981ab1845 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..130713a1fb406561b8c8865d533bfac5543d64b0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d92bd1be724bd99c92ea9bf97fdffcee3b7f5ba215b1bd81d1147ccd4904c98b +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..94c529ddb64fa951ff6794e51f5209e84cb86f7e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9856c9641b1e8b7af1db9c7e6dc2162c155e75373ec641163de4cb68e9a6d75 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ddedad2410b2f9f9dc89500bb7ce292736134b02 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0adfdd1e25e8634fc06ecae934bd79e7f27653ee4b6e4cb463ee33adaafe48d +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2b8c3de72b31baebc5ca962bfcdc4090f4a1f88a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e166dca8111a3cca05b7153e8f6d29e718564de16bdd174169b939f4161c5a87 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1a89a25322d3fbca2555ee8818a32a04a115e69e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63c87c61a97305c3e18a63dde5d90774cd8be72707487474e3ad9cb8e4e9b309 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a47dac75ccaa7c0a45514190b1b2a5f263f66ef9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3ee5241de96304305497ec215e7e152ed4ed9fe2d31a3c6640b3bc1b3c008d7 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..feaff88dcf5259fa5aa140ff505bec16486950b9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc1e62a1b9f92d8c11a10988b3385ef8c29fc5c54a759b6dc28257d6b7af9e5a +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8fba87e1ef088872110cebcb31af0c19d9deb5cf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a30ca7dc54fb1430d0767fbae91c5977bbb639b17050efb25a1006e54ed470d6 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e952cc79f68abb605b54f0b729fd7714dfef9e35 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c7b7eed48decf85e6894ca5551ffe2a3610f9cac3f1311d8586156fe4395d4e +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..52ad57de1befdb5b2fb322343137b4a820a3385b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:348d3424cfcd226d2beef6458ab49ae9ef310bd2584710eca71aefa3695290a1 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..99efced2c3d1317919364e32bea84fa182323428 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc903c98b8b47a2aebf15733241cbf4ba69fc896860a4e982942f147d3192b07 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..420de202113a928a0dd099676c4c5fb02c1afd16 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6806312eff315e3212b4e203f0e487254b91e89d4eb3d5b0aaf43701cb20cf0 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..42d09493189ca1fd5e320fa57f15f27f621db6e9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56e062f4f6713811a8d17488a0c1650b610c0d7aeed68d2c2799955eab9b8315 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..11817ba28a7aa0fedfa821098edd73a9053ccb5c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6ba814f0971b37f9b686618112248cee2e9a9bc0e1646a50e44120c10594f33 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b3e35b5797a3a12bf21cea76f881fd157ce09a11 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebc4721be5a10fdd388c05d4008510814d30375ba343052eceac0b85416942a5 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..527e2dacb9f16aa9027707c39aa697fd45aef9ab --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10819ca76b4d317df5f58247e62117da3b634391b180ee5dcb1096b25c258d71 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..94f093ed4c0be9cf97ccafd9540ddcfcdcf4df29 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a237ad9371381b28410e9a3f0ba6d05415c63062b13e9755c74e54a6f5de444 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..97baf0defa94961efb38feac0c63b81ecd75ea57 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15419a84e20d2c7e67fe525f910c778a39834b2b1e3c94a14dbc2378dda18de2 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e1c1edb36c736dfbded315c5e2269260d1fe776 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a76a0f1b767d8c740599f74b7db19ba23453441d13d551c5072134fff322257 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bfe39b4030aa831e9955812cc17b9561f9aec2f6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d45e7015fd34739743759b618324aad39b6d86e5d0622060480b2b713cefaad2 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6908cb51bcc5730f9d091824aa46848d1fd3de3a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5d114883b4544bbb7b0f1bbfa11fbda637bf7e15950bb6bc8f44031f822021a +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c8752e24fb385a942a8a820957a357f0f821b2fd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87f7284fb1cb9ce53442e3cd8cd0ce995127251856c6bfa56ef2ead05e6ad025 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..55e5832455dbed98fe9fb2fe3e5ce8485e5a4603 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f4a009f00aef50e552a9fda83e04024f063f8d58e141e04be6cb78f8c5936cf +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d86df732460361a30d38a3f583b756ea3dda4f1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b7f3481732a70b15f4060e0591c8a742351ddc9840074ad246825fca1779a9c +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..847498edff0fa5ad53a78aa222f30184f270979a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b63b85cc0d30c9bf79665aba9c3bc11687d4bcedb2e0ad37c37711f5248fd04 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..147d1eab62e4de2c99188c61556ff262f0a0a997 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d56dc3fa101cd4ab3f4f65337e9869a92f540ace54dc02ad6b9a793e4aecb269 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..18b6e4c2612889d88e999287c9c365777346ef2e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77b29d168c1b682cd5e7b6d67765c443d4b20b0a5bfd0ea5ed203a5942bcefe1 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a90e33c9ce477880d135bce0b8e682e69280438b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82d471f47457c769a9aed223cb508be4e3510d9ee6c62640a2620e5c7e07b52c +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2bc5d7a3776b8abe3cae9cbc2227647ab790a64 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b30aca762d69bcb969636791615fa6cbde07110ea425d0a0e0158b7c77bc0e76 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..30f1c252f0fe273b7cdcebd74863ccb2422c68fb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24aed9afaac8b0fadecf5ba119565cbb32369390b0c28780eca9ab3615673de0 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ca209eebd4dbc90c9af747eaf3d76d1789272b6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0468ab44a96a47e5242c0dcf43ba4cce679fa3d5afa453d3bac8a0e8a5c54515 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ac5b2254d09c1113ed829cc811c315999e4d5949 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f9d9ccbce97dd4222003cede158ec0e5f50b853f2ab1bcb63616a45212d2186 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1d4cced06d8a8b82995c32e4b040a51adefe5904 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:377397beda35d89b082ca26f6cd7f6308dbea6b30c91c0c62f01667586f8b818 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e24c39696986b4b8fd82293d4c1d643f79ed4e7f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02302bd76ef725d32d33e5bf3026dab3b9ca1501438dca20b3ee3c32a1083574 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_01-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..507e802998586edbbc3b57c7b213a67b87a9eed2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9afcf9b967b0c5b08d9f83a10cfcfed69eea143ad362731837d7b42f43b134ca +size 153552131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_01-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_01-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e40fee507739b3dce33826d33a3d93bae0b43a34 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_01-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:631fbc0b38df4752ccc6e93b21ed9d8f3c4b8a9f7eba4f04c4804a2e2c31f766 +size 153552131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_03-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..619a4f3149e34b26fc6c7e3872035c1486ccb3dc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe96183a7541c43f4d18fce24df0357c44c4647cfdc6ef263280dec5769553c7 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_03-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_03-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d1fff6fa5bfd1f415a6f29efb0ad9165aba2cb2c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_03-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14ad6cbbf544612c9dc9eccdf06425a987d335616de75808a37ef00d4ce50cee +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_04-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..207ba0b0fdbc9776838e758ad7acd182ef3bee05 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29c021eee458fc2b1e5e066b05d64081b8b59068b7e3139a45d5e555bfe28d1f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_04-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_04-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd7fb0c8f77a198f5c4fcb4107291a1b09585fea --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_04-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a26196feaec16f11371e3c1a198c12216367b32817e02d9b816d2a2622f611bc +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_05-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..103696780e37b796fa3b1e1edec6e25bc203f549 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef304126b6173dabbe38c22c52c54f645940929d0e1c4c6748d798aa28e19a14 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_05-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_05-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4b2dfd9e994cbfb11fe72205521ea284235871c1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_05-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1eaeadcaf4903cf253bcdb83b84ee020ccb0c48fe7a78f14e1facd75e4fc9686 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_06-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f44293ebf8c51bba84c0b9e5cdb4a171b6e4e81 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e195ddff2197e3a034adf29d72318b791669396dd734d75e187862978b84ebaa +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_06-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_06-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3e63833bba22fcea4eb6a96bcdc3b7a76144aa58 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_06-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d6d144049c554cf01dae5fcc35edebd61ee47ed406796773dc4034e59cbddab +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_07-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..de038aad3cb3c7571d7bf561db060590489b40b3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f07c25fa1fd0a09443ba705595a0c71e3c8ef5872918c5a6a62faa1bf024b406 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_07-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_07-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bbc8de49da06d6d723f96ab475ad158635a4d445 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_07-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f48281ae7798424c22067b4d041f6adf9dab5e34da1e9572e1d088c1c93e2602 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_08-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bc179dfe635280ed86e9598eff891a7bce553b72 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10422b065800df5965edbe4ea5bd5ac53669bbc3ce08cef41fd1c49a0601bd1e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_08-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_08-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..188e54c14ae7e76a63c88d3e4db6270698daf7bc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_08-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0748e8be586d2dd7e39533a2dab8f47d9ba8fddcd3a0cd4e63e360a103d85f2c +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_09-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..31b50234f0ac33bc9a5ea27bc02e1fc31a6c439d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d3280f9f3b2fe504b47ea794dba8a474f15d297b34ed1976345a48ed5d96b0f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_09-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_09-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..00ccea7504c1894c8012821c4c255a7699bd584e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_09-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cab40f25b63bc848cd707da71d27f23085171b549081b4298a7541a5bd8703a +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_10-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..de1403079e89ce228d56344844265db534daedc3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:238bc548cd0352ff524297d8519a86cfe17d7a67926c4432800f2bd919a6b0a4 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_10-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_10-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0e7b4fad76cbaa7d3f9be149b99d725e86af6d6d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_10-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db107739e16127b0b3b2633aa97e7ce5b38e8e6981ed5eab3ac62cf86edc1c62 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_11-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..21e69d9744934642d3831469590fad4ea924ce74 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0578888f54c58be2e165dd93c18736e2bf5517106f117667accaf2dc923bdff4 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_11-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_11-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a7b9f136a68c89208a91c26bcfabe28728ade075 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_11-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37c060ec67ebac5ae2bef362e0c7f086ba89e78d99f6133f0fe02dfc7ef618df +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_12-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd8e0566a39ea7065bef60e1db0b934f01ec9a9c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:408049fe9c953853c193e4d5ba57a9ea9519629e8f8c0fcf97af03505185a699 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_12-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_12-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a590c86152ff69af024dad0a172a4c77481e8259 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_12-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da5e0025d43085296e3dcbc6f0b118e7b55b6eb55695468b93df9e9eec4b300e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_13-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f4390fc93b4d1f6697b2895422ec4bb36751818 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f658e4b6013f1bdd8e51b25b28baefb105821724676af9a978e0bbfd5e529192 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_13-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_13-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c7c8332280479d85419efaa56d80721c69e231f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_13-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bdbed9f27e8a0d0c858903ec3c2a195854b37a55424790c8dfd10dfc40946a2 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_14-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..317c69d3a5dbc234cc3414f069f098bd00236148 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2546b3c0e5838661e79ef16c8ef1b96bd80d24a9e9bab12a1a38655957661fb +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_14-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_14-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..18c622b953516f632285788bfef5b577fe0e9d8c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_14-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9e19f1188cfc24ca555994e4e7b1607afd366b8fc06094e54f651b9b8b63a20 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_15-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..78208a6430142c1ca2cb7acbd146bd1b1574d2a2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2382552680e6015871aa347d025371143cc28cc7f6d73aaa36b76f8e59f0d005 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_15-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_15-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..27d7f4d89b87c99e251c1f4b269420b06d5163dd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_15-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe4c4db6de5bd0ecea5b45249d789163c4f43e9c311bd001d6af40221e93c04d +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_16-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..171864c1923e62ffa90a0964680bfacf06b2fce9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d884df1a23d42dab701c24d704483083cb1b7ffd4243d52cff6bafba720956fc +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_16-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_16-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3d4f97cfea30153bfa02cece1aeb9757cd9800ac --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_16-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec6c102506236d984bbdbd10d2f26e46d8c875da3fae915971cfd61c0287b752 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_17-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5fcfc3c69699c70a6d960ae237f1234986010e6b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0decd3272718686cf6766081253e0aaea903cf174f7a18983f84d1ab8d0ad8b0 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_17-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_17-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ee2f2914283b3b8c75c924447a4e0d59cfb99f0c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_17-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a54a935f82ea363535fe1d931c8c8b849e1565e38ca35a339e552f4cf0d36c7 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_18-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..58d5bc7e5095923671e9baad58de1d12089a29a3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bc35645741b848a993b21a089c0cab35bb03675b73fe516b6697db75bb866cc +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_18-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_18-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0c73fc4d1c2dbc1cd0ae86b6bdc194c30a8c3b18 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_18-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50c60681834589483b0c4631cd5bf6324c934f3ad6a833b61d875dc9753aa441 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_19-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6112feba3c0b5fcadccd36ed29d75751866946cd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:996e3dfe773d623e2be3eb07e84a9ec092ed8e5a3f22cb00682e6c92dd01ed62 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_19-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_19-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6e051a937b8a8c765ea3fedcca2bba461b39359 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_19-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77f2b04065eb9642914ecfbcce252d9261e6c4a9dbaab6b1e42e10855a6aeb69 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_20-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..42b8fb5787654e36978bfd8904ec5cf3f00ab8c2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0949d40e9609fd6d8fd98a29e72b64ee7e53340a5fd1005b99b2bc2937d88b8 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_20-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_20-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..004629ee3623db3b63a3ec43386d052a7c01d0a4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_20-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f7ddddf9230ceee1c176420cd31bb38c11d83ba8fceb7101873a50c3f4ec6f1 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_21-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e33f56353d98f845e27321e2bfa26bbe4993767a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3f3ab1e77f55ba026c26d8e61c18c6d3b785f0194400db72cebd650183ebd60 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_21-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_21-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..77c63506bedcc06fe8a9cc23730c8c3bc5384ce4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_21-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfeed8293acfc5297928400ee26c6da063b88c9bf692668cdd511dc23c059bf2 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_22-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_22-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..792ab3b94ecc6a1926edde3bb4a63cacb5c707af --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_22-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95765795e1754c713a50ac6435439ce25ed1a43d9694ddf5b8ca41fe400c374d +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_22-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_22-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..231002efe8e94e3d767f759bd4cc52e310cbed35 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_22-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e0da3bac152769e300289d923a0195922c1fbca6a0d749f679d9e95693b164d +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_23-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ea6ad5ddeb11722937b77048daf53f5812ed3d6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e371fe4bafc08de574c6fae53f84df399303af141d563d1702b655729f939f16 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_23-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_23-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5e29b2fc0cae89fc757eecc22ce8234b193061ca --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_23-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fe4af7b3906f81e383f53465343a33adc077af37bf8b9311d1479e9c48a0b67 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_24-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_24-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1ba11c83ec799247f1ea16854d374890813f407a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_24-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a2038e7f64586d5d9899c2c0b4ca6c973ef6de932b6d813119c4f4be1f25e3c +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_24-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_24-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9fd6adfbdc74c1c5087fa15c49a5a002880b1d04 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_24-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b53d784e27de04badfe14e06e1d4d902e4e6b78b9d0295533c45a7ba564906b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_25-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_25-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6155220977cb8ed2bd1ff6ec3d838f9baf5da10 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_25-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48d8978bb4b9d6b7dc658114fbd6f000e0a3882ce5885bbba32e62d777baf30f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_25-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_25-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8e7df8a542c4f9434c0de724c4002ade2d2d1530 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_25-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f25c4188898fa9f8bf5a5bdd4ba298d2bfdf8ae60fc2e2643aa95df1c54ef9ac +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_26-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_26-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b048974dc341300df75c5b521c548cac72b1e041 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_26-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:616c355d292eaac465276024f8f1be48dc728e47d2c1a60179d81f6392559ae6 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_26-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_26-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ed1e1dd5ae175c7f5be0e2a3b537a02488e1b0c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_26-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7d90913d2bb673923be623e7998ca281e9690c3e661b0bcb8a1b984a79a5abc +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_27-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_27-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..437d5388cb774414329907a55cf6a3de90b613b6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_27-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:248b3e28b6f06e4eeec9f1e065a6a53f3e5eb22bc85048d83d609973cc011e15 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_27-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_27-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ab61973e9010d8807db9db02408fe057c37317a7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_27-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fabfea23e89b04e840cefaf43a16ca50de186fc3de22a278a3053c25d19f163 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_28-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_28-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7e5a1fa3c120bfaeeef64b774627c21b22338708 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_28-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e21ea8aeb3332be92f4b19d0fabdc39a07d84947a445b75bca0aecaa219d8ea4 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_28-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_28-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6e19a6f439254136dc42f9246ece1a5cbac3d78 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_28-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e824c67b3ce7c7f2aaebe6f368015d82db783bf0c819b1258150d6ae5a18f87c +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_29-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_29-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b92615086fd94193f6c41534bda17872422ad840 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_29-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cd733d895fa142de37e738b918c69ff9b3bf8ca78a7800694d56c3c41519403 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_29-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_29-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d13c92791c516ceb3016d943dad8e02cf37efce7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_29-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbd31bf2ea9d52a178f985fa68afae2d6727aedb39b14b3c2d0fc79b7cafca57 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_30-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_30-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..229d0cd488fa517f427ecf97b884572689c89170 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_30-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:255714010298e2c659b3bd72259a74029f3b6a38a23c9ac2d8a6269464d45cf6 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_30-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_30-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d43efbf2c8c77f5c7ef5bbfe2b68256e2e12fb09 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_30-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5077d008e0f9459291131a33f430429e78ec319604e66716950699403dc9b689 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_31-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_31-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..456447199cd2f04a1d9a2498e6d3f72c61ee4063 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_31-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2a65190773399d99c4c299b3ded16189f016cbb949c48686aa7c8e33cdcd038 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_31-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_31-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b32506b862a973c391d05f1890ccadcaab4f3bff --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_31-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9cb36dc21427df75d957ad7b6b2ecb0c29a9b73c53a846e86a5accc2af2fe9f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_32-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_32-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..517e382f3431d8ccf351892470a77370749dcd44 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_32-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74f12378b26b9edda4b0a24ad9ba31670b2c502c17bfe30f3a6ef423c4e112ca +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_32-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_32-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bac97dce894ab820d8a73e4d971c10f2b120e896 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_32-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90c2ec9430ca7b497bb4d2187b112e1b0c98317b26504cbf2904377341080466 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_33-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_33-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..721d46ab03d1632ebc2efe655f001d6edf21f2d6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_33-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5789de07c41d8a97073707a965f377677d44bd87b61015476ca5b71cd3fd12b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_33-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_33-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e80f53abcf3fa7a30e5a555925a77bb0c4384ba1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_33-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5852207f0c42a2a4eff0500d5b45899e042e0e3879b32ed47ee29feea2e456d +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_34-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_34-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..85a26d25c867601cef81f042b6d9f7325109f18f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_34-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12739c0df3fd376ff05bed69cdc0a138e138d6bc7fff90206a939786f2f2efe0 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_34-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_34-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..be68dd7b1cecf340f8daeec36ee52116bc89d7ad --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_34-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35ada055372acfd35fcad7e798148b778c2b247df61cd35f0a83911c1c3f85c7 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_35-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_35-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f5f5bd3ca4ae0f9bef83f65159252d7371be3ad8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_35-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dde76267d9068ebf0f82ce39519e9ead278f97e214c629fc2eaba3376a82784 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_35-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_35-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9697dda747faad5aa4c9fa72048aeb466b09e3db --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_35-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:099b0a10887123eceaca3d086b65cab92b312a924a117041192975bd9006435b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_36-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_36-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc476eeedb8836beb5f65ef76a24fe3fe69a820f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_36-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46fd5cc15a2d72c6d6586e33d1bc26844e3c1dc9dcc6fd73bf4f415172b8e0c9 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_36-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_36-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad017adc70b35e924d79647eb9fee80470fa16de --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_36-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1113caa3419168ca9d64603b2805107f13504b5e17605944e0037a4277e77e53 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_37-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_37-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e365644b7452856d7902cd423894938035913f87 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_37-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b987a09bdbf23f3194635248be6416fd351c2c40473f0c6ed64a3277493df907 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_37-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_37-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9e22785340997c9076b433f67b4ce66b59ef19e3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_37-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8370bafb8ff25566c384c846a5a14acbf4e0676d4ccdb53daf49838827981749 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_38-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_38-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3315a78337c1bf0914159278a30359ea4dad88a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_38-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1da8413927807e0e9835578f1eec0927456151314874259b1a59a548cdf32bf0 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_38-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_38-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3b0753e3e0b6e1ef101e5fe76688037a48e7113 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_38-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd39a0a12de2e69a374c5f1da1fb1cc058efb5315250c6b4dd81eec0168e946 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_40-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_40-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1a8281eb3b45d54399ae568f989c67e9fabae6bf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_40-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14f6e90f442c27fd527798ce9235bda64c7f9f5b558cd2cc693d94c8022d15dd +size 12483 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_40-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_40-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e7c8d6dffad3c8fd8bce93e6a769083fa0cc01ee --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/layer_40-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8000929fe75296d64936fc1e06f0c9c68ca9b9add59ecff6ea1c412b79ed4474 +size 12483 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/mp_rank_00_model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7201dbd75d7f7d4dbf2c3086187ea75ca7c440f8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55a941129bb814af627100fb6c008cc474f25c5bdfff2cb832462c5519aadce8 +size 51443 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/mp_rank_01_model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/mp_rank_01_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..17e7f8de2ac159da0725ca2cbf6dfbf6f542c667 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step20000/mp_rank_01_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:317cfc22dac2e7cc7ea6cfdd154b2e7f185452dbac2c187c85a1a740f977bb12 +size 51443 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e9d0990d6964393886ae8b6cad00e73f532687eb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d38ec7ae779751d455f60e2d07b32f7f736779802a05a709e23947dfcf67220 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f6cfee81cc7fb0b0047ec5ddbe194230c8dd4fd8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0a9178db22608b7490e16beb9361ca9f3b32d6b7d31df0cdc664b662c26a0c6 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f873c9f4510ad342a4aa81b5cf36524e9b865cf6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0179a17b1f2083ffd2523a95c0d6a06864cbfce3cd84db40d133dfc568db32b3 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6bb88be1210f5e6719ce2989f336a8753ee50930 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85d7cb7fcf1da495a9ccc633b875d37da0cd6410ea695b82b8811439967de5a8 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..11ec03acf1fa05f3b1fcc7966d399f40a2815d55 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f374af13bcbc3cff06109d1f895d2f7af847bab8242ac7695136167b97920570 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b80a7c1f57d329b56babd35eaf476a0838070118 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8beda6bec5cf11a8b3a3b6cbf35f1170e2d2a7bec3239a240e41cd98f8b9b9e6 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c58e03c12edc629ac4d58eff9bb5771bf05517a1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2951b67a423fc60f2a37f6957646f05d18a9c10832c39b0e9ec56b422a1c43d2 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e7eb84e81f1c13addc5f8a4799ccf163e095789 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08235d5730d32e7f39335e455accd3912e9cd39a7d598596b52ee1a3978296b4 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..df895eb18a8e5c9e1711678dad07d052b56b68a1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b55c109c38ec980c2e6fa65e99e1b640f3ea128e242f0072489cb6d8c8e8791e +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2980833fc23865d8ffc3d47235864f010a71352b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f95b07ba7e9f95a9e10d75ba5ad327fe56db1f0243473cbb9623f37c1f0f5a4 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a93b90b6a3ab59272e6488ba8da7023308b3be11 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f76a999fb2425a4ebd6c17920aed16b6677fdd91e74a488399c791388a66bcd +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ac5af390f5539457d00ca39e4ac6c5058fa69ecf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a3a5dacc71792e53b9fd714dd84b2ea17814f12cb1585d7964c1ab4532087be +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..520a7981643c89f7990a9d7831462c92f70752fb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:170051f038f5d58410e979297996e3a3604bef4638f61e426d558d4ecb446a51 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..61e38b2b5f592be9caa9ab61467948ddd764cde8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56ff3071fd7e9731d2b2bfaf6e6da5f150ed08b2a645f3ae9d5f90bdf24eb17d +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0ba360b7ca5fdc2f2b16b567e60ef4b635ea4bf4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc8268219c8f43f3ade2f806d38adf308ac1d18594aec30d9cc1e427c92a98e5 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d77b413487687bb211f3ed5e6abec224540630b8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae1d666d16a18d23876b50c192ab9eb3a5de9092854ed84b418a068e634fae27 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..47f49fd3962c45ac8a35d4e154091036bf740ad2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6be250d7a766ca73e431cf49b484787fb4696eb6a7278f6317f0aafa01636a4a +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ef894e2ad074abbd449c9d3d562a5b8535a274c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83bfbb2acea036c5912faa6552cd5bd280ec0686b7597bf3bcd3b4820de0803f +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d49569358cb39ee21c23589240e783d13ea7ae62 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f421f0f3451ab2a978bb1598cb20f6f30e26b5460afa76fba193887d879c784 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3cf2d922d317f6db5edc6cc51ae68090fd7f661d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2fa25963fa02ad6f712e52b2ec58b40f7761d7d5db55e85b6861f8ee7c73213 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a897e2a5ca34713ff32bd332307cdb3bb43547d4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:263cfc486cbe3982f073a8a868127252c4f9088a805b3e2c2966b9bfe19eeaef +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..436219c638bc4ad564eb677a496c2758be8d89d6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d624bb970931588b81f36b11b52cfb74ad468ff125ce46c004f643d26c072e8 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0c54cab5e3d492e440456e49a8b193e624e1269c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc97f369ac4507860b9f0a453803b06d213e851ce629cf87b34bf5ed1c9223e9 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1daa593b93e5139e997c1c9ef817c1146d840462 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bfba3e3eb8708e0a92f680ed936cbac3a04692640710296c7911b03b1aa895a +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ffaa8b8d914d8f24522b48bc984e27f394605272 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:854373a708eccc67059680f7697b0f44c1dda98193923f3c384dc70563762170 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f661a4fcf0566b97c1d1c7dcc5356132c2db1364 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79571ff45c995a21ed7e9963110de67ec3010ef1afc1abfea4254a835d8fe01d +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ae2c6990a9220bcf93c2d50167cc39772ece547 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1d5e60200c49ac74914654e0fe97feb510ec2d4c370e47ea1196622fcd120ab +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2419474dc7ed21b94a6d8b1e0312ecad4dd18781 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c33b8a3b13be4adb864751e03d332bc6ff09fccb578ca66c56bb9c39dd08378 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..60de8665949ead1362e3d16c864c77aa2a3ae845 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c0594b5525d110a4d86c5a1dacb2ed6285010e61cbf92d9c594ecec4dbf8cd5 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bf3d4b518b3409d55a4a8c0371532cffc7aef77b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f0d79da5b6c3d57575fa70e056197aae79af3f59750e1ced2b531c1ace40b5c +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ca5b27de5b5495b1b059a3a6b7e514c7e1cf151 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c60ddd6f417efad2247c7cfb6554a53a1aff9036473b201d0bdb9f225859956c +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ab704f730dd61c0e4eadb4e110c84731ef18320f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8779468d7f0d620c200d744ab62bb62befab9b69f7c081cc2531698c6f92eb5c +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f8dba717e191076362b6068734c6748425d5276 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b7a8cb966aeeab32ff4b47969bffafd5782b41df8693955b890abf34f09b119 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..95544c2c6a6da1053a2dad496d4548ae3a4d5e07 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f00ae0213d63e299cfda912937d692afd52857cf86ec104c05af2a1f2dac276 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..81df420a5713973796e7516c4a4afba41cc05ff9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83793da410aa0203afb8e3efc33ada5ceffed4457aa18d02ea0aeaa3096410b0 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3dcbba536f2a89558b0ed9098548cf7ad04fc158 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15f4188a1a1c113f916da0f66e7c412500ca820b7f4d972b54c4bfda9b3e0185 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fdac9983bbbf4ae7ccb5bbe0e9762fcf9fd9d575 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15240e26a68a7254b5e28dace882d3ea5bd983f2a44b05a71ec5af77188b398f +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8abe391759b7d8cf1b67daaf5984c1b1fbcd134e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9005b4d9d463567633176b339fe01ac2c7a69a746a22473b0575389db5a1017d +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d0175735ebc591ab2d97444e1e00c58f1c0d4858 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1027e5aba5297110930d7410de4efe656e664f67c686a77b5e61c25ffbe4ff3 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e8816d83e069c5fd50e90a62753c7831308490a3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:864c1211f850ab528e1a4b89381f4ed5b3ff325b71ebf4fe54a28645db106dcd +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..700aecbd8a97b69297295bd81af47f0e9dfbf70e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09e05fe3d0511f1273fa4ad4dc9aaf1aafa5a8ee5d57ca904a3bcbead4bb29d9 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1a35e5e5b19351f916bbce1be53930042114b22d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0ab5a55a427289e8686a70c620f9e9f4fe1d6db24c183700ddf4226ee60c816 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1ff55525079e926ebf671d5647779b6572599c48 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12019cb846c0e0f41679bd77cb9b5adba76b44faec1633c76457159953f34e26 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..faa14462efbb3f02e4cb2ab13a692217d7897f98 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47152ce92867578f01c4de05dbd1eee6c79be12a3f8ef96d71bae8706286c951 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..69d71175c74457c59c653f8d96d39760dd9a17da --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e60d24fed71f304306fbebd865cd87f5cc724887ae30775c25448f2aef7f468 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..29ce659087ba3e00907025e3d9397d8ae233764a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a462b5b57640d0cf914eb71a2775ba43280a0aaeb0b4d1c4648c483d3f2c8b1 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d3542a8ddd334a33e9ad6757fb553c4a082f7e7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50a3825901d427d4729487b1ac2dcca3e949640be4d93b5f923e437cc9779f87 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f4bf66719d6d6c67ef1746b8c1e4c177d0fbb9de --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2623e6c3c49ed2cdd5baf73ba804aadedd772c0c8dacc2aa49b746c34db6919 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a150d4812b8309f33b0e762966925f8cd770617 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c445db8f35d6cf9cd7dae541c50ca221026a7b302ff04282f7caaa0dc51dcbbc +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..81170c6146610736f59306ef4ffc6e1a080b8830 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5b50c077ac274c6867ba8ed9ccdb45feef189afe0510bc67f8e7c544c2fa8ef +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f16a523149483059948cec8d6d365e8f6e75061e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b2d85bdb0db2b991a8c1325f51848b264a3e82c4130dae7eb0e03567ec6e81c +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d6dc3c69564a8c2c1b2c87002f658bdadf1f805b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3422f574749f0dbafce995ea6be34a00c8f112605c9d0423efb8ab9225589e1 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aac3b7d3002c779cc3a0957b8a09f2a9782a1852 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:904c3f3eb9f43b6354c9030af6f07842806e5e1cc9bf37f5517ab22980a22b56 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f8827a89033a08fcfa8bafded89823c8bf21739 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8a13425b1d1fd4168d8d28bb8e9f6045f9d6bb050490775b5d18280d4bbddd4 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..099e28bcccb6061321074545e485bcdc2805b2e3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56839b4f66c5eb49927074e8280be9c03a039739fc394c99bd6b10260df9dce9 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ae4b93055286a797508a9a50b01d0b84edbe89fa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9112a011550215b8e8d13f619dea36d359ac23b7773c8fe63225686d2d041fe +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..610cd975557164a45363c056ec99c47eacfd510f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdb9d5c6a2fef53568f31f20bb0de3438b0b2c02bffb789d9ce38fc00eca5e34 +size 167872941 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c70fe2e80659a2884c68dba1e4afc1cd2d93ac36 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:747b7e627996d35b08df332dc7bec4c3acdef03281a645246b130b0bee57de4b +size 167872941 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..82eaffc5290d798c92d2aeba18f07958ef76e466 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81486ef1f76d17c26408a678ee393fa332accda6790071f1bd529ef211177f7d +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..48a3d11bcfa3029bcf159f8d7c3c25c13c98da2b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccfe36a3cd2adede0ab17ca5e5ee3b86067497ba499b13c83a402b834f9f694e +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c0f5f245e41f7e7ca849f758dac896bb12fac523 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e43082da75f3de703a66e635fc7a592c25f267729bddabc0cd8c7999b3e53784 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f19b0de92b22f6820cdd47d09f338a822ece8e1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0e13241c4f6d40603a8e74dae337dd5fba8cdb3e1280adf268615c8d7301a41 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1e274d179e1f1e1611700b38efc9a3479e1c0e02 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7caf491d327faffe825eed234948fb02e2483563db78b92a73afefd846770f5 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0e9c3c2a3456781bd7b72efd729528960c3256e9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f65c30a9ea91cd07765d4fd8ef7efdeba5aa0d305bdd75864e82fbef2ba5705 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..32cda91334d363da0c09723eb4a1f16924a8f6e5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd09f55e25b57bd082f1d35735743e9b649d9f1abc173fbcea2d8ecd46b6d9fc +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..47b92d0f469b0e46148a73c281f6fe921dfed786 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f25cd991eeda0f613706bb6bb7323a911df7d727a4c22bf022dbad7d1ac1fe10 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..281e2f54b02cb4912c6aeed3deab560135b6fcbb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fcaa9842b253e13bfb9717416ba861617f21f1194fa1323822e65dff960a415 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e8a7de08b79b023ca4066d02fe50ba46f17deb38 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7de571a519db5a826b47f87b769717b4fd81c203453a9c54aa2addbf915b5ae3 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d0d41c6ed0ad1006b6ce978ad21498c910c08e1d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44c8054db7e66d847dd821ba03c2f784baeba5e5020419da0dbb60bb3b086738 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a5ae4a725b5c50ff681b5711e598758cdf383009 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c83ac7e72bc474fc37e029624d058c28b49c74a216f68815936a2cb2cba5ec09 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..03c5beebacd3260110b18f3a1dff98d840b7d4e7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc4260fc439d1e7b23328c0d032518ca62a48d444c8b091fea540629dfbb87c4 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dce184e88472ef03b339de645dd5d465624f8436 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b68b2830f6a62561728715ad745df063137f7bdecb61107e924e567b9712396 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff823caec347b94b7d156180df2e14dc31267920 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f201c22f1880c86e170e3f323fcb083687fcefdd315d8ab20f081ec12f1d7f87 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..abbf9bc163226c70b25e4ec32c0b33825db25ad3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:280e7c4a4a87b2942a8540cd20b4c0acf256d6f24be65dc065b7ab2c9ad2a91e +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2039e53ad74d10bd5c39e6ab951e289c0875d7e0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7c22a47bf3732ac4b4c8f3939cbbdca5629cc15a79941bf910d5163ca01ba50 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6c373fc2df840c2c8e6b76d0245a2c9dee13a28 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f9046f8f4b9f8bc2f08a78fd0a7998b6ed8b96c3b76eb01c7aca0c118b21a0c +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..425451594b254317a57ac290d8e07984acb20233 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a59a40d1f921dfc0dd941c7882378c3bd97c2ef7a573758a41fa838e6b3a469 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a7404456204c0ad4e80141c0112cae73f3a61132 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74326374c0095e3b4122c5b0091e8c73a8d95ee1637385578e72bd34bb2ffae3 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b8c220682a7fefe5e1d37ff65af0696897234c06 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f5f8ef802d8fce722ec5e1e36464dc1dadc82b37301a6209ac0e3dc226b4e7e +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..69a231229effdd4d6cc04e413bbced832b9f3ee9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9704ca7ea8ed65d7ba2cb4345ab2496f108820d79a929a3037ccc2f805af4400 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3efea745cd45734974a0cbe53846ec07eceadf2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:325724f5f630f8720b8ef5f79d3a215b69286b77fa18094fc3a14ad059d684b9 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..235d4d0961393370c89b68bf84f7235a5a1d7958 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72836bf88b6f1cd9d7226af167af4d79f7c0c4156f573a19aac2b82ac6d6355a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b3adb42ab0e8ab84572921839d01bd37b9e1e7af --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dad7105fbc7c4b9d95322c576bb7e6311bc1007d91a11d309ecc16ec6b61e956 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d1cb67acf271188633a563d6d38e2e1f00074e15 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f98336aaef5bb35da33ca5fd29f5fe76d9c5d5730cd2d3a15e725572860c1ce +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4153220142d099a71ab2325ce05ead5f5f715f26 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06eebbe15f90f32fc96f574293e43e9f11c5a6b2b3df0a91ccb4bbd3bb2bf677 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..33456aea1ac06d88bc39f1703e187e4f5af00c15 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2c99408cc43b10e3ed91701421a0cc8ba41434de85a851c9975d8cabba8cdfb +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..18c9adb372614e9d51b32cf035d909d880c24fb2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:415ed1dd42d64d76b95b38fe713ada01494f62316f6526a1dbe690e35cf9e5f2 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..48f8287cb2fa57f7eac32b1b77938d7dc7ecfa21 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e711253546e082c8369b2821e6911ef44f9c800c59fca64bdcde47ce34c4eb4 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0c7e382a060c82bc525d749aa61dec7ff5de7ff6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b4010ee9aa1ba42f3509f5bff58cbefcdeaaf48f414859772dfc5e10a0b266a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..573f57a81954d56b7b1fe3f3f9928b4d863a2e00 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83b0b98fb2dcead34fd736ac72bcd30400e2ad8b7264fdbe79dc0d1572a3ca80 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f8711cece3e1e8bf4a1010b7a0ceae9a3a8871d5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bb1353e13fce2bca513a7fa7ea0304870f3b48730b1baf0cae3bdb4ee259a75 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..adeef8584d1ea09ddf09896804edb91a3dff7802 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a89628638965d879bb222ed1fe39a3064f6a7b2fc6b9248fb6a96c5eaf4339a8 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e09c42ad1df244ac62e85d91c6b050232e04b88 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ae2c1c7a22c6b1c37d429fddec1fd8d859d87ac7837977a114e4b4fb4954785 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1f0e2ba82418964a7e909e0545415d31690a32f2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bed68755c41be557c6c90c3fd966ecca2016ef6f6c8bb09e5c1305af348c0330 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..085ad211e6bf0a8563d16405e3d5995703b91821 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01aacae7877c59214a3bcbce6dae51199a6dd0d015461a524a2f27027ac1eba2 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9bf6a31fee9a5590f061eca0996c06f299db90f1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c5524ebddc540e27a0d0979ad7227c4becf53315f021abd7e37a889a77b3092 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e0369e393b8a7ae2e5c5fc3e78a9f532ca330b16 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2a97fc14aea82030c84badb808ddd6d1f402b8683b4396998f24a3a38507f8b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2baeb1bc8b77301b24cf09a1d4945d2119733730 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:497c75ce7eba47c97585b7f52b5806776e728fb9214de7623aa6bcc8399b3761 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d862ca3dfd86c7213ad15dbcfc28580f5034ad9c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60d5cabbb0d68c84944342c7baeb55f92b3d26a239188ec4a9a5af5b323a95de +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6aa9b16b09cd8e4b056153e6efdccbf79306fda --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:918a79f2fd860f3e065805cf7ad31d7e0e87810fcfa79947fc789e4919c89999 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..885c395596326c42c5a2e63f85828ac511945b41 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13c9d5b8d7e4884bb743b1dc3e88493fd130a8b011c4256b9f74d14c4a45fe29 +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0c6c0e3daccb148b6146db6b729de9641e07c6cf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecc7900a5f973644f019af70b9cffec73bb43fe09e6efd993487c942bdfe39f0 +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f18a9f6e11bd506cf0a00091c7e53e8d0e3cf4da --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7cbf57a5ce3e285021f4c2393bca42eebdbc16bd686193de94fb2a10ade4665 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b47bd9bf550db1e1bc2fd40d3b94081d968578d5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:030498c3535235912a4dc2f6cc55d6359ba3221a6db8bd48c1fc07c1d546ebfa +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b1aa27cf63e3d7bf060a464b398a0ae8ce5b1ac6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67bfc011f6820f202010a0742985df310e6c2a7901f919d7c893d599398e7cc9 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3d82b7b435c67d5ae6a55e39e54a9b0bf6b96819 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc10768a0999d83b52fc37ff68f9d1bb50d4f17ac81ae293c820eb5eec0b9af7 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ed8537695bfe494b2c616537a78dc8ada71000e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e6d70665430e242e7f8b8c4160d09403aa0be68782e80565361ae037e2e49d4 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a82d6a87468180404a4657eea98171051b11f6d4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:405a75e09b17dfb1ffaba2bbd7b8065781e847dbdce8ba6b28192749b9baece0 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eeed9779c6c4ef620dc5f205fefac75b5e203e7b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d84da51084882469855325709bffa623bd2fdce8f8d5559b7a194efb5d1a460f +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1be7164587b0f90c4a9cd49ebc47a36e26b40b1b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03515cb50711b32bfb526fdf7cac45d51415afce21d6aebe8f0ed2f8549610f2 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ce6c3f3d7d9a8059b9a62c3afe31c6f1008a7428 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6a1c5f06dfa13bef0c7b014c1fc22605dc062f187cacbf382c4f3297c453481 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e91c22d676c6169f26ee30250b1cd9738531cfb4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7cb2e69fd77925337fe49f24a265cbd0f0cc8d0bd00e77a044bdd6ddfa40b7a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..609ad7a71bbc9b9f5447f737981cdf176dda7014 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8de1af7b72a4fec213a5915aa67244fccb8580fc70e714af9fadb5d92507629 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d230e171f9a9b7cd7bce57426ce28e656d478a39 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27ccf75353ce82ef49983c9a76df109dd5fe521634f97508d08bff6dbdb2c2cb +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..77a2c383534f94a83ff633955de2cb9498ee548d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfaef519881a3511203b052cad2f021d22d2a9d057c84c2099319585be87e065 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..65c23cc66b2b4d430b1620c9299f8629cb88db31 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75e1d7df47cb5c08348c7ed0c160ff9324371665825b4c7f8e2b61fcca26e0e7 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..13a73e757e71ff448e9562db2eff9830e5f86905 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c1ad50d36064c4e3d99edbe5b07502670549d73cde05842418a1743bee44ff0 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..40ae2c54dfc570fa4f9c5a29b8a2a8d075a1fd37 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d614fb23e9eaa3abdd6b66ff24509e89d4c22cd9826713bc5a3c0767dce6cbc9 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3d3cc837fdcb44c5fbcbad14e485126502a7099 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ef73170e10e4d2cb2c76dfc4ddbdbe518d2c4e823c01d58949890ab4f54dbe9 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..27e6a1b54ab1c38b16260b376971d0cf1be48bca --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cde3889fb60e3b6aa4347274089358bb14e1060ba00332fd77bd8df1a2e0345 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ac069189f6c1babdc2f00be453210752338a5c1a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0841bc81bf2044937e0811f3d9889905390b9167745df8292ec2cd7dafaa5452 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..46ae5829681169fa8cc15f73719eb3d1a3491f36 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2154bcf436bd19bedcf08b286753a1b0ded799044d76c3608e44c47e9d3bc252 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5feee758e90ee7df6a359253c09300da88668f97 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:924365b99eae983fd9a35d54dd993445012c58d0d8c927f2b8fa40d1e470b29b +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9df1d38d47d133df57eb5b5d484e6f8792ed1ce2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:775b6d15b5abce68c7d5f2ad1ca49ecb52947560e4caf1fe4990592b02743267 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7178cba6b036d8fdc5e9d9dc7d5fb98db8f53698 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db51628d9b608c2086c9f7550a070bcca7b8d3d1e3ce4d2a555ff002e632a170 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8cab916f4297fc80ad1ed3919f1158d9bb48fd6e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05b64e5db36d063381a4b853744bc601477081201af151564e40c1877b62cee7 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..844b43c8c35c5baddecc5f00b76c71f4f7b98ce7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:300bf77aa1a3e301b5d09e9820dd6d74f4d76a5cc4aa107fe1508f74138255c3 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c1e651b79df4f34021e8d05ba53eff12fc61f13 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21fb581b80abe18ed436b729a8bc6d8514e6cb1902977ebf20d7afd3981325c2 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3d5fbbdd9d97b2d1f12426a6fae195dc0d4cfc9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ef0f15c81c470844f86538a01df98d50eb18afbe41be970b8357ed4921a9ce5 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4acc6abef4d2bd2c4be8bb7dd99d6e03b830fe8d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c2e83bd10238c5585e29cd5c7b5d4b632a30ea4c274273f1ba802120f5bebdf +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..45a54eeb0f6a6e58ddf009fbedfbad369447e732 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:578646b0e9e66c6c3605f37c7a60bbd3cfd04f2122208ee2f202d9043760bea8 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ac991b54e8c4486a6319a03b68645f09f896549b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4be05f078d2b972b096739178a6255e7d0e06ec4f99dc7eb24eafcc6a6020e01 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f8daa2bf91c6b90995d5856ff56e10e4e557d56 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fe087213e16fe066140ea4fd65bd50f3f3fe837addafc3425b287e4a8f4afb0 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3927fe993112548bb037f96c5829fad020f0ff31 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6da20d47d17c75c71b28beac344e5fdeffd3a257a4cfddf74ddb1e9f2b68e61 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..36430dd5cfc3af328e64de81bd7825601c7e95de --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c235f18d7e167efdd5befadbd216fbfaddf89abdff0ef7ee4598dc5789dfb675 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..73c04cfcd8d9d12e08a855bbd1abf5a19945b322 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fd46dda2c56280a6246ac34d221178b109a80bfc6fba295c7da1104ed4b604a +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..138f47e8c0ede9bcbbc4f217892eb0dfa869ecfc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd2d9fba8844ea31db2a4b3982e03ad3672fbe2849c4b1cf7c6e9cdebfc0063f +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a41c4973192cb6c652ee3b40261213d9fdcc58d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:542d45071351730a40bc64fcf794bdd05a36deeafe1e387c56ea3f35f17b3220 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d7678d48b040b370ba6872e1609bdfd2e6d5d0ce --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c0d89c48ba73d284ad1b6ebb68d04018dc1112f0678b98ebd72764fe9c71f2a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c971a598c82098503c0124d24ba26a49e3ee5844 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf3ae2725cad9ce181234d9feba65c3ddd96c488da48d7a813c0a8800b6e87eb +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d1250e7d31fc80981a016286a2d4f10a4cf8f0b7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f5b10328887521fd4fa56fa0800e8c299bcbb78b5e23310842c9dd1008bd3a5 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..81407b7372c57d17d6069f97d26fa5aa90bfed8d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:404a1b3f2a5677614d87679034eee4cb755ada365b91cff5017d450710467c12 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..01b0289ba7a18e26168fac491bbdb62c049880bc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a77ec140bcf2cdeea106be06f77e607e8d58c8c9f9ae282b9026e16796d70f3a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..48502b61d5f3216a8c1ba5b7620f72201c559002 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9ce758c0241c3a18dc094376db0df85b836eea57b7eb449cab362ee72185b5d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..81c434272912167d47e14397f9459b53fada1cc0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f82373e22adb48120abbf06c73b0784bab3974d1e3d7cb0fcf057c9195e21446 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..46933b82632481945a5e63d99a1b9c15b58a0561 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f742da33b78f929b24827d66950feae097199c2c9602fd6c7ef8d79ccd26f7a +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1ce895dcdcc3a395abee350953c35f143dbb1939 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:386ca48a8b56b592565f62c9825cfe6206fe74a4ec793f0d2e8b5fdc091f50c0 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad47055af9ccf4cf0ddb7f7b1b24a24b0b8d35bf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6eb109290f60e298a8c541c88875d5215a2f81b5e537a1182e4130d151cbcede +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ecb05f7fd55f4cc7755652d51c7e2c783db4727a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69ba82ad996e36a60b53fe6a6306aba8c0287eb1f9a4b9c2426953a77a48eb40 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a00125a386f175ade9e0d52c8768212bee6a5139 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00429f9011ae7b70ad1766555a64d3d354bceb1260d631d241b72c79d84a6831 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4cc6c67d9d8b425fb8c2c8e341ab5acb05e8a3bb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37f1c062f31e73e7e0af16b16a6afc6426f626260f747663731c36e729bc4b02 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d037f40f864c66a2a5962125548173da42127ad0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8bf10a1bc74ddbc104301ac423bbbd84bf0efb040130931cfaa15af4590b098 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..99ff592b246f941addda7641643f06b4004af622 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7abebed0ee86c501ba2bf6d68b648497d48a2ac96c382b4a7a94fd476d31d561 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cacee18dfc677d74d8f1138bafacbbaee8fb7125 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a126d4ffa01a15027969f6b83e3a10041c13e6303b9b72f71fa828b362b1ab7 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0be671f2a1ad99c750b9a7e2e1bb1b31127ec85c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abb4a0b1c90b978634c60c09afc9c133683a9d8105ad562835d720d9a78230f0 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dcf49ba19cb409c5b8edd6908bda7d9dc28efb42 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17713ca169e58ae2fdd7d9d1c120150580cb91dfa540f3c6f1f2bd4698c2ad26 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6065e9280ba79b46103e81dd434a1440efa93a9e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ab56b707a675b4071e97de46d7508e6762544ad7be1393d5454acfb2dc6b1d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2af79e6cc86e9d9787796eaef5c391d193706602 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df968ffb4559b8c30e8fb6ebab7d711b56e3ca07d7baace57e11520d49d4a635 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1cbd68b7398553808e96423c34eb402ce992a36c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ecdf588b9e559ea21a6d26ae8a788965e04d0d94becd9614593491315cc97e1 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dba3af3dc47d413d5959f2dd56fe6a847bf478ec --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ab2278c76c7d86e3773f3f1cfb46ca4aafc3203664f3af1e883e70edfd6e133 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c83c715a563429ea6573f6c09bdf2fce313affc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:052a286db9afc03d801e639618a0cc4653b3db6d5a8c2fcd4cdeee5776723f1d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f8362f6732bbb4336eeb70c1ca8640cec68fc027 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b15b9e9a2d32dabedb7decde44fa2ffecee602af5e832b70035ffb5b8c3c7be1 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..79ea73f4c956ae18e4fbe77731b0a133c9c58645 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:366ad4974aa7b0818048bfe4fbe89e9036d5b000ee1df0dd82fb5975496b5924 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..28de0c5fb6244c3745e1d1ee50e344a516b0e707 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e9ef24d78aab029b81e6ab6fbaee7f45cca1f10fbea16c959bcdd34af76f2d5 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..71977349196468a26c59b1dd37c818f0f42a6f76 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3dccc41e1caf758cede9ad8ccfcad4fa22b6ea3a0b39b4144469539fd262ac8 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..611958e9ff1c9478a0409680af72056f8be854f1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:479c8c04571360a9f7d68bb00fa5845dcc8f12054d5f2f143c99dd2b3d1a39d8 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1ebf9827f8ea027d3ab064bc4445cdd5042032d5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61baa5c5baa630521d9308396bc9e5ba14de3f527ae0786a6420b5c0101592f2 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ea9454c2513beb9dc8309358197dd5c548d37d44 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7f02f85135f347bc1c95bff80b48409f470166fd6a57ff7fa13e1fe7ba17cbb +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..580a28efdc103c7ee66096113ebc4167989c5b60 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff35b965a84cb940d2e5f9a57c6fafb0d833125e8d3fb00471350da5e696b68 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e16377335c172b1b7d702b387253785682ce2e17 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c56334670b5ab224c003d6ea877528b30276f45e781ccf2c9dd7ff2a13227d6 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..539767e13b572061ad92e0735d112012b20d159c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:564368881eb3e5f817f651f82d5ed3e59ca693cbe7c329cc5a0a149c92bb9b29 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1715fc983ea8f466a9663ef4ccb1478f6c7c2d2b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3aaef3c47cd8e6df4ccec9c083247e20f28d46cc88745c17dc34b5a22d95ddef +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5de08a045d8b9c001e081f779b9f7a3694d309e7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fbdcdd7acfc4962e961e34ad0223b93017b7794988b7c8342f1930c796624e1 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2aec013df5534a52c7f96ab025a40446a9bcea93 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c1f68120181f3e9556ef4faed3e223705ad33331d422304f444cb7b6c9e8cb8 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d151fb35698536b1d4e561f8cf953cb8b89a1eb2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57d7f6de52f2eb84c0a616c1d4efe59fdfd09463624ab71ba18389020148ea26 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2da8b9dfe9dee094f08ddb20bdddb4db185339cb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:792f272f2ce0337ab323eaabc47f10a6ca466eca4fc8378c807422ef3751de5d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f150372d3b0598ff4e829c77cd5245c815fa0b8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b5e64cb02678f069e2803c31bf9c75415cc5be98516089514b4479b39362d4d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..658bac72e81741d966e57ce16005d5f33306bccf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:474b3e1ee37b660e78d45e38df4a5975e31e82d1d469268a52103c85cdae6077 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1806ec8e3a29bcdb0b30b052a281d58c01c4c436 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35ec1fb4c2c7874df786088abb8eb9f1085b973e3a34bff235a1572d10eb1caa +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..226ee056adb47d61d55e15db70807349e9c39dd4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f7c000e75bb3628cb736ebb641d449b189314e37f08c0fba2e4b72c9781b69a +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4dd635194175e08d9bb32a30f345cfe065ea3794 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07ed92275e53b88663a4ed08d505d4c991e03b493d586370e2e79e11a959fbc8 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8610ab32b68b9883aeb83a5b287559cfec33d831 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cedb9928517b99f6522062daf34059f56ef5f0d4db67bad1053122dd4be283b +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f1e9639b4768145c19ad4f9cc4104034e65c68fa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7c6f32cb0ab1e7b04d8b880ac2902f3e88db632ff63e79d5fd9dc992bee25bb +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1e5cf0dce540b5212423e9a91f77096af80e5234 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ff7dab48f649daaa9db7d5858780db92d8c18c4795b037425f886272bf6cb1d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6c9dd3d67689b6a4a0873b837296005a02aadc49 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d69a8398e71f5c9ce72c4cb58b306c33dd70361538f8f02b9046aac79c42b8c +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a8a7ba3316884407d57625a0c3b95ba60810ea2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39a8720a7658808937f62b4c4166016e7349ed7f0aa033e3fdab887f50942b77 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f40eb5c2ff587119beb68969bceacebb7cf9182 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f037dc90b2f1e2e586b8fb82fcee93a8f9cb639dbd4de714e2bd82cf294f820b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dbd10665334d8ebf03f8845d3f7645c635e37ba2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:792e8f0de2deae4dd19fcd28aba0f397dc5cbf5655929bb646ed34863f8f124b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4e5357f7157647f1a7132aa1bc331168f171d8ff --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b3b4bac4468a0906f4b03c3b4254015f7dbea54e4b60b450fca391b1ed53af9 +size 167872599 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..181b6d1a3cc94494ab0b02d80775c129fc8aec75 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b2e997b27c7b63edc46364ebf3e5b3062f0ca5f5ac20d1abce15f77a3c50aef +size 167872599 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1f5fc2035098202ed0944bbac962433930404e62 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ff58ce0db2f6915cbe325800a87209cb3f08746fae1aff92ca80e6ed30690f4 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..04b15e3147548ff3ae8e8db8747425182367a49e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d1b7b2c01e95cc6a4568fbfece00bdd47231042cfbb0b8a9c76e46e36208697 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6860955fc6098feeaccb835e9a6d37730652f0aa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:253f31f1dc40835d1948d09cb4212ceaa81056b8d889decfba962a0fbe931065 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d0ecae39aef45c2e8b0a2cedb5d1a7699402a48c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9bc1337667f1556a68c9eba2766cfcd6ae68d4d659eda732fb95fdade958f40 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c0bcf316e2d407eb69b96c8b1b942582b7963543 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f625210207c2f924264fd0915348a3125474a34e6ddd574f9d4bc3d6d4a1ac8a +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0337613fd80c6edb714f0e502a89d82ed2087eb4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7111e7ac9f2e1ee5bc01b976686464d55e36fe9d3bea8949b48ad3509b220ced +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..af13b2fe0a53330d6dc2502844937b665e5e9eb8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a873331d3287cb6cd40bd62cbddd03d8303e6b9a8133c8520fbe6b64fb8e476 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f873f59acb9725224900a34b110cb1ed19957080 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1387dc278ac8641b84122c4345a28109e2660219d00b21358d0d5b7d1f4cfbdb +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..43d4238db80de9b3b8c0d17c34bd77400489eddd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc7df62220e11a9398df0205394aee8587c409d306fab8668dfabf8a80d3bcc8 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3befb3632fe0667aa59ddd830fba14d7b7b04f9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10d238ce2070f5b08e8a5db001d3870dd8738a8b7b248fda84b75734c7b4334d +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1d63baf19bd690dece5614f48e5b7def2c8bb0c3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:653221c33d6b41812a4022f78dbab2121f36dc42adb8c800cfb61bd539963272 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..acd1158f948b6149534eb925468f84e41fad9d35 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7de90754f8b166b6e003646c8f646047ba836f3310b5995ceff64f150db20730 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7e75f42d5a7e92fbaaf5f436433af8bfb4f92909 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd28ece73d0ff8744f4ab7d88c1cd88d8d8432c865264df90ffafd84ca79d7f0 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..26adb898c223cc0b02e24382e7f5dc567261cb73 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04c82d09e02051622fdb7f2b17436286cbece6fc3bba3197874455f6b77c3012 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f8f2055100f098c45e89d3be9f58e5d205a029f2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07a6cad4ab5b8b00fe176887b7925555fd85ff78b8ec9a373dff3fd4e774ebdf +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..47e483d305ed0293033dc3d5187ec7a32ead9a3e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32ed62c2a5809312e17502a9a6326b8b657d1d53165c985fa258872a4bf92571 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..32be84e2aa0eb28b1d3a16509a0cd9d675266512 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08f856f70cb08e817752f185f69e51cb8f10fd24ba3d71a701ba267a84992fc7 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..686df97854d56e8d8d8049babe9955f2463bc4db --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac28563def41bb333122889fe689096efc4f6670c00c133eeebf4cad8ba32959 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f4290e139cd6719cdd39740d4a6f4f963237971f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4711740041c423e3d71f2f8816f68fbd05e06e2a533ee06da227044d76f7099 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a93e51e7fa4279829f81dde921e2439a5c89f210 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2245c323d1bf4465bc575442cd04b1c1caee18aee73224e258e64abaf8bee3d +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..52a5673353a746e41abfe914caf0083c0ec530d7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:514a4737fe46b94555f2bc8c6dcba33aad0d6d04ec7f84ab1f9979aa4cb13cfb +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7651f5c1b2a5622ea10361e0f743c04aa095b47c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8af98131e3a054cb3255f20a6ff88291f2d2d158efdcabaa4e1ea132bc927d3d +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f64a096aadc8a3b946362234942f54fa177b4b09 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:640b5ef2091532bb652391ef887a41dd89e5201858fb69e668f5987849d2a38e +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d998e8e15d7f878be22177e7d70881bc98f7ce44 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:366e9395da47ac46412e94241e5d05c3d3f4b79dc73b9fe8f84d1ae2777d07a4 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..35f9ae88a570c50bd1d85d58dde85643b1dfdf8a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfc955bd152ec24a733e8f17d33585b9bc987e8e23c4627cd2b02ea480231674 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4fd9be2b3009c47d02308f304afbf2fef406fe27 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6556f2cd2716575ee448b4099b8c3ee48ffea7bb5858d6f39d0d9c40eead7f95 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..94f3717f63b529828f24f54c3455055177a53cd8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b46da7aff936e03ad6da4a658dee924dbfdd125d149c305beea246165706f6b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0df7f9e18896207c7660b33a3d8b6b62f776f946 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe6a239a61fe52629e5ea3937a4da28ed7f4c8afdd9c24065cfa79d953c76eb8 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..37eaf26918ebe90ae1f5da23841fd47bc9c73f5c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59941852e7762802b0045ab4d7a40f6c16a3699f887d855a77876ff433de4188 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eedaa17967b85788db59b1d1bc3beb887d63e321 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:080c57ea8c5aca4344f419ee9e109fc042cf820d02cfea2eafa0be2523f12533 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7e46d2e75f71a3ebe84357bdaf5b38fe474a690f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa4c5801777cc249c293a2c8305fe60837ee24b1eb869d6d77c6962cb9699393 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ca526a9b2bb32942596921565e79e8553a7c1f8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8086aa9bee5a8336af972718146da49d2cdb63ac6cf3d143ab110fe6716e9c1 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e9810b0b0ca27d600fec8ebba60ddfcbd4098444 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81de8fd63d0746b4a95cb45aa83ef9393fd117df2845d925092413e91557fd71 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c80b2b5b0371da4107bb103db465c2a13de2a2c2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11ba495c5e4e44ef1a09394e60314055ed4565dffb05974bfafdcb4e24a852c9 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2447ba153d30e28ce975d7967cd8bed010681376 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86d0b2a46892d05f0744e6c4dbfe43d8bfb0ea58681c7a401ddafdef38e6e79c +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..abc1e86c66428b513888d5eebc522f145bfd0a85 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d97b132ad56ac7bf0ad20259ba22d73842d671afd298649f4511b9aeb5817d9a +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a88ccd8bdfe3fe1cd92a5fb3b8a495bab6fb338c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f2626a046e2f5dda3bcdca6395db0d3a9a8870b6a827f3d8f5c5d05c68309d4 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ff7347a10dff8ed056928c35c2b897fd4124ed4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2234d289fe9e501e74d3eee01f2608df29c73edad1c1a5d653b63a7a77966ab2 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d6d5682097170c26ca3613a9103ea7fc99892632 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8894f8429cc32e92c607168040b830b9cc94d6754bc21377eaf05807aaa798c4 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..daf50ded6430cc851634d9472160ce7722e08b83 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:406465aad4081f67f698d652f085b183659ab2a44b84c94fe3b9eb1a82f61e0a +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..21176143a238b1301cabb9890a48212466d445f3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41a1faa4dfe08403f1a2ff6d6aa4949e853233e305e53a2eb7f704049382d1cd +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..28b24881cca43047c39d09d05a210a048036194b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5647b68117437dd3421a072bde75b2a201da31719ba14dd205d889eb28ba6305 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..735886509106fee83edcc7b370f377e763f14aa6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b8e78da7333ee07083a956edb657829725a2c2cb87d5a3331b31c2875cbdf3e +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..63eed0f1dae85f99cb325e6d313f56046d6ec0cd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbfe789bf8ebd2dd855efcb72930d5608743c95e38c5fe8a7492d0df287471e5 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..05cf3f40c28be31ab537fd47982a83252f02ea98 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ec04427b99c2237b8a66828e97583d8e25d7fbe830878131b1c7c839415ec53 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3728540568e0d83bcb2ebe34a91339971fa11e40 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe21ee1b4ffac53e595de0856ff64f2fe1957eb1d4f9ca1d0b0f8e39f2177f22 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..280193b05c94ee4fcf69c425df8c4fb17c454080 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7edb5a031f61fcb5ebdff935ee5093dc8b2643fb1cb1f7de51a8018c0e2a7ad9 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0b9a2b512a5cc6e56d47cdaeb11dc476bc02e3d8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:747690c178a2fb1a0b2ed203abe68fddda54a13a21ee3f428d3795b666f1bcb4 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..baf14690fa65d41ad8fe7cf5630b059df7461e44 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:288c9b913a661907ef98d229ec7b03f26c9dab6c80163dc5ffc582dbaa1e95f1 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a097d9040b0501af7ac8cfb5cf3f42462ee00d9b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edefe0ea39a80b9acd43b541fc0db0b216500bf8674dd1a4aaa39ce66519419f +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..995b12a6143bd31d53bba04e6f87ee554f6b9de0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5ac8bf87f57a88553e6c2bd1343dfecd9684fd05b2b8ca7e09e75ee4c96c4b6 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..367d757a0af077e2bae4a0b46bc5c536f2679f89 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fdc7a98952b86e9792c3b699f78ab7e49d76e9d28562b79c7f6331a7b780d5c +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1d5682eb1c3677323794003c72a9bb38de9cda7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45e6209f9700e32934b527843e9a292c9355dce8e8e756839b47e2d97651a996 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..602514de79d0e023034c09ecf7f197c6e551cda0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10533a7b6f77df970dfe49ef251ec9d0682647a408d40b04981e6dc31f0a3b4f +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0a888510d1530a1c643e849726ab832e305bf236 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73c8c5f53f2bb56d7841abcf77a4bf1e007cbd69fb076ecc8e4f8a65ef2468f8 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..038b7f47145e10c75c043c88a4152816fa577cf5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ade67636c8c9201487ac363aa38abfc5fd7ad62bae18409c0b17b9e8e445959c +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..30fb9e92c8e3cff76dd007e1a2a80b5f8d29bcec --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a49595d28f71473070d01debe942cf7ca16f10f5fe7ece9194b6b4067b5652a +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..008471586d3526e267bfcbf3720c1802ed7e890d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1eb212b93c481daf5e3b5a0ea3f4f5df3a11c2c3ced7ff3254681c7c7284dee7 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..929999d2de5eaf01632c276b153efecf8cc7ed3b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66d1a4c744608d39fa794638510da7562ef93014565c63d2b6a51398b2645e15 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a721f092079ce8f725daf18f68a4c2fbe496701 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd8a360688e37bdd834427e809b26184ddaa2ef8f2eb996badd283831f85efe8 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc4798b95c5bc8a063d9915d250da12bd52c551f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b085f2c402c7812b633db35d63868279aa6022faa4a51666bf3c00148078dd0d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..033ce38dc017ea85fb6256cffc439dd25b91199e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb712dcf311f5e04a7192800f24a6f57de827f930e968573e1bf779545367c41 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1b410777c8c91a50b0efccb5431678b0d3758874 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:091edd36bedfe8c2bf485b77e1883362cfcfbe73313038211300c6678a2da664 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b17eff08f8111b9eb81093d0ad1971d139767220 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b9223529da7cbaa60fceaad99d596f62f4ba0c5fd8f2d712010ee35742a592b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a2d20f6b47118fa1ae95b111bc056563b24fa67 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad7704954dbfb6e988481a528867f524f78b6e4eb1922f759b7bd9fa7302e9ae +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad4e5ec17848258c6a32a3c894941552ffbffdd2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f8fd0069c64642e07d4cc7ce1b28b913828bd9ddfd1fd22a87a89a27344301d +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_01-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f70da4a38a1cce76115a40fdfcc56a3128e055b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c573a1149130db40c6e3e0c12c3d05fa187860e71af47fd446bb5f05f2675ba +size 153552131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_01-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_01-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6da9fb1b66f15df5669958ac9a5d2f5c52ba9d4d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_01-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6062d710e64f25f69d5121c3e8a43ced11837b8fcb703baf44e775ed1059388c +size 153552131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_03-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d173e0cc7027f64ee77785312b7d3a1febdc435c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ed042d91c60e5554cecc26be429c36b72db84394e4b4c99d9ae51e0fab25b6f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_03-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_03-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6ce8f6b9a2cf32c1d165bdf97803da28dc5e5fe0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_03-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc7f6628047490328db7ba5eaabb7ddfc9f3b0cd123d0b9e3c1bae9ffd7a53fc +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_04-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..700029423c337b82ea617d71f78c4d170513c1e3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9aad20d5f1915087aa4d07be733f3ae8af28d021f958348deca7c272b4a018b3 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_04-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_04-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..45d838935d66c72c8cff4a74a3220aeb92798041 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_04-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d5d9745a865c2f237450c47259bdda353adb063298cbff976a012c00ea50fc1 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_05-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b527dd8574f0707f98c4fa97c0ee463588e45bb4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:300d3770f1443ba3b545dad99e97d9e40dd587b782c10f538e4c45e8b4710d16 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_05-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_05-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3acd1c2d7f72eeded300bb6d3f1e19acc866e42b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_05-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8e89600fd31755e720953ca44d12656bc70add1d3a843c48696860e2b132f4f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_06-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..761fdf2e27818a3bf48fb18b44231a475250dc20 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab611b85d026b3aa92128f457ac10fb67978629d616e8ce98a2d1f9713278a47 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_06-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_06-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a0653280c78b2769d76177c59d34b0cfa5f4b28 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_06-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2431c1dcd452b8553cf55aaa608d962399012d462279275c16419b2fdb01da77 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_07-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d08d9a0e13b36a4e255f23c9356e76741374e91 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6085fe4ef9fff02f4642aeab1f093d96133ea0cdbc00c1f60eb92b68e15e6e74 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_07-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_07-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..44460e2e631e8d5253138a212d15bf28c4f9c2b6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_07-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e475f40cd065d88a07c6d04984fce478efa8ddea200188cec45cd7773508b77 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_08-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2cc2693c5ac76614bc3b5fc7378e21719840173 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65c92043798cd131652be7c19c0056dfe48770859518157c61f28cd81c1c5910 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_08-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_08-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..59f6f2e5127c78b0507a3e288818df62a9bbeb95 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_08-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ffaf7adc331dc2d1e7d657929375838a02783b3b197b1601c88e86f8934b1cb +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_09-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f910c995dbe36ea99feda1f5d47d790f5e4d8c82 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b621698f9d91c904885cd54322fa5e604ec524a385d97fdb8aa4a798b433d95 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_09-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_09-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c4cd2c223bba16199ed1a85bbc6d3f986193f819 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_09-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f039d57241ad012074c0a615c4d77fbf184cd49b8c278c7fe34dc75c0c7bfd3 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_10-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c347ceb13a3081ff064eaedc05ae3d1374a21be6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39dbfaf8d03852705b59b681ed795a5a5cbd1322f02c392bb0d44d85ea7d99f9 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_10-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_10-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d645ee956a0f597047704ee162cfaaabdc942eb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_10-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76965cd12f3ed66154a17337172fc70299f7f309b0cb9eb26257e870a320c913 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_11-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..46aed8c019e1113bc65aa4c9bb2510fc9ee3ce86 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:196687a9573c602af3b7ff57eb8ebbf0058f64d1c7b2db393b61acebaa55ef50 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_11-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_11-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a61661ef797eceb5c01fa4a1a4f009ec78b74b98 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_11-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebfb3f367ad4020de651b92b6c4a71e7a56f8ce53e16518312c255653069d95b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_12-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5005f650c95d442d65c91afaee0b46d0a60f6566 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b010558dbd657eacbd42b1dc044549626208e3e29953a1eecbf9503679b8df47 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_12-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_12-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a48d9bb7f966937ceefcddec82f80730763be495 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_12-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:486dc6314587ae26e862de8c3da82b5afd66ce26262366142e5b69091317a7bf +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_13-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7d2c2b71d2c803166bd859fe53262704720e80eb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e36026a70337d7f38dd85e064fcd24d6a3b68bba9e4d75a4e1fdea615072a0b4 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_13-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_13-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7360b33833cdeeaa7ce35f90f0639540418030e7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_13-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edc433428d4f7c3f4fa15d8dd651b886eed8698cfe86ec93a05bcf99a3e36740 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_14-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7bb66ca3725049b0a2bf4cdf0c8d295a1408b2d7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:852eb6590107bde0d530a3527dd1c57880204c9bf0ee92fc8932b99cc28e4d20 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_14-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_14-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..99d1295d6def86eccb48101eeb84d9fd7bb63eec --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_14-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d86d7967a516a3bf91b4607148d6b20d61a6b080e3489da3396d79ce5b3370e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_15-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec79a9a69a5e99c21b92d6d89d5d92c0ee9f58da --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f60d78ffa118d2f35fb12398ad7399c6ba597adfe872d053d44b8bd347d1b4f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_15-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_15-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..825f07e39fcf40423588d980c53bf9ec7198483a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_15-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e8d20852c701c8cc16cf2f9ee29d89745371226739a556bf59c6db1d465cb04 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_16-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a76a883551d5ddffd04442d45f7a3a93f0d92480 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6a2efdbb21392fa1d48d10a43615107b21d63b690a05ef7f32e53a940471dd5 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_16-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_16-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b888e637baf18de47c5d26021bb9ad516d789ee6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_16-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6529558a6d436d044fc3f2ad5b6dbac8d119d3cd886e2b310ce1bdc4b601734d +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_17-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2dadf89d92be79cec5944cdc79058a37c9c20c9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9f71ec43212dcb9ab96205ef177cccfa7af1810c633456b110ca379c7fe6156 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_17-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_17-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..80fa2a3c3b393d5c739c7f10ded2853ebb62d79b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_17-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfd8c3dee53fb7965a57c8e4ec178eb5116d7c8372e6eb94b6402c8fa6263794 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_18-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f9708d48251a1289d0a4c88269f8905fffd53b8e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d22418c080c70133f675d9103cb0d1c1f84e615667e24b0e8b8726990bc30504 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_18-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_18-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a27eb3ecfb38d95c70deeb5f219752be43534b38 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_18-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e3153c753665d9d4b2dbdabd5f66b334232100e28a2cee6c86097e327cfbe3c +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_19-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1d73ca601bb2821e3aba2a0cade5158a03c67d6f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae4a5ec268d426e6e4de4fa5c8471bfa8da038c8ae607059c8415b6ea48c1929 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_19-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_19-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3522f63e020c7e87d6d446b146e52a17a9e80afd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_19-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fc8a759720daca32ae8a07d96a9974740da1db2ef0604ef127122d92883b5ad +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_20-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..68d02b6b96b192cc181106cdff77bc86c8a1afe1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:851f5c79b61633ef5b3b59e349bfc5dd935ce57cb934bd21be2c634e6fb49419 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_20-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_20-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad95abe2ca9bdf9c6671cf38a68cc981e32eb3d2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_20-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63f531a02360d53b13bb468c669289d3b8a9b1b70d22fcc008a27fb129444a65 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_21-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..af1d4187a7294676e1964decf1c0df30b17c300c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d21341528357dbd89e8007f04eb9c47697ff706f233820ea466573feffaf08f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_21-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_21-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eea29d1d7df99c3507d1a9f4f6408fc550da2d12 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_21-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b1c844ea1cc5c018c9194940c967e839efd299b699573c6ee0dcfa9dd5240f9 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_22-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_22-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..850f12f4b2df5e2cda31d5ba4e81a972058f8642 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_22-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68b9485724ebeb0b2fb0719e38f61e6ad3f0bdf2e10a8bd017f2401c454a15e5 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_22-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_22-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4b290e7dda39ac316212bead327f7a854315d81c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_22-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:530dfcc22c39ebf4aafef5a4e29e9750aafd9ba36cccdc5f108f2a99fc77d7dd +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_23-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1f4ee100ef679cb0b32739966a8ce95ad26c5479 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08c05d6212dc3ef09e716cff039d1747653c57c5267bd1a892462a27c754e789 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_23-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_23-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..92f95d058d1a2c16e16de8e4b90a70082d36efc5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_23-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62899d8b64fe09bb5c8e25135403e08732bba978fa7fa412f8adfa7171d91d2c +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_24-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_24-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..802369c2bae7d7695325ca9ca6610ad199110d6d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_24-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11dab657ea135ceef7cb795b0d3a28233316f0503e4a5c83c423ee77ccf3f61a +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_24-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_24-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..21af8a0feebd21142817d2fd6e120de886013fd1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_24-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60b02bb58e0e52d6b1d525740a3bbfa89defbe769566f90e0786c6ee9e865980 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_25-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_25-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a936e47553d7f3127e5e9d6e4037a04e59201b7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_25-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66e06e25e0a4e60b12bef29c4fda87704d3b9017b4d4357596562dc4778fcb78 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_25-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_25-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f778448935492c0da4894cc1ad4475debad200e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_25-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb354b29deb39a0d089f840162d454a53f135765dd6f82ccb96eb13bbf2cdd76 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_26-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_26-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d90801a1880ba75cbf05698ac80ee7bfb8e6f517 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_26-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd9f56c01d24a23191d1645dbdb3045392511a6dfe21fd7e5200ee0d56694346 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_26-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_26-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fefead37e12645f3c9586aa3da0530fee0c73815 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_26-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19088abc69e942344f9a2c677f273d1412afd8d28bc739aeede91f7edf2001ce +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_27-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_27-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6aab725cd4f988887a850b0378814b0e50f08231 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_27-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc0b61a5e20907b1935980f53fb578709595c07b496ebe5dee5f44ffbddb6672 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_27-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_27-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..85af60dfcfb3473d8a201fe0e8ce99919c566ab0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_27-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8c6fd8408c36ec8b22651898228a297105aeca78b0f9b3b29bcecd027a68181 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_28-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_28-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9fec8b63bb079665f0e6b3e2452e054cae68f8b5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_28-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fe43103339637f43db5a378a2d8a741e84d2d6bb5f03f849747f52cbfb15524 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_28-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_28-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..476d74f85d08a4099b69b3b40647633b8d3d55ae --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_28-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e875bb3bd00e0a116199363753d88953595d0d0578f3cb60447bdaee8bf9583f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_29-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_29-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..157c3fbf8fb216b52e9beffc346e9cfad1830bce --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_29-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43985007756db034b5335b92f5e2fe0d2ab464901f9b6dd70c67f611fe7205ba +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_29-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_29-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..496066034bf8cb196c227f87e7c8651de6f54400 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_29-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8236177eaf8785fac56244d458c1905056d039081744899060cb6f40b8e8d466 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_30-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_30-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2194420c8b8c7ebc0417faf399fe039f64538a21 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_30-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b0603cf1fdff2c426a9723dcecdf28a173504e60234e212e991426e11b5c27d +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_30-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_30-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..47c5ee8315382c4144266d74c43a27ae700b574c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_30-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a70423374803b3a210743f13a81a98111b9ca30f931ff8f027ce711ec94bb5ae +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_31-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_31-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8e7054dd38de5f08671ea2fffb39b5f50c1dd003 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_31-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d75a62eb39283a37a35c1afd55ae760445124ec523c858f1bd9e1662ff8eb8e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_31-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_31-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..13ed6221ea5107b761332eeef9039909e2a4ea86 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_31-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03c6c8cd971951e721adb2810e80047c353baae38f80da06f282ab28c5d67fd8 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_32-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_32-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d2cb6fc9f179b5fcb050b1fd04414b1840d10f8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_32-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1c6fe01d72df8c3e9f12656d627cb9ae7bb940f790458639a77ea320c8c7ddd +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_32-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_32-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bf8fac5751a2349db363936b6d83e896d170caf6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_32-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96e8e5723b56db848372d24d3caac0c18f878faf2452d869100bc16f2f206605 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_33-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_33-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d42f6a034b672751cb323a8d4d614de1348dd1cc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_33-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66b98ac8657f698be1525d0a3e70610cfe422dce36fb2def6cf05066ae8d16d4 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_33-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_33-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..85e3c4d49165af71bc1be933db4f58de1de50824 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_33-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac51bd8721a7ed8133704c504a4547c6c1e120ee6d098f323f5ec73df76f7ba5 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_34-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_34-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c9db68bbaf86ddd25b16551e9c449849c523597c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_34-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3acbfde6a1363094c9b37d3e8e3cd7359102dce04735a81be8d5df9dfa922459 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_34-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_34-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b795061288a28b8a9448f1ff006d3f3436ddff01 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_34-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:097633f784bbe68b1969505e2b7d0598f81ea4547daab4acdb2293f3377cf642 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_35-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_35-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bf58e1f3587fa144de30562fc74bb23b2249de34 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_35-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3163db7fbc9d8b72b7aea0d19a94fd76aa039380e03b81d5aaa7733faa800cd7 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_35-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_35-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5652950b994f090ee062494250c1e9000fc334c2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_35-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a68e9b401a9deb33772fd2023c7fb5bc77d0f923ab2c154e8f00406e24974a1 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_36-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_36-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f75d366881cb1b7e132236ee64d48f31f82b1922 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_36-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb6d887804bd118085c7e9d5759cc3ff3502e51b1f29d5665115c541bfdd2075 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_36-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_36-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5cb31f57299d1e523e410052400b75ecbcaf6996 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_36-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35d72b504bc41b5fd50da250e06defba17dc205ec1c0652d16f398332d6a2976 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_37-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_37-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b4dc092604fa47595b4933a980c8871ff627adb6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_37-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbba0aafed5e7b3767d9cc8c8c9f5280093fc4afafb8ac7be4afded01b2c8afd +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_37-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_37-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..703464b01da4fdcfcafe832eff68fbbc47c15892 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_37-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34e8cc85e05680bb8b52c230a68c60f3ecd18983d95fbdaba8f1762383db93dc +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_38-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_38-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..84f349f470d7200f9cba65f7376773d5c1709a24 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_38-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1b4f79b8388425c1a4d4ee5d2e20db02ec01de23958bc1d323c7b7235a965c3 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_38-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_38-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..de7b9350227edb8fcf3bb0beee143a92e4a2dec3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_38-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c9a25ad246ff4ae8f7e9c0f36094a624e681dc0caed1db048e961f6c0718dea +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_40-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_40-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..01241f3ef36dce6fc22bdb0b507be57f01a2e69b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_40-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e6caa519898ca4a2a3b570c1d7b594f35b80a813e53caac2006bcf0ddae3248 +size 12483 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_40-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_40-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7537e92ac99be0391905769f4c492578e248e28b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/layer_40-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5402b6e10fc1e2dd86703c5b5c0771cdb6cc441becff57694667622ce8246671 +size 12483 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/mp_rank_00_model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..70926f91964b10a10f17cfce8bb5b66ca3c0f734 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84217a0b1700fe09cab1cec54e392df5b9d515cf9abd95eeda81c0e5d34ccc7b +size 51443 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/mp_rank_01_model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/mp_rank_01_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..259f3cf861b3374c54890ae1da016f314493c0ff --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step30000/mp_rank_01_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7d0490ac4a3955beefd33184cd87c7d2ca25c8ae66ac19bd772d77391576da0 +size 51443 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f41f777799d01215a88c9047eb8627a31a63eb45 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:042d4c05a4ab74406f29cb85b80985e655590068b2a33c3fa8e57bc3f26887d1 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..31a700cef59b9ca8963d6d6cf2290fcb2170690c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42650cf04ab5ceba6469a67b9413e3fa641ad230799178044651c7be00a11ce1 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0560cf5bef79e8d1189d17b7c953f3fc0f5aad0d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f95fca2871a0174bab4cf8cbb5282cc1abd8b92cbf90376dd10e80bdc1983e5 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0e2c52c87b86d3adfe675e032c2d16fe4c4e5370 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ed9c3f0eaebb6734bb04657273b13197a8dd7e67e72a080cfcb273a1512d7d3 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c833edb106b36606ed4cd1a4b1ad799a3b56645d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9868562fbbe43d2b63e8bf22f62e402820d607f4b0daa78afd5703c229cdfe9 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d661712b0db5f4a56a91136a20edc6c88e9f0d59 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:848250edde679c221573d248e247804846320dea0c6fd721ba2aba796d543b84 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..df1dcacaa08cdf00aaa90a942741f9b20efd4738 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49c7a1ab5bb420d06f5b7b2233cf20986839658cee1c1cddbace7b836cc99f76 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6b76ef2a9fa1160d4d47a4b6fd003a260df6b8c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b007a229d783669b3410c1bc9f98fc9b7c79645a1f4b89b2eb2ff6b6acac0e0d +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a3c2564b7d4850f3a271251579d8aba4b6c0e23e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33bb38c21f5c2b5f9b7487f10178f6f0f3cf179a62fcc43903ee8d294e645dca +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..76fecbe3bdc275eee25c6d659c746557f152dbfa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e60107ac0b09d84530afe2e881524b1bc86ca671035930f335d512f896cc7fa +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0cb764de8a25be145f2ff0522bf836533b05b35d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab374d7e1c67e2441a29ab567e55b7206d61fc23bc81353cc2b187b8642c2e0d +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f766754c90c8930c84ba4a20ee415115daecf17 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8abdad147353892a4d36fd09ed3867690bd3f1958773e6b22f5c477f6c5f8439 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b7a8b0252668fd8d0df0bcb20700067fb03f83d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f3198b5d1649fe510550af3e96370c9ee1b991317b6774b2935267dc5645560 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4c00dff6b343db9c39e28182216f5821af07de11 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20925e029047a84faa4d21b685a31d29d7c6b252b6f2e19511dab8a4b6393747 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e92535feb29cc2f38bd83537b8c239edc3b7d607 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fd1e822473e9bf13d0d2f8fea529a067ec88d6e8bb22d0adeafded4d84a830a +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b59254f4ebec030a55399f45d9e5705f7997d26 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f140b756535c07fa93ed2f34615dc587064bf9d0f02ca6ff5912be30b135968 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a62005836aa094f0b0172c20f24e3d585e5bf3c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49d87b9db584fbe015fad964b007211ba1523a503a27453730145bd6f0a593bf +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad04d9bb31226b86b879a49bee9c4825e68f6534 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4532ff3ffb34be9db78b414a51d147f711381277ec0b5d054082b3d96959df5b +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..858c93b3f92c0a11195a1e90dd4c8b90d9d0d628 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bb418f478aa5b0b843a498b1b4c82c0087b262c17735058c04bfc80ef23cde4 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..93d899166cca7d7621107994501b489fc12841a2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89d8f9e3faf4796cda955c2c16912fefb6b85dc464622452cf28850c5001533e +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0dd4cd5411a310f98fac6b7cce963ab38e242ee6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1eedc88a3a8576a208e793ec49810e89bec7c0cea24bdeb46df032935d4ecb6 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1adfbb534f7d3bc4ed9875f2f58ef5147ee23f40 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89b5a2ea5e91b3120e531d74814fb3d8b7df9025c2b7085e21bc5d013cac9195 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..410918102f278dac0d44b85b48f2620cd9edd0c4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b284c589de79e272acbcaa4b582e3a1de484d77eaa8d95fadfd6e787d6eb7ffc +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4874e9afd2e01b48e801331eea7deb36de33cd20 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e17fea44ac94559a9af8dc810f93d7a8b9670b4fa5a8fb9491208acaf48306d +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..48a44254827a71e8216c047088e3771f5f70232e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f387b7a36c23ed972688da66683979f67e3aa8a043d8b315977c7ead17dcf36 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..081d3ff17c2b9511382d26d21a01ef3b12545c4c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:314fcd10d436a99a795e6dbc9b1a27afbd59ceff1ba49cfb9f5274ee32331d95 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2c80d186b08e63a00c93f8895026a6ca81257d86 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e6102772817cce4cc54b78fbcb4e45a6ec7df314e644bce35baff0bae4520d5 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1feeec64663bc5f6b6abaed136bd7b51aa9230b4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0bef4a66f23a2d6758d78e97ca61e97bfa2ec654eda3a754597b81481d72028 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..806e2941658289ffd6e9a3a7a808afdef99d1cb2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c630e29c2f16e3282139e4da8cb8bab6b49b20b6a0be3ee424f8d9c23df7bc1 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b71f476650c3acbe664999abb097134b9e7dbfb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd136b066ca8fdbf03fa4880aad36e4a00d741fc526d222f608f0c98409d29d5 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0a7d4d7165039563efa3c91cc11dca52fa8711f2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8dff2737663d03a344a8661f9b76b59a18f37c4d0e1fce386c6ba434c4b000d7 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c45cf8f65e14f8dca68b124be2cdf692834a7dab --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b563bd870b21b052ce3654d36b1621baeb2d87fca42bab3512a11ecc6299aaa2 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b101c6e0372638461f8ed71f683422828a7957bf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff0ff41ef4e7da0bee1ec8148474bfc44dab990e91d8a0a327db5b5c11c71188 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ed56af9feec44f17ca70ce0458be97171a77bf02 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eba375172aec79647af9026dd94e15af4aaa1dce0d3400cbab4a10b354b89dc1 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..284b36a5ac55680cb0787f7e0357af8efe428820 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:179057f67cd67728f966d0d0de43ffaceb6ed96e4c0bf5cb410bf4fe2bc7b370 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..40fdc5d6be2cf512e24e166d5aaac449f6c9a134 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8bfa9f93b6cc87c214b75337fd9697dfa7d17b1254514b88da076e2dda53b34 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..86a98ef6328044ef2fd41762f573555f2d5c9dc3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79aa5c12772067db9ee20dec2f44b6a01da65c1a27cd4a488e11822913cfd24a +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ef168dff8892e4ba0cda2f2c2175df9292662ca --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4024b1ea107f1aba3b578cac38395b3171bacd47c197a019fcde1ba70c6ab0bf +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..826b1a7a9c4447220b8754283507f9446f0c9316 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e53ea8cd70fce3cf0a46a04b0ef4ddfdf868f3af7046a4c3d1ec7d9b26b6a6d8 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f25fc6e6736d5b0051ea945f06a9288fb7e39888 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87b9fc20cce9e5211229fee556e0d7fa741858ab4d61ea59fca6e69b59c6fd91 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6ab6f0518c01f27fc48e9dc7fdee987015a14236 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9bdbdb69d165a5df9704e52dbf12ca1303a5886447b9685961ea0df78280785 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..45a12adfe64f8a87872f8e9c73986665ad77e4b8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee9e5f2a81b32055aacbd65cd10b7bbec3a08b13ef0c4847a8b9c2fafd472557 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7e8d5f68933b21d838a5eeaa97c3dde60fa993e7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa35194f08c7600339ba2b911e4a7ff5eef7635a8ca53a245d27ac13b4b3f3c3 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0ad363607a9127c7ae5ccd138b193b7cd34d61ea --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7073a03bd63360ed1b0a0b71805e80c406b843128bbb36a689602cd41884ac6 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..98225057c013265577353683271b0f05c9eed9cb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6df41a433cf3b5f5d612204a0d52dd6069b41527d39b551d0ed4ba79d851f05 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f6b0eb015704120600f77145376fa99276ab910 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66859c58e49b61c28b921b518c0885c61b6bb1eb1af49871ea71b588346c6e18 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..734405daedd3712e3d4f41ad3117642ad29b08e3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f882e86a4f52e3fedcbeb3670fdb98c3a1917aaaea02436a4023f049ea7fdbe +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..75baaa8cf0a02d5fa202465089482670b73d7233 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8bb236103077026796e95fc026b39d0af4b1a9b66b32245cd1750fbf43003d5 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c27a36a81933bb17ff65730911fb3f9721f26971 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78dd7c99475d4a143a25ecde414c19c3acc88d2441f07ba3e0022503ba5fb023 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..19e2bf1c1538b1a6f316395666a83fb177f16ca3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a85460260ae8c11331ccc9f449a614ee71147c5a4108f4a57f30824a5ac9b963 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..59eaa6e6cdaac05f50c96266a30e30e161263905 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8c36301b7167d0be112564aefa8ad77a59d7ed9a1d7d1f7c9c8e468b79192d4 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8744ff5d4c750527441991cd3162c1ab57f37e67 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:575c56ce8f401d7d96baf30c99b0ed0c91a04a31826e137dcb9c028733b45737 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3895462d00c3e025110e57860a6b9fda0595cce7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:164e5c63b2fde30e000b4785774f61b43bae78d7e93f8f30648de72f195ea925 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..98b6860d5d12032e46527de292b1e76abc2e3232 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3de9f0cd1114a3d32e455f6b1ec9ea7c5dd0e8d49a3352233c003e4330f1c9e2 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bc05c19f2a3d7d53e3f94358807d86be247550b0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c4d74eb200751d8028446be83f8d8f3c0a75b349167d310e58d61e1a61a1cd8 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f0042e0238dfd55ba3348c228ce0ae916f0004bb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06c9f99c202c79e791d7e2b3a8d02819907365f01dfa3f06c7890d1c8d51c978 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2dff98c5905e2e59411593b33a693250013892e0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bab9a952394813ae87ec6961666f70c94ffde9a47988547784f720128d79c740 +size 167872941 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8be996bc94dd755dff525aa0954a293c87d70ac0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5a2fbcf0d173ef46d8dee16de72f43c947093b1c396381ae0351c179b773dc2 +size 167872941 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e025f09620c45bb0ac8879f45a5add1a4c6d4284 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c97a65b94d42e4f1121feaea84e02244354bad51e482b8f5564133d83b0a3dd +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..97f65aca28e7f78cf1562cc2892c3f9bf6c90f29 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3081c6ddcfa856e8b77f349b6b035d29dc134640f00b4bd969910feaf552622b +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..01138e64312ecf896ba4c9a6501fc469ebd53c26 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97a33eb2b4493bc65c63f4e97a4035d0c1252cc65c5991000e3adad420f77905 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eaea1b5da762d0f84d69c24d376b1b41ace375f9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ef99f2b9befa4907bfac8b41dc33722ded42fce38ec831d4fc45651f6a98a2b +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..91767778f038b8d9ff34cabd626ca31f4f580870 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73d31f3a02eae0ad120953eb59dfd56e40234f64120f727e898ec0701a3d33bc +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1b1c0f51ad9eebd4d571d189457c7988cf97731f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:850f2d248b2c5ab798e658ad75d3363963ec5fafb0c59a12aa9da3380a1f754a +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c0373f1e553f4ebd4dfb70e0fbcc16449653f0d2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ebf7ce728f6bdc418aa81d7ec9ca946e3e7342630190b1a99e2e0d242cdf4ec +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..26a1c2c9306c4e6a2179105be54256f4ec6e3342 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ad1a0590afff676decebafce2853bdbfaa00d36cc1be52f78c42ac67ca7ec74 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..31aefaccb3baf5607f7639022292547e18a3215f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1042060eb3838dbd8888ad9a7032963ce5c49722d072e01c279ad385be216af3 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d17f2849cb5afc3751387d7a1ecbb6690dc1fe80 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fa06f588d417b1e82894c7b5c88141a9bd46e9cc1e448127d7cb05508e3f3de +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e65a910287ea5296caff12097f0cfb23aef4f2f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a827d0b8b74552f4a6feda91a296c498bde50faad5c1c9ca1900f58783c8412f +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e0a55aaf5f91723f1176940d24ffb53e409f5af3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a05d81f980e835655711afff7a6f3e5c169b3de1d716e510da2a2d5e3ea8f73 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..902632843ed348f83dfc2db1f2204cce6e24e359 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:434539f2be31d8093e8316d4c9c4ca51d1752adeac82297ad3a3f3e3fc5bd840 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..069353602fc6ac85dea86f9f16deaa4a4684c625 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6357e0c3e5d2aff0b1bfdbd24ad07555c7a24cc004223af87af456f1682c3859 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cfbc7a83451acd3761a1b1020b83cd1c369fa9b3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90468b8268e5d8b20219d5f4a7a518ec6cdd87d63578893e66b95fc736687942 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b03e04b0a64327b5d0edf701274202195588c449 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15b92928a533b4395c805be129f5599e0ff956a658d068667fe3f17c51663b7e +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..028e2afbe0fc9c4e28f2996e1b4a30edcae97d41 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f5364af8f878132c1624072abf95ecf6f6ca3e4a26864c1e72368920665544c +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..16a168ba1994fd2edc369330efb67d7d80739fd0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2d4e88b6591846073ab5c4b9cdbb6559b70f455defedf991fb0fbe8885729b8 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..92a2eef638d0a84770ca68afd2e2c22395127193 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b924c1c17f1e846af3ee82a613dc9e3df2f6a0940d7c52d3cd9fb9a6575134c +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..32317ae3771e9c54cd57ddb4b315ac2684c43234 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e3e62c8a1cb92e3bd22ecd0880a90cdd129372896166aaab44182b230d15bb2 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bdb8c5131e201fac7eb243efeff4be7b34b725f1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7aefd4b99c9540f2cd06449ef5a7b7222eadd4983fb93fd5b5dfd4376e58f87 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..02c6173d65a0635cb6dc9b1987ba47a971afac50 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adf0e3e34e0e91cf284d12c417e10d6c4f96f0a2a305b93f283179f38d3387cd +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4a29ac8d65ddb65fa12e36519822ee40b1251a9a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32168c218c6524aec0ae9712e164535d8c5a78308787c4ba7715682323c9839b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0cc86e8dab7800059d67156a8231693fc50a9b92 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c19ee86caaf718b5e0a97a8b15640aa0c0a7458624035a6a58196802ea41e67 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7e0931b07559909a9b1fa1817af5bf78f2337cf4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a57a6dd38e270666ccf2d40a3d98418fea2e3eac31829dc61637235cd769875d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fafe70b051d6db0e9dfc394206009554cf45a9d0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33e9c5489dc1e5108dc98a64f378c1ba2864f143d967963600e32dad299a9414 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..54e7dc1c69cd6990eba66ed4d55ba90ded0eafc4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9d3fe3e96e2a75b9dc32453933cdc3574ce1fb981763a5f780367b2f9a80bed +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a70db738983b2b76eed5e44c82b4ab63d87ca02c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d7698e8c06b092e4e1ef4cb89c12c2de0e734df2944c164b87d815099b00713 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..205b38731b355a2ce785ed0103934f6791462b79 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3787290ac26c482d3400b54a4fcb1ef1a640f8306b9ba91867a969119a57ef9 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7b3c4c4a7c367b4a4dac86e98fe0754ff7c66ac6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f92d009cbff52251c84881d58e417da2dd839b9eff59da4cf875c66b59d0d35c +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2c04d87ebe02995907573a5295497a2963042d1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ebfe9096abe7ef01a0dc103b27aeb07c89805e828a05f40dffdbe39ba0102c7 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4010dcb8c180761383e2423a723ebc7b1de892b6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d5a00dad6b4675419db919797513375a20452c6ec0f832c5ba88847919a4fbc +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1ed096e5230cd223432213f4e362b141e16e23a8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85ad690fce59e9f1777d96ff741d8a42e70541268148cdaf6564b38c2df28a36 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb876a820134603e54c53682f2452c0db8a37675 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb54cd78ec7e4bcd38ef612e10ece99bb5730bfa959eb85617e7f3fd6cab092e +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..81dab1733efb65e7c26cf61ca3dd8939fbcb7ff3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d448fc660e671567333bad43d6b678fb7723f3ba267513766afa0c143fcb33bc +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc2631ee7e039b549d0a65998a7779b20d50ae14 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58b155c2c8c7b0f6e10aa7375b8e619d0aec860ea5ade8e436e4987ba7851748 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..410961272bca55fb38e6a48e54659bac4a4fccdf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c559cecd22975b3132cfac17c5fd752a0d7e24817016b7f9b81b56510422b6d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d59652f7cc7b64b210cd084983e3459a1ccc8142 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b931aa43f8f7308d989c54253ecf733c7158e4b027d5909f80a7c0c2a93f5062 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..759180f9bb3266b61f16fd948af140b3b7713397 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e1d180a3bda3dcdab2483c9514086d70592694f7e6c8b55370c131813b7315d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e5b665d32a90d3ff0c20c4a91da7ede0a86dc46e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d653c31acc877a7cac9ba68314d9d1c153d1d078aa1c989c4b06276581fa8211 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ea8b6b1b1c79cbe4a237ea0624fef15a5191782d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e1309dbd26329de81b53514f974bee7dcddd9a6c647eb5e829e7298d6caeb11 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..819e5ca1bb525fad25b68b9710b6b52ef68d14b5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42dffd968d0d7b91204f331429ebf32e9d5e25f6a5c98e5bbb0198be2d999a26 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..02cb8f0f684b0c38299bb289c2f18bffc2c97574 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30baad8e25091cea06354caa9b7ec5053a7b4a1ec03a1be9c4df82b399f39205 +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc7ee4ebee43769f49a024dbc7972ac4421dc44c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29e1b99e7a2acb79481c0fc6e3bc9cc47926eb5995c03944661631e7b4527a73 +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8be4f73d329055597597e258448ebcc9aa1fea93 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66f9731c7013c719e06b8bf27fb0a63772096668cd17dda18d765bf628bdb51a +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..53713c6251a41e507e385743b3ff0263b28ab64e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e6139e940c30b198a42a4472e3f0174805c15552e2ece69858e269cb00d4f08 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c62ce89d889840057945ebab1ed8cf1d0df15f04 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14bd819ebfe5e18ff6953ea9bc695c2959838afd93e2ffd18be3576ea4bea040 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0987302d2f45bde6c41c800ccb3b65d5c6c41701 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1936a746cfac1cb74a15860ebf379633927f127e70db27ff266c45ea22b2584c +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e64a67506d2390c154489910f1e109fd46dedafa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:937e473ce4ec0e7e2d271b82e409d5a1fed97d1b9338fafff70e63dc2a9b83f2 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bdd56b0047ed7b537ade8953e1c83d9d9eab903c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be992f94a84e4662c7f492ba5320a85422e77b4f80e1a45f02f595904a5a3d83 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ed3a45d39064074a4fb32fb16ec37ad4830db8ef --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f523381d8b9948261c40cfd8429e1a2f7a3618180e0a9fdc9cbcdf511b5cee70 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4b1408c7490527f5d9cfab348e0e046102b89714 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c4c92cee8e051bcdceff6e414db31a6f8b52b315bd764e9b75cc360da216388 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..07cc94beb4bdb8232fd4f4e50080508c994b434a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b687be7060649e689fbbc0761fd9ae6c331d0e64d9667bbc86708f120c0f5ac +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..23cc63c4631d17c89167b75afa6dba374b928956 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86377148cfdc9cc5b6299c3626170ff0ef7d03a7e3925c295f548259e11d20a1 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6dafc7ca42f2c3cb39530e9cef49ba8c0f378333 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49ebe88fd31fb8571ed4042678edb336e21028c7cdde8e89eeda2ce128fa68df +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d64ffd8f7bf62442a5772ccafc804d576705f6a4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80c09d26d20c4f78aae3b87f4a3d78a94a9062e8095224b57784b8c70ad4488d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d87c4f171df0742d7ddf184602cd55a053b46ae5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af84382fd1fbce1d45f88cc51a953d427cd6bb89254e2de0b3dd42c3b8225dcd +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..caf347f9e4e42bd577bb8700ff8330803e6a2193 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f040689c8c69077eaa5b863a8baad89c446439f91205bbd6f0c08bfeebb93b3 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..817c3d71845db9368d95d2f7a0d480431795e942 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49818818940367b6c14d88e061e7c7809c10992687e1b86143bc7c4f33079f66 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f180deab2de6de2dcfd28e2d47aff6d6ee4d53b9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db3baeaa1b7152f000fab7bb18a9ab745ac322756f0d31e2ad14acb28aaa7c86 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d4df81b70e6e0a6f5ad6bd170df043eee8cc7d7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:adbd7bb49a11b7772da811c6c74e44534b25ab1848faed66cb377e08f6719d89 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6389ec694ebe239006f522eb18974929b1133a82 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ae38e53578d9a997909e7d6ee15c12eb091c50906882ef746fa8eccedd86c0c +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..72f1d21602373d390e87565cd1826a375699a9cf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab7bdb670d1c7f32cc4bd011cb7e0b42516e383f1354ea559e40fe7c40318503 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b49b8fb77479be0d1566156a7137b68b00d52337 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9c97086e6a5d1d581a46541dd6e6b3fe00a5e9d94af860831512f075874dafb +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8e9ca2b399dacfc4d9220a9872b00f341c478eaa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ba8bb6cb9350a4ea86f642780cfc6f1b5dbb240212af524f3f52d2aec3e2aba +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b4c30382ddc6ecb76028e8a226682ca6478eb81 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e6b6645d99a23e100f93e33934494842faad2a619c663d18c9c2401cf26aca5 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e45d77f454dbe2dc7c005ba62a48c262616ab99f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f0f432657dc30b417c49e14bfaeda99b75f8fe4346b739cac1e089e384c6a16 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c601b36d6c7c12df0bf22954a09b82605f7fecee --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6af36fde7def02ded66a0b8ed14e2b2b28dbb52cd68789861d367ef312b2d4ee +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d3b2dc41e7af6aae59d6e0e828126132271f815a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:476d8ae5a3a71cf43d3f4507a9d4051ee098a74d9f1268869207552e87ee69cb +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e66b53d9cdcb9e160d68e084a365418b3184dedc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd9ba9188d23733bb7f9b80828120f7f468dfe15d230b2a1fcb273109398ca3e +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..badd3203e8b5deeb3e41a66e54d91391ed627525 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:626d71f21a2f19246613bfa7768c5a6e476210ce30097fe39b313e246e11f0cc +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5e556c1391dad7bcb137f6d5f2e39da440fff32 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f800199a0818bd9b536faccf43892023c9e0c336333ba5a0858de981a07ddb1e +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..792a812d0a9c4b2e5af0c1badb51feb9469e714d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53567b46da0a47474c9f8f85a818e8f1c92b3dde298d86d7875745368ca0088d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d48eea5b8153a56bb284ee05f6c8ae4dc5bb0293 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8f495d7572d5c5486772fc47c69099b1f13589fd1639d7457169d04c36cdc59 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..10b9007940ea55948f8153094e13b4b9d11c50be --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f93a5a28dd5f52ca7d7a42d56aab9b350fccf435f12523aa7c67016eb688e52 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5e907d051ea92a2a55b3429b70f98f496f18f3c6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cd20dc006bba1273caaf28219b5442d894cfe85a1406805ea0d00652993c94e +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..004433cd22c2b42da4535b9b79e4c3d2f7138301 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dd70039ff4ada3f81dbcdc8217c78cc1f0402602ea3206e591a6d2e9bc7b565 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1fe19919dd91f3e4cf3ce53454dbf0a15e8c27a9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ead2c90dad8827b72488e3bd8a2caaf33349c952255b3650198c4aed45265d1 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6922e53d8501fb72b55ca6a64058695c6f02086 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cee0363ce0df9610da45a14c173e8d2b108ad44a58aed598febc6aecb493e7d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c4fd8b60cdca3771e134ed0d256976c4882e5012 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3214d0ec68e968321aa88cce7ed95e2fa3fe3a5697178d9a3f63e7b962b0bf3 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf5ea975da1a72b3f59e434fe2b2f71bb377af63 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1ea63712395f4c0aa1f8801e69f5bc61656ef7b059f2636f0302f8720c2d9a4 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..598d415b7e683fc5b0386bc2b11a7a46bb93fe59 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c77a65588c996588808bcf0a05aca5f46aee144296704db8c6258e0e3134a6e9 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d07707ec6f383de18ba88aec81220fe896840256 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec11df16d2d8f3ca332c0fec17f47f43f46d055a7152ace20343324111b3b27c +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..33c4492b2c7c08c65937bd056236a620aca581d9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03e3863c6e9245f192695dc156f89a04bf0f56cb6b065385464a061e4e87dd83 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..28fc750e61e0ac3d2d9c5e3f0fe44d46f3879ce6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ffd55ee477e4f90b69012343d0a1e858732de022ff2d2ed1c95f98cccc14c84 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8fb47192c7945487fcfc604caa374a91870499cd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe82a4c399523fd72f3223d5c7e872a30b8978d3210fcdd1a285cc177c799de8 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..329f0958e09ca59bfa473be3594f1a55ea05ecc9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a59ed208c16de7d519647a8c5f85dc9417489f4a4b5a06a537dce1d021e99208 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..231025b43037eae566d2b7894f8987d5937cf30b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7eae2c292fc14b600cbcb07fee955457f6b2b7a5f9ab0edc0f902e779fe2862c +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6d96fa74558667f969601c1fb34dc6931b2a3349 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9367c3e5888d2beb06b2f4ca2e416685f50d97bb15d6dbf2a11dbeddae77d3be +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0eb0208b1333b5443b16b299195c3159c1ee8199 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd56da419accf4464e69a04e6a9f1718fe4910b8a6676cc9b1d6a49d0e785ac8 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6de67d68aa4a08b1227fb8547f56c2cb06594612 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d56c24ed28b836fd8dc385ff937b019cd9380476963d17c2b750d20c7efabe2a +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9fde161abbff75302afc70049e55c77423fb0fad --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbf8101c21d71d56d2df66230006d9307bfcb3e6113119350bd4c46ae6749ad8 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e4b0692e674ec62e31956f3579a536cfd88a2e75 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b61f5e41cc08051b94a231ab6ede4c61e07d8f1a5ce622c678a287b54216bbd +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f409514a8306914a154944d834bba35ac9f0144b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5fdeb4e0e6f8ea1fd81117ae6a479b2a8bdf72e853d96fc992a05d61fc659a1 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2474a9343b47ba7da01b87065bf2d8db9f7a9319 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:882d7a58e21c91addc9f0e578bfa02aa5a54fa2cf3c42a2d5b351e520eead3e5 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c9cd26b7f550e7b95c784b4e66a350031b54993 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df2d94f4fb50cfbba8edabfb31fb3b0921820674a6f8c92fba420f4cb0105b74 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f20b6eaf396ea5c0d92abeec47683c34e78ebd5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:641072292778037ee8c84a9c8c3b07a2623f6ef288b6460da3947d7e4170c463 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4a0026721cafe59ed23a5c35f27e81682d3fb09c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1eb5604343f9f46277459825bf7d9f3e995565da58e47a0f2baa434a13c024cc +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b49fb652ec8f834049aee936f62b23e1d34190a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e646c5aff1948356df1daaaa076da006e02f66ae46aed23910595fc96a71ac2 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..215f4b72e1b812eace6831628e7f903204cb005e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1df15d76e4e3146fe57615c4200199fc7c7004c9526bafed4779b466fdd47ee +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..68043ec7eb0e89bf6634950d0b53b4f5a64802ec --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a98dfaf3436424865263aacebeb913f61b6ce4973564d259fafb7cb681a045ca +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..64dd7673b52480813dee296f40e376ae1c6c498b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b03938245df1718eaea8b0b38ab6eb3c811d3e3b9ea1744a9be50b2a329835d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8732d5df738a7c9313bb5101c264e64a66cb2fb0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d3675773011aa35d1665b72670a628f550c6ff63a8b7e8cdd10ee2db5ca4583 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2419da0dd93109e5a584d99ecc99373ed974e336 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75e7418fd145f37723f61e0c0947061fe94ea68f5c43632998e6b4cfdb3128d4 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ab1a0c3e5145c66f88a2a5af6ceaddb8b4c4fe3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ed85e978ea044bf694eb2025427069c777c3afc4453cc7dad3dbabf72eb21ef +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6728050ab123c8a413122608bda8f9bd94ab376 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32387e8a6a72f994d3aa8428d777aa3299fcaf98ea2e4a56b2f56b5c452d2383 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e452fbc3bf0169e2bcabd34bb6492c2180eefbd3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06c9bf57437f5622f517d836d517de1db8953a4839956af5f75c59b7d7817103 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..993c89fb61f223cf9003aad49cd25df7c52b8cc9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67255354e24bd8cc305707a8a46ee456d8f130d8ac70d13598b961df2994aa54 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..85cbb5ef4fda46a3cf0c94f79c127cf493819c56 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f18b19a549ac126698afd2b609e7c5a8788df38bbe3640b8c5bcd42e61c2be73 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e85eef1ef96685105961d7d3fdf4cf65771b6f14 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d9f179629ae390b7770501ada002030e397d3223e208755d22b641e5a614535 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1cc92d1648bc71afdd59708382210d3502d617a2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99e5ef52e6faa572414bee5ce86383bc9fe6b2c5fae7ccfb08453676138b8f1a +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c20b1997f44ac50193168dcb95f7a6fa2689c39a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e97a62dca6bede2732cdebaa286da014e245ba7652e421b71655364c28888182 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f8227c501ff7d7655611d989118faa09d3726242 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a039fffa4c0cffb8586d2902cd5dceb2b9315fa82b620fd56e9938b244f48311 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0a2e094c55e10521c0e3b91f26c076c4c366f95c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0681243001687d0a3d347c6939da08e60cddf612da27579fb65587022005ef70 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..46e4f376af06c7a0af798d3c2bb42c9fb93e9ecb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4632d3ecab382610c9e1d4f3e77b27ba68d38a9c92dcf55d1d058432b4ccc6b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a1c8e3d814d026d1f2ffa0042c66d08d6acee76b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bce69e484be60476273d41ee6b3480947eb8feeb0fe5e22d30ad579a39353db5 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..12ab0e21431049ded756f0348a110da0482d008d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:640a9a7fc21fa3944a26e83b1d257826ae729f363d79d17126cd97a23fa4cae1 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ef22cf2db76aed711983cbeb45810e8226e7a6f4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79e4becaaed363ec77526fbc537003f338d73812be4a9580a8ec10dd7707b439 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..45926f246bfe4099eb7953ed0294e542756a623b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c052c40c4967d3d7fa3e61806f854ee67c2fef84748040506d1a9d553760bde +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a49615f159f00407fae0355d13fd81d2b0851891 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:959dae8fbd4a7fd54e4bd22f9e9a63a408c7841c73411822d4a783fc59290e87 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..817cf3d0223e0dfb76f66d5526264ee685ed0a7e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:028e31e4d8c7a414bed350dd399f71bad8e9b656aaa725a626c75932369b15ff +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..724eefffc056b158286b126ae568a4773dca226e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cc55550843539dbddf220b0018f0cf0d2a3c7adad24ddd9fbe4a3973da22dcb +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a3f1b8b838d0133f9bb3e857ada51c20aa1c66b0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:651c148fa6fdf35e963fd2814b5c961bc05d0ca909802d391dd7d09be3b2a18c +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c20cdc3a1962a62cc80c0ee023d1073d82e9e288 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2aa5e571e7ec2fc9e13fbcc4bc303afa30d13a60a16702ccc3de4afe1857ea24 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e10441275ad15a2285205cd9ca32baef6f81a4b4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dfde414f871ab59d54c1637962d7c0fab74358c01a6591b0b7b3194b6bd3b31 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0472998578a3b7487baa18b401bc64081620b2a8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75195384410d3d34d88a33cf9bada62b6ff8d3d081074b202c7bc1090a93c671 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8339b04672de73aab48f4922d333da8783caf574 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce32cba8d67a68514e25c74944cde0aac848e2d7908da42c1cad1ea45ec07d7c +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..63dad60d3632f22a1989abde50523c28ff44bd6a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:110535170c386ff490fabaea7f914158947910ad132ef5726054c6b89f833f42 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf1dcb6dcecc3041a67b986531139b5398a37443 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74b33ae5a3c87235504dfaf628bd34d456b2058864caf05b9600ba59a97c4023 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd8ca4eaf5812bc05182241f576c98051c8f01e7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0eac1a81865f698da3e2669b949cadb28fd662b589d41713b4bb20931b9ac12f +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0832fbe1d9ba3e2bcd161d3e07465796cb9bdfad --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c277d9d9f17800903df0bcce66ad70a6fbe6430bc573c51bb44937542396f8c0 +size 167872599 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff1869bc68be3fcf6aa0e41a467e50e7a2891383 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbf31b38e153b36546eaec8fb4f99c3d4db94b03faec9e617e8f6e0f11105205 +size 167872599 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..877bdb69068fc4b7ed733a17bd9599b9006a13de --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b15859f7f1deb1cd95d04d36cbe49840c5b0eb913d2664bd1193d3ac9caa448d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a4a28484c420028a85efa222974b60c0f5160248 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a98767bc2b6e2a59b2dc1f3bf763fded8d0684172300352919025c233a6a0ec2 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1cd027699c8fdd1b9e71f2a55feaa6a23e12365 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13096ca992e0ca39e65a8e4495e0bb95b6af795b3b81e0820ddb0a8a50c8466c +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..709df3f9bb149842a898c3732bc886439f4f2a54 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3bde1e26322269f4a4a6acb9c4a43d7072d763e6c606cf2c00c3ea4eb00c925 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a09afac74102ae5dcb23e274ce7ea6bcb390933d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9adbe4a92577ecc35e00263a50fe71b4483e9e747c5fcb18e0fcc1e7b3c96e9 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..adc4b2ba09e9ab3e641d3dc960acf4c6f17d89e7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:097bff416559a7a64c0c718ae23f5043275cf6a31ea61224c6d78278d92c1328 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e5f6c9fdb2376db2075db9cad086d3c786f15168 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39af67e81c4c1cd266c38c781c071e4c9ac38b9fda4b1155bd840e5f843ea3ca +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb3c7dc44e08060177bab3b8e51e53f0d50bf646 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0cc43597b6c13bc4f2ef1e5b61ff96f14069d02b8980036c489cf7186904fc4 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd7860ca0224d1fa4bd139da5211930ebdc11c12 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:309b7c6b44cf9502761548e88fa6b51e8b53d040c9d25484dbce4cf74f0d890e +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3c048d617720df64c20336b2a6824d31b69bd60 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b621fc96ec96034f6597bbf1c2230662de81cc840ba7005fb0a648658431d9f +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..34eb8f7a7d495c87186b6cac96a8450b19b0a671 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a40f271e89e982ff3f2a9fa61dc84641e9ae13becbeaaf95ed4b3d36714beb6 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d1407cae82831a9af5201df965cf5f3b767f158 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a253935224abd07e56226edf1a4ff3b5f24b13795c9bd7524ede0824fb48b62 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f8b6170519a754754dfdb1fcc40d7cbd0a4fd9d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:425478ed7418508dcea80be117cb16cc19438c9b188bdc07c917caef979fbabf +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..84eb406faf8bc44a71e31c7388aab6e43fee56d9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:895801bd83ea58de9ad7c0d15074e1db163db996df03abcbeeb022d964c85e0d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4cf5736d3fb259b7edb159715afaab9893b30866 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:667224873262aa64971c590f3ece02abaac5c7f9295eb78e171b4b6de353f7c1 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f7aacc1827981f236d68f27847e53e7f6ea83758 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f935270bed978048a38f4db494b5d37e42cd2576b9a4c3437b0d92b62426de2 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..743d7063989197b42384f2e615901455653ee097 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa2236f13a99103807fa082a80956dbd866173c568d08b997a7511414d76a44d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd3e8985f06b0bfbaddfcdf486effec26c534907 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d27f49491c85505ae9966be4dc9fd20d99c9d0bbe953cab937cb4f62600cb600 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..32be4ea91a38fd6a73e5496ed10ffb5c9ecd43a3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4664a5ed4a52d40cfc822c48be726f8ea2a169cd914a4b6e9f9e171d3f4bce16 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ffde0218b68869f50a1dafcbfdca5f469de4ab56 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e037b6404363a5e5f674e77e1e24253b31e87ae34f1980164985638c6ec60f3b +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dd75a8483020b9037c96e42c596d2fbaf4cc8f54 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea8fff06e4f1993cce152fb09d14e2938a4fc794d72fd21c3e51b0f1041f4213 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..28ade2d2060840db5e5e819641fd63c94e58d32e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05d8a3c3af33c731e262429ff13c58d8e98d88d3e09fc7714a1478b2750121b5 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d488a5c99415d87f6ab1ed18af05ec61a60f886 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e22f6bf9f80f40b056ce1a6959b460412f87047afdb3392b41d5ea0e7ad8e99 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b05de98b683cf4461f59c5e2b169c687bcca1233 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bf8199b0908195aa98dc7cba0f71f5dc2c323624a4a9cadc193371b43ccbaf8 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3bada34406b7b66a081d6ec6f9a8cbfec0f0e4ca --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8c50ac9b8ff43f3b0de5d7f0927df6172922367ea8d8e8bbb82be17e5c6b3bb +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b41f036e2e5b7e2768acc9c9b0108d3250b7a40 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12416c78c67131a57f4cbf72942d3a1ebad2aebddd608a059c4a3183840fb6d4 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b26169daf3ffe0c9857b9f9693f2cf5343422c16 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:090774df18bc72bfe4ee9c2760bb80bd89f659b2a73fa4f0cbb8b3202936e27d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..17d57df3d7956a93a23d5dfdb53be2cacc3e7b51 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63f069f5098ed33299a33aca8e2de136d662c77e440e638c274145d94f76e482 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..270874ca1474d612d2f5153458138df38a9edaf2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad2d019ab03be9a1c43c63deba013eb9e04627da0c01bf92a41e4be72c04751b +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f6c36e9a27d8fbb4fe0fa2021b213eae8f348d1c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64d7ee600db1a8dbf647335727859fdb6ac2dd1760283ef09f9c3f703405ea23 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..74b5935b0c609a61fe0a08fe1e6c64290bc2d72f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad3bea643feda193de92e4b696c1135f7f16df8835f4409d9a110b99d89184e3 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6a058cb756addd79d4c52f16943cc781ff62c62 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61a9ff8d45f9ff56f47ed78d6a9858eebac5b20b6475c49ec20e748d66aeb8d5 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..816116b87d00720181de1f8ddb0bb7b638c31e94 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2b3914a0436ee4b0d794ab06c3555d6d44bf2f8ca85243203cc588765dda34a +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bcbca318d93633ba4bf0d80cf14ed057ab244723 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:518fa1154266362e2643d04fae4575f70bd095360a2604a9937df26dbf18596d +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..548d7fd0270589f51f92b8cc77804b0847453c93 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f027444de59759d67d2d05b4af9aa0fba6faefa4cef1351eb23b2015cab444d2 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..93d2e50bb8f32d9a8e008c2007f0dfabd5b91a67 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aee1fcb2c765c3c4cfe28a1c68f4c4de47e6a1af366f5b1120a67be47f7f862e +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..98db15176407f70b1910533e17483888825aecd6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2a0dc35aa002280e4bb5fb1c3322e9d1b7fc998f6dd2761f5b729a6bfd7098f +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0846aa413647ffc834d1c1b9d4895e762518f261 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c06dd9f4f967ef0a150992998251e5817d5715249be635a37d5b057c62d256b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a0eda4e4334a62fb36d4881072cd389864a8200 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4dd12932252c7dfffec19d08d00e72688a3d6f48486df8a966e3fd14739b302 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b07358aebbb03ffa9b0bb8e1b56bf80c2c85bdc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0760e4c2589ee70848e052cb7df72916b4ca002aa9ac2a5f3842562695980a23 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..14b19d9e7cd6c298624f7b944309a58acea6761c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffab202a4a8f80aa672f73482d436ffefa4bc9741bfe4784de2fea3b1d9f5272 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a015b3799ca54a189c5e9cfed460b65080e0687c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32eef1bf1de2a550dfe3d721e32e83a87851ff083029452c412c32901aef2d1e +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6d5a84af50c16286eaff2ab95eb2607c3f8d4ec5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60d06be72f3874e3c0eddb83e3754163a6dc32bccc7c151a052095a640814a70 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d2d7022ccdd020315aa3cd6ea58053d0ad50a83d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f10f3539c539d12b7cc078c45bdc7e3b968c9b5cc8effbacb451f0f763ab97d +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..12411c0523bc74276e65d4baff20d89b57d1c99a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e98ae3ea7cf29b737438eab1a593ecde8b0dba5aa0fdb46dac6c544b3793230c +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..61f9349de1b870b971bc48a66f023e86287aab7a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:643c6b80dc2de433b282fb90fe6be46b5406964e2127cf0e68230cbb1ce92dca +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..74b7024d5e641b5db31de91d07f140d09f54849e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c57fdf0f44d6a27a784864ae7f64dd3f169e1bf4acea71a18f3c601eddbdc905 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..64a4d4f29d9b7a40ea71e0eff210f7981a9aae14 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3314665bfd2c994c57b7c456b89a8a4218bbd1e1e19ad6236eef968ee2780cef +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ce5b9c15ce27d393fb4e3c58421e892a5ec1b655 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b0c77d799b0a44089910e319e8ea0bb7f51c21979eb7b81a6bfe17a49f2a169 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..35f6dcf1cae8c86b7898abe7a2a9060a17a9350b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5769a8af10fb0c1431fc734790e8a96ddb9fd1a4bbe5f6af51ede958028f83dc +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6eace9dfd846682e0fa7d4de28d529b5a7b89128 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e30e2015e55062cf640b77a013a106a7acd4cf33ada11abdfd092098dce85938 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8962a123d879f2cb081b9e0e4c83c8eeb4732ca2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3db82c7acaa76245babd3b9f9baadca9cff488780cec6883945fa2aa3758d491 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..06326019e3898f83e10a434987182c5fd1678ec1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8560d1215f2d716ec89056496006602ddb334dd1cbc158b10c1e6a0108cfdcf3 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1b15e566fdbc78e3cded4666e8b84605cf883610 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c586004ba3c70baea839375a978286a6a0441c140351dbf5e945478ad2934afd +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..78177bd103651757311981041765deb578e993cf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0df04aa8462bb559757ae2b015a7d7ccf049bd527e9f1fd7ad728a3e44f48d0d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..661a1490e527c3b266b9994b4c3ccfbf815e26e1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d21586b2b67b4f6db42f89cd7156fc65ba273368c590896c5e6aed827f8aca2 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..54e59855fbcd99db59c30dc535195e728b8bfb2d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db1d77cba13433482779fee29fe7e7873c8876968aedf0bca932f700c440ab11 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ce971e1f554dae4bd15f205aa1c372401ac1d66 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c7ec136eb1c2085d26bdf394e6155b61ef66f799406a0107250be104190c96f +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..268397d5b7323650f9f2c1e9a6945bd7d821f72b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:660463ca18bd4f16b67c6a6ffe86c0156626657c461c963d17e633fea352d379 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c814c163d8d7e03f31c294c0da45995c299fb8f3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98d254c6d52970f7cc482b553630babc7a888c7144375f8e62ac0f870eecd61c +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dd48e4c5f8360ba6bf7980d690284edad7f1446d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48922c6d0f1e48e80b803a533badc4d015bfdb95693cb19a73deafa8f0d0edd9 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7cf8b4e9c9d80743d447a33276de302c9a91aec0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:129906e9e182358fefb300ed97d0132a464dbc0dcc4657d073398154ca6aed56 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3acf72b8040204ca2b6e4c759404aabeb06f7537 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a65e0d67170ba87ad71436e5dd3e98da38a7bad7659bdea2910bffc86bd9b25f +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e40b929b6beff141ca5b7d330ce4ad0b82484a54 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f4c49ffd894b6a7f6521c5ac92e98ebd80878e196ac459ba38fd0de94465055 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cea6ce579df8337b6910bcb6e6b3fb5ee0c51207 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0509924b3998ef9f6be9ea81ea35299570a6429fc3fec44bb29c510c3b3e236 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..655cb8b89baca11072f5d9f811b64f02e958bc64 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3dff1894e36ed211a6eeb34841340308b928fa5645a4d2d675e651089c847394 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_01-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ed321ccc1f3d9bb29dd1fcfc0302c80e9104a284 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d5567b9fc6c7ff64aa420b9a642d6cfc9001c51764616ba172cea998ee102dd +size 153552131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_01-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_01-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ecc90842bd368d594f6c10e6bbbd35d7370776a0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_01-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edffefe471dfb9ebf1e72c681326ca73b553d35be02f7f8e3b5769beb928b0cf +size 153552131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_03-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1121c86d00960e6518646360d728942e415d933d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2812421cc727f30f9165e10c594209b60d2909684cf3823dfe3b2dfe4d4809d +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_03-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_03-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a5e417671bb9e0ad4b088751d50f95219704591c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_03-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa3d498f6a1c25e74a2c0b6cba03a9d252b321393038deefc22f2f142f6b0a44 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_04-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..598136b3aa5941f66675824513e20c96d6b7e6ee --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0a9045a9c32c6fe1f471ccadef5e57d024fbeac4291caeb6630fac3da7b5f80 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_04-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_04-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f8b2fa998f85a398ab2fe8ee368e9413af2bbd2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_04-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1f9f79d02a354599f23a5dea4f4d8a20439b41c6305dd07596a1eb8c52d682f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_05-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..57350f917db4b33cf61df68142fa247c98f62d26 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:920c8959c51d00da66ffb71861c825dabab64d88020204090f93c0170f95c0e7 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_05-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_05-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f4cac3ebca615e3067d8eb281e2bd562eb34ffe6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_05-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5216d5d377548ca3aa713a1d13070aa23899dc8289877f968f13c910e15cfa52 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_06-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a9c15e64a5e3140a816cea6f1391e9e0f3f2e639 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e427abdf7762930beb95056583ac00ef0a91e4e7dc5e12d7f6e2dcf817ec5e5 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_06-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_06-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0dbd19d59b8fa787cbbf40878ad9cdb2b4ca90ef --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_06-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3b427ef647e965680f72e5e658ae2e555bac2f6bb2634d9cf363c0285744632 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_07-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d44ed8d1782b3f9992d9f5337052394ced92cba4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69de9381b9918b9808757c714d8d146d14ffce68d98406f20f1ec56273295a4c +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_07-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_07-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..26b8264d50e9b44098231060d807cc9140995400 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_07-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16aeee133dc76865140caa053b2ca74b889cec589e0a2df38d39e75d697481f2 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_08-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..72da89610d1a3f0d4a60fdb089d773d5a9b52619 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b38cc9bc1881c00a3c37cacb1a19bb8d39dc71771b3b6b81170fb05dcfc5e49 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_08-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_08-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4fcbd342487e1ad2bf28d53ba2980a80f3eff229 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_08-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9be738d294fa62be0fde4e3ccfbde354321856ba848b7c157d6ee34e44253e4 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_09-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f3babfde569ecc1908ce104c424e68ce9546cc05 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f157f1598ed16cfd0e9bdf7669ec8e96742df878675356edf7cc85269b9c9b7e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_09-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_09-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a85c1f371d43d77d4675b3280a1f001b0968de37 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_09-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f51db709e867c3fc9f9804eadbb95140a847fa4f819d8dfa320723bf0f01fdcf +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_10-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..31aa27dfadcee79e92bd21319dd3091caa1add14 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21fcb05eb93018c2f72ff26f570af6cffbb01e2f0bcf6759575b4d6f31661bb0 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_10-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_10-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..16956158ddf598288f49f363c0adc7e8af85b298 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_10-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a388149b0ad2a09d9737d93c1977c3fdc759df67693ca2c9a3942534d5935c9 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_11-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..92f92cd21baa525dfb4b2a2f950c9f4a3f54cf75 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f43c61bca892f740a24d1c2223194de0d10eecd211e336d18757b2394d7ad02 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_11-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_11-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..22987fbb5e009aafe8c921ba09d643190b68ddb7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_11-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96e3829caf56ab301b8deba531db6c6683c0fea5427501790ede17a4126cb663 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_12-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..edbab3b25d2b0c3f8eb66a5d3d64f17967c65922 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bf303e75d9f39c8555ce372fceed9ada11010d9f66366669634fbcf8dc61373 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_12-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_12-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f79cd5c3857c47aa4a534b249f651320fc6eb86 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_12-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:308a1bbda1e80ee283b4ca131adb98653550fbf7049ad40c9cd988ec913ebd4f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_13-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..453360bab99622395f87a13e12d35ea6a4284bf3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c243153254b12759fb25dc9bfade722b44f8cd88d998774314a409f60521cb87 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_13-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_13-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..874da399bf5efd8fdf9e86d930f89a88f883d75d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_13-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd302f935cef1c7397d2c8cb905a375ae6166ffee2f2a07b9ec3188a61944b75 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_14-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dde9d381db14fd80a1e4cd74eeb8d3a0ba068561 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae99fcf67ea7b70af3988493db1fb6c91a452d486fcfd4b945d8323bd18501d4 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_14-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_14-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f35e31764fb33b64db0e43c2dbf209edbac82bf7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_14-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:349bea48a583037177d26f89878a5da1b6cdba45a1f83d59418b41b82bf66496 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_15-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c1992df29b18c0c7d46466a3518416b357f1524f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a6dab050d1f7febb6f23fd7f5ddafa0b2cd336b0c9cb3375aef3ca39da65bdf +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_15-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_15-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..73f99d2520fc6a798145be134fb0bbc4553e31a8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_15-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d4e903f0893643ff30fca98785d8f7a29711c8842e10266bac690e87e2cd9e1 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_16-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c8c3f2d80b54bd7bb2bf42e7c9e579e1b44e5d1f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fed0ff484f7ac81259c6923f4e386a17bd60b7c0932331b92a59bd684b9e4d43 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_16-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_16-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9aefa75fe7c1abeccf4a9fadd85e6ad96d4826fd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_16-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f8392e321c50724959274fb5fdc19d204b79f69accee08d024866a1f8f25b0e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_17-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..25a6b7eb4560d7a15f04fa417032838594857166 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:168eb5e2a1949569c3a0841a096fced4d33546c4b5d3466c64b0fbc3a309fcdc +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_17-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_17-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ae88b9ce2f576a31a9ec57c0ea339cb616c328c0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_17-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad817c4ae014f451f0956b3b00bef83ad90eed10c4fec248c641f7a6e3edbd07 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_18-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7a2e956268263dab534cfc4e3467699d1f4d7f83 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef7cb01b943dd4532d78697f4b86a0565432764b010a0443e06be256149efc30 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_18-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_18-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..535a5458d9bde8c85e780eea59c49c038b009f6f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_18-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f97a52d9b767a44cd9d5a616b0709dbe2e75ca747a09229bbb79d115c0ea9b5b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_19-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..50bd2c2a5f178b7411e7b0a65b9e8cf9c8c84f8b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd484b5a65bd7b033c4cb87c3a8c37d81fd459e89980391e4ef288d1dd12d9b6 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_19-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_19-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a44e895139f7a64a4d6e0e29182d5adffab49df --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_19-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a550a9236fd11a755878d2f242e3d34b0a3a2d584535b6f24320d581e172abfa +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_20-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..29337416fbfcba76b9e3aab634c198b321db77bb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ddf89431a1ca97470417458b9ce462d31e14efa37ea5618f0e14a15a409f523 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_20-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_20-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9bb9f0815f3413d1d50de428ad12a679b4d911e4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_20-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6868eadb204cceecfda64651fdec34c90fd4970b5f23b362a5aa510d7e95c741 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_21-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f86930e58174037df000cbec46e0905de461dc5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0ee918d394b9f906b501933625118e3f820f152456771211f9b8a648d9a358e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_21-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_21-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c6342edb985b12f813a39aa8ef25a870da6e4dd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_21-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3bd9aca7f1468a419bad9c1877d38f237256879d65c8d0b85734e88eb758c81 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_22-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_22-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b0eee80795db4cbb5c413430d727b7c76d01caf4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_22-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba21a3d261de35397305faa3dd53c5d62b5edfaba40113607b1ed4a12c7624ba +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_22-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_22-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c0b6b508ab7c0c2dcb11d8dc483940781f3d5d3c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_22-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe5d4daec441bf190b1031b764e4f282f71aa46011248a9e60883a49faeb9ef9 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_23-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..38cffc2a00a405816bfdcb29f6582bcd1ddd8c3b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:568bd11c5ba384e0addd3f0e77cac32c48522509d1049587785341f53ff34d41 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_23-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_23-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0937c047e1ef6f6d1f809dda4fd0ae13d8777acf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_23-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52d55474e5d7f7e12c542d055e594fa4bd91ad862a538aae8f0c0d83ef0a8c12 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_24-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_24-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..52ece88d84460e5779404b44671d620c4f7b4abb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_24-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:514616c1aa3ff1029f0a343e765c5719ca95736d40b98e983aa58f12a53180e8 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_24-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_24-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..50a94af1f619f138b27e20af70bed4a5fa5d2f68 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_24-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5a919b56aefd3e98a92df08c242c791e041629b48f7248561759ee42b59a606 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_25-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_25-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..40affb04ae4f104e0d497a36a6a5e34593994c06 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_25-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:859fed2b65e532b854457b2b78745b43d7547ab14df454902e6321f7a63f79db +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_25-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_25-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec94dd9b7f477feb403081264f04214f5e6c887b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_25-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:990fa0f6d63667ad2da791745fa6a54b25ccd6f6dc15198494cea7f6fad63b3b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_26-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_26-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2b8df6f55fb5de674ef1d06f7fc341bca51814db --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_26-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:842be5a489cb8dc4fd93b5766bf99ae12daa7ea60d9d003f8a85a6febf2d1011 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_26-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_26-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2c0ee6419ba22e966b3546383eba902366ba25a8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_26-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0a31cc9e8042387d27b1d8ea8a7470fe0543ef07f51c2b443acc6dca733913c +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_27-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_27-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1a90ea4829c750876b644898ed2c6cffb6fa82a6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_27-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ce9849d2c0b440a089f8143f43bbe05c2fb431b1afe9c26c4791da5c642d7a0 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_27-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_27-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..014d8bd385e4ea4b99de965890b4e4ef0bd90b48 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_27-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:213e6c4d18c9b308503b6ab539ab389243031376079fc485da7d9b15d2834103 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_28-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_28-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5432936021e20f55abc2e33219109677d259839f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_28-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bf3a722db276292f1facad22060eca723bf17259dcb936afa92e228fb5ffa59 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_28-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_28-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a5566899da4412fde598430cfeb3c435cff9cf4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_28-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e670542cc90823c2263c5f8ba3064f6349cbf74e1293d7ada019c5c8bce08a2 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_29-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_29-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9bd40a028ccee707c88c0b787d4d17f8f5276c3c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_29-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76ee8ed769883c68bc716ebb15afacb17d7ce191cc1ada8306db33bc9b2ed836 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_29-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_29-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8db3bf6c6b0296f9b49ca32bda45d07aad2f422e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_29-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4f865d5cd6d2a7c0c2d2fbd6efdadaff43bcaddca9d5fd50eb48c953c901ef1 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_30-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_30-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..210d2010c3de924aa396f41a3b60b9875d1fa9b6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_30-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9591cd17b5ba553af3b2f4569073118fc649b5b66499b6b0bc7f49d1060c3927 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_30-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_30-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..003626c696a40516c7a7638dc8bb60fb9c24b214 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_30-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c704b41b7c310ea13890a5c75e290c52998e80e0b539ec073dd2e47b1752d0d +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_31-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_31-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..deec92168d25ab4e6d1cce45f5e7756c628338ed --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_31-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:227385b765cfe4538994bbb5fdff3d182b8720ed595e0a9a89d294fe0bc0a7f4 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_31-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_31-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6dacab656e4b6bfd87dc11818bc74ac777ac3b0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_31-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84eb0ff1ad0e1d7010a9a4e66b11291ac619ed3fecf4cc35e104612cc455c03e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_32-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_32-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ca641f0a02e793680ac1cc11d4acf790f57f4b87 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_32-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da6aac7e6e9907d1df44add6c3c6ccc25fee3d2cd9d47022f40fb69f7d0fca93 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_32-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_32-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0f9e6147b3f0b713426a8e94706b2e71f755ccab --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_32-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd9289b682f3b887bb5dab045ce68d5a8bcf7cb14e32ddb445d9858a3b7c24de +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_33-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_33-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..34cec1667bbce85e3fab837a6c8de65e8896fde6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_33-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d33d66817d46f8b6b36b3a5f1de321452a072f901573d2bcc7121d8da8463500 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_33-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_33-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3506b0e12ab6efad6660d2ceb985f575ccf8be30 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_33-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5c556241a06ce36435512302de454be97e4411ae09bba08179521fc8d0818e5 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_34-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_34-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..86cea4126afc3f744edd074415fe700b83a1ca7b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_34-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33d3d74393851a1b673c274c373fb19f60fa3bf74d6d023ce50bff38f430e927 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_34-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_34-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2316f3e00d622234808312dbc2e71481c282e28a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_34-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c15bbdb0665e7163d81abf42399569e74107e4abb76022e0c598b8c9c0e90cf9 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_35-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_35-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9291a0224cfed783741a4f929742d0161b75d8f9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_35-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22713a0672027b6e32bba46ec71b1d73e3272a11bcaad2cbdac6ff8872f4b3f0 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_35-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_35-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..59d8717342fc483a94d6888e914840f4552eef8b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_35-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31f0ab530ad3e50173e52542fee6fe2baa958625c44f39b6084ba71889a9ab25 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_36-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_36-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e65a6ae4133a46f13442b416d05dc5646a15f3aa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_36-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3652ac56a9ca96c6ba918088218f28391ce86e9443f51b43490eaada21adeb7a +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_36-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_36-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..695fa1b1c2f15d4774c4db5b039b17ab8d261e85 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_36-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf8709c6fa30cdb4b7bbf8d5e6b91deea1bc843bb447fa3c3434545419ddb9a8 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_37-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_37-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..25aa269f566fc78060ef075b603396879d81f88e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_37-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12b5cbd7f86c3b4871c5915dff680c5c24cea5913dea5e393d169f4ef112f731 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_37-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_37-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dd70dbbe9d38272e271f7da0a98946a91f530fdd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_37-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae63cee29872534965892280212cf3e6487847337a4adfe7fada84a3fbdf8823 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_38-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_38-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0cf59728d91d1433a377792a70aec912b88b873e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_38-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1a4684f15977bd84ddefb08a182a99325fb65fb4436678d651a0b66c1fe8ac3 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_38-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_38-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e9a1b042e0f7844fdd4adf5785beb6af0c688e41 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_38-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7431667a730270a82e7de555b6323106af2e07e05c1f01d6afcb0b37e7fcb2b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_40-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_40-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f438b441e953bd4d6667093a7d729b3ca8ffa55 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_40-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:928da4624ad6f78d5889c0227c872f4c557f2df0eb7fb96c31e703638005094f +size 12483 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_40-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_40-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f44b1136ad7857272e132c4797148c9193cb1a29 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/layer_40-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:490aca859e424bebd46ae2bc4d92e2acd6490563947038f1ac2d58eec0262ade +size 12483 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/mp_rank_00_model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..52ba0f9206034496ba1ec399e2902127e77ab26f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cf96db1e1cc9e38e4ad34fcd395d2c4dd7a8aae9dc13632561f3415890b6cf2 +size 51443 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/mp_rank_01_model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/mp_rank_01_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c5389693e9bf10a7d3c38b3c83983e88b6de62ce --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step40000/mp_rank_01_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03dc18e833df18e90fd3bcc183f038ad79c969b53e9b24285b0ed1ee349761a7 +size 51443 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ba538fcae7eff7495119fec492d4136eb6f6540a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed29ee1252527bcc13845fb296c4b86c82efb9d643fd26b06810d9dbded255f5 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..011c579e8ded66e263d0728d13dbc7d076adade6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58fb202eaacbf77b984f4893c5026122757853d01736558df2f7949c3101ee7b +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..316bf74ceddaab79c8b871fac14d3d648f83dec2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15cca00464290e13d53381ccc732d51e83505ca0b80902829d498bc145a5beef +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..91a1c52d2c9306d828a8fb2438ebd9a57c18c4c2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4fe21ae9353c7f26b9cc06fa3ab984b4759552b52cdcaf16538c15fc61e0c9c +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..06d3d3b5f4da5b61402fc87a0199ffe1cfb3866c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5bddc50b969a8a2e6008a42213aa07a681f8d12aebc9ea74faf944f06495810 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..999b0949066f94b1f5dfeffd4e8ee94b2642cb64 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c781d5bd10f97b760ac869035d31dd4a561fb24df3911eb276ae53449fcbd1b4 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7e34933c269c45e84c035d30eb37d70c97f85d5d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c08cc0e54d5771c211781f3e4f4d8157441fa6e3e378a441d269e976ba26a46 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa6f453e54d93f66d04613e7cc2678cee07af4c6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89a0ae15069513d26ddd078438848cc3ed215eb38582284318556a7debe25cd8 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5e36fa19651453441bbf6cfe61bfe0691af2ac9b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e8b020237d756cb431c20355f3008ab455269c79238b6bd904d3fac2b19ff0c +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..73a7437316654cb1f83669c41915b95ca4b76a3a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19c07afe97fcea60080069181093c012068b3fbd3a3b3dda3264c7c8732a38af +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d7340dc142c2c624e86257c27710ea67d59436c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01162f1e1f87b438ab84bad8e9d551873b47fa219e4bf02b3d5a318bd211ecca +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..023e050c41dfba18ac4645454e208c9ff6f236ed --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6109db4404c86f5a440d14aba553ec895b5b4e929fe1459cc44a4eefc4700563 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b3e7e034064dbc10e19f4cb216622562f085e22e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c9bc3253d7243c50d7e03b38f1533ca54db54499c723addc24389f81aed7fa5 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..582ac3909a7ad9c528130ba8baa9c7b1834e2b20 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a97fab30a21a6dcca58b54062d75a13894d27b93dfc3f21ef3a2a300b787af6c +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f40854c4d6bafbf437f7b27a70a7aeb68f5e8b02 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:985a5f949f615ae1d12c1b0bd3aa4a7888abfcb772286b8ccfb209881c080c4d +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b1fa6c1962a26ef37cde1c00b2124ec91597117a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cd5618c5f7215d37c05d21e19b21c35e6d02ae939d5995ec4b16ce1ad5c5dcf +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6c17ddd7235f80051d28718838a21e6cdc38a45b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b06fd67332eb8a551e47dc4b8eeea6a9358083e79c2bd86bfe7f6d26801de31 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad7d1023aa56d23d272f6f64ebd953e3366fff6d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d53ab1883590d41a3cfe4bb862089c4975d7528dcbdef40c19ac8b8cf4b06558 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bd0979be2d4b5ee21a8b52840e73b2ed3bfc3060 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41b5660a8f58810ca1bb7b8f52e0e30961221e8fda5fbd4cf10901f73af21843 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ea1527a9027197b488c651d5e63fcb39cfa9627 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70663743e2079ba1418649ad8bf84c27d36d81df9588ff89a54c7b1214912e5e +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6010c1f3063f03fa077de8d68ac242e8ed44d2e8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b79a530ba203b6dbc55c0b4f1043f776e758821814f668bcf14a0f27c7eac22 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..39bdb8b07ab580f0f17d86d6207a790f68d3d285 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d2136c9a43897984613e79ecab960f74158d05f3e7b335d7d459b140e389fe9 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e9c0f3c37470e9c61496867f570479938c37d38e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1dade36de957312f13755919c9c0fe4035427d9f8e07d19c0d6398cf54e737e +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..24259aba7c46189f1fdb0ae4d90f03f17279b63b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60f18ada0294566ff698c693b49c3be846c2e1e8ab8e0ee00a5517833c27a3ab +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..01a5bf779c2ca7b1c7c675aa0244f05a8212fd48 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5991f6a39fc951cc1a6babd5b66e80125dfc9a77721614e113a982c95a5f19dc +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..156662538d7fa510d33cca9281c54358c5c221a1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b80ebfa459425f3ccf7ec190d6b149c12069d9a50f9790748b43acf31eca76fa +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8bddb9ba77eaad8300eeb53eb392a1ddb42c6a81 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c33c9eea0687450a0efa873735e86c8f0adde9df76e4b96d67231fd034e85517 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..73c6a3d9bbc7482aeba3e02906bc89e3571609d5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8b23dc2b595becc9b642fcab67da5b498867a34789778d8b03b05b0891c19e9 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e44b719169edda854bf00cde7bcba2f51ba5b64b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:133cb6ae7e6d77f72ae43948d3be0eb79aa8682830f51d5e77f5a28e1cfda21d +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..54c27ddc75209afd0a8bf50a77b5f9eca3d02907 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f68cc212e97c184a12e3a0138776324294854834edf98e77224fb1a7f4537e8 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cfa80bdfd2b3b36bf90332946cf33709538a6eee --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54fc0e0718d65296aefe28e13135ab8c7aee43855321d932b62d350270d2e60e +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c915e43b4d79305ccf3dff943be771656b64fbb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41585494cfe80483798b1740cc8c9052d8d783af1be42a985ceae507babda136 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bd04c57676935b2df71146c96bc8dc570b1f838c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5796914c6392c83f841f7e95290f9671e131d122106b5f8c2cd36d3ae736f8fb +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eeeec5f0afc1ba88e81f86108c9ab7bf8fdcd55e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d79496186719dbbf03f59d75ce6864da3cefc524cdabb3be898f01e7f8d16cc +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7073610dc167ea8c4627a98fe2ef7b7a171cb601 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f041cbb84ec7ef015f1b4e3452889c4e2096a4869bf3083e98e2decde54298d +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..52bec82e3a51072c273778da49a93b5b0ea6a50e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5343abb8c4c037554b269af2b0abd70a457da7cdbedf6860bb7b607e326fbe43 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e5f9418be768d5bc89d1b0c39a564de4f0f97c87 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78bdd1ebe65ada33eba6da517a6e9a6c4fc547a6059f106fc449b26f09812712 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6317079366bddfba871b630bd5700ae1150b74e8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a07e8672942258cfa86238875039544532c3de135982675d4568dcb071f7225f +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5db5b4406825981a8c50c4907a1a7fd2855c1341 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdae5986f852b1b6ceea36e23c797f0e780668d42fde909aac633903c38fd109 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f14ad20f661192274e8f646a83fd5540ee6843b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90d816192a48e4ca4b6e0003c992bb1cbbfd50ba3044f7e927008c99e4ce8550 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..97f5dfc632dce582188afc42c6336b97d75817e7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca44f4d03c74d5af7b42bb1649ba47943fa0366bcbf9520e85d9031d1de9df3e +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b478ac85690071e474c3b0c60e7f3ecc640d4854 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ce4415e1e7d8e7a35eb467c6dfd17662c074ede10a81a9046cc762a0f685988 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..65a60de765566d1b26629c2ed0c3c7dae07cf8f9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7492da46c134cdfde2862c0419f2f485d98e4252d2a811c4a49ab3e1d2619b6c +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b9b197afaa799afe52ee7a51a07790d86d268df --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0807824feb68dd141d7a4f3e51cf2b72aeb0f6a47d8aa97d79d451ca1ac44352 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c7df2e885e0be6941f33af033be40bd0576fb2a1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d02db9565b6257c660cfaa3e04aeb1c29dbe44aa4abce6556fab905fbddf9816 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c4404f1e80e1bde76c12443c55937496dd7029e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b691cab68da21fefa8f77d938dd5822c822e810a773b064cb80f473749f0bae +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..db02bbb190406cf7cedc78e266bfc213cab63591 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:117f3899a2f32bbb15cbfe98f8383f77618d4cedef44a2206fb0162ae58eda3e +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d4e8f2ee26afae1aa87e3dc70f6415f415d4503 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:046b67b3e8363fcdd0ce3d285bf7a1ce1ad208796230e7974fa0a7588241f890 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1542649211117e2d030fc1d384608d4b343cdf9d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e510921c5b73f21ecf23ac5b08cfa0bab2d4b756b87dc5fa783528a2ed3d7460 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d5f88d5194adc6b2e0c2cbb94ebbb5f1c411d6c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43ab8eefde8c5c4315eaacf207d879853375ad7dd960081307a7874b1bf5057a +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..13c40eb944cfc60277c1cfb7319ea7c5687fdd45 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8f4e5016fc5272515f16e97cdf5f5650a6dff7ba3c3061ac7b586546dbafb8c +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c7c6df87fd904825e4897eb45970c6d319c05309 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa531dbdd0603e71e924c168117b56a321eacfc782ee2b401a264ee645fc453b +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa5c1d05e518c2f9e738aa427c9b7efe86235c56 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b0a36e356024caa64da07d20aedf8007ca5bddc9b9b52109d9ee8f2d4f92ea2 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..62cdd305bc0509dfcd08e567d68dc5d2b0898965 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b88516d16508fa6d2c5f0405c07fa9413547392b815d58a47a0eb25b93ebd2b0 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..302e3c77ad21c9ec4219244ae1fa3a6392eda3af --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d679c432748e0021eb6f8a5cb1c361f33f2c4957f636dee6b0de3ef8272aaa1 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..09dfb65c24730f8c484ef88cc5cda8e177e7d1b5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6d84f3311c5991e68ba14d9021b3facc7d5137a4fbfe17235875c525038dda6 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2b345c5fc37b11d016fac574c45195e14af1796 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20f556050d5184306e25a3d334e2f331ed33f3675328fe5ec67d7fbecbc2f984 +size 167872941 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..97a9ec016e858dd8f60cb815feeb0d9f2cdff209 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:076c0c2f468cd0600287e2085dcdd9c42614a8d1937c6160dbea8ec92f861a02 +size 167872941 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..68f4cb8af9a99cf46c9c392ef9fcbd0eb2cd82ac --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b204b4606051476e20bf35185f07a876861bf36b3b739c16ca97bb7d3cdc9c13 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..957588c5f596ff0ce9517258a892c6a9d170e79a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61703a86900cca5952f84eef5a3f1b614efe861e16682e08045550dc17b2d922 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ae426670aea58327a625714bef0d5846a405fb4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c34bb9d420a444dfe9997ed2a89c490337c60951748b3d558d7ad6b92839036 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..70be1e72b6240db24f216956919137bfb97304b0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57b3de378a5eeafd14630e0f903cd3d8083df5e6b1bb0ad43a98d6fd549440cc +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..00694ea1bc7cb54e0eef19a7bce6b5e0331d6b9e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf5b892153bac979eb8a1dae99c6e4fa149382e62ab88997c8978a04e089da72 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..44038d8d4be15bf8cc2944628132758b5c7ed142 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efe7bbeb2cf2a3beb787fbd261b15619a1718f86f0c061a7babc24a5b9092632 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5a7293d7bdf94badcde010dd3e84a7bf9bca83a7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40b1994cab79a5fbd97275886d412687891cf29df590c2e396e85ae4476759b8 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e5bf7884fe63af06023e9465588ef627cb72869c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35cccbfc608aa1044d89239f780b2d667bee7fde41060299eb5f8ad0628b82bf +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..526f2d3b7dfca66031bd2c80ca920b1ef8bb1647 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78301a89fbf5103d67c8ecc37240e3bc507e87cc87600574f3de7d1b1f685876 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b90d18e29315be19cef503aef48a41508bc635ac --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41a053b6a81edb8fac9082a71b60a05044eb19f4d6f56ffbeb7e8fd86e0bec1b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c651c6425d282f8fc640260b2005f2330a543dd9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2436456c631296e4d62e6cef1b4dd19dcf85494b674b29f3209813309d0c315b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..021a72200f9ea920a56616e5568b5605704e03b3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5176dffda329987ba64bc97c575d85478a497e3a62141f68baf493f6583a34b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3295086d65023280890845d945a9b042b64db7b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb5a100037604f4cf60291760eb9636cea487e3cfe556bf414d884a8627d6ef2 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa300e2a76d07299d7c2991160d32602298bc049 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83243aaeefae7c9f462c42c78de3a8c112251a032ff6af2d0117317626ad332f +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..206857d4bc94dab3d235130b675e723592e7a98d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8843a29788d38ed33e9d2abaeb1f1498b897d9b354bda4966540dd076a5ab44 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f9f8e373365b54723ca69f12cb652200c5236629 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c96f067d9b6923702dbfbeaf8b2ce41de12fa1f2087423e69a7f8c1031bccf9c +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9302c7c25c6a8a8df6de9db894b67a402d649823 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48c262e969d35d3c3fa043d17b256bd95c28d2844ac5321b430a7b4fe535707b +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..05b6bf226281ac1774bf3fcc1645633303cbaf00 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d2df0b095c3bb3ba82754709782a89f7601db0997ef80ac9a52f6734038b48d +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..78868a9d55717eb756fca7a38f20d2d99dc33651 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d351054d85774b23644a320745eca8468623215354c3a5a873a41b6624a25a12 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..235fd6f7939e8ba7a5fbaa8b91ceb9bea860c14e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b17aaa000a0642d68f979464e231c1d8840520a73892b6cba924bfb253e168fe +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..91453e0c7588dbd33ab3bd1b3ce4b21ea1704aba --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bff7bb25a3b4e999bab2d239f17c6627143790ef5b2edd0f9aa3ea868bc2fba +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..adc35e4db5d869aa4d51992e427b830d26ecd5aa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f38f995bf9e022c53ecd63c778af41f1bdc9cd17141764e18814eeced6c35ef +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d659526e9d007995c6427af0e5fbc8eea20a825 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b5a00589187d10c2bccb61cdd8cf9a81be093c707ce0fcc1fe7f0838ca41ebe +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d0463dbbe75155d0aab94eb50528a442c02c7fc9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ead5a4050c6030aafb235d5bc4fde18c28744e66ec15b3bb5bdd34de06083b1 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c271da72776adcbceb83aa94f2958445d729c53c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0101b06324345d3abc507f88e06dd69e41190641f97ee1f242333ad339e8ac73 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..88cdaf2f0b8bd2f75306fcc9307460f726dab074 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b068af688f348a19fc1d36200e563a63e59996c17639aaf4a13c2269a9b3630 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec724ce0605ce4a1da9bc83b78d1926cefee0f6e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:710b6b929b5893d21e65747997e7b183847f71367d1ecfc79f80031d1e19c28a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8155e60c2ed1ae74532b6ae6b9a9ae60d4caea9b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b4b42980c9956ea74b26284b885761143166225c779a914e179382c639eace4 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc69ae49fee92ed459842ce27fa0c5431499dcd4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cbcfa41fa0c5fb1920fa94270c4a175936957b4bf5f46f1ee7814f8a7cd34ef +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bbb2807a1a667e2527fb19bb00fadd108ecf02a7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56345493d47620eefe9577dc1ac401f070079820b5889a3b2fedc80115e74952 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..507a6f960d307722c5bd4fe7c87829a63eb80120 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b38be83e2a8469004aef07f621aceb7a442404fc8f9a2570b7eaf48e686c2f6 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0a00e4b1408c1af9ffa648d265c6da07700efc4d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:324d941f0caac3a8b797ea84bcf3882ac37a5b297bd10efd03102d58ffabcf9b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e256fe7bacc11ed0ff32bdb67ac3d23a7e9b350a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32cdb3c3c1cfd5d1262b89d1c3ffec985e4cc8bce5fa0b23f8a485cca5fce4c5 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f0d86104dcc31fa471a8634489ede7e7afcc1d9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0ac6599cfc533642c382bb00598095a269e0707901b07b9772d7f6541ea9410 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..29db5c8c23365f21070bba1f71cf85585b97892c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97aa32685902ca7e1add5f2c06787f5b9bbee01917285a1bc7dc9fabc0445f90 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d8bad1eac54ea3a5b3309a5b14e791be2d6a4a9b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a397947b6837676cf91f03fe2e2fc2bd2b48426b23087e93beed95a6c4f6c3ac +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cac4315c865325592458405c264d3096f9cf45b9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb39966602cdf076e0ce2626936bf01563310e53ec9cfdc6e4566844db4e68ca +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6640d83cedd2cc97a44c2324f70736ee720cf0c5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c62c08dbb693a373df5a488584e0351a28b06385950aa833354ff7dad91f6600 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8e779650c91e9b5b34c8ffdb6d9093ed04543dd4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91a534cd2049215aa71cc9b8ffeb2515ce42a44d2ffec67ecb95e8f56792af77 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6539388adeb78f35dc7719f95e7ff76bdc61d372 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de4ee893d4fdd76a18ca517927f014c97d65d2ad6578b70bfaea039ce1fd42bc +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d6f909afe3faa2cd6b8b09de336a420342e35e09 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82697d5da4d40d6eeea0791d7209ac65e5b6235e6f447ea34dd464626944a6c9 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..50612342c7cd04106733115c718b98829e074a07 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f6385571b35229482067abfa41edff75636696d016aecf9814ee9a3c5a0140d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..86efe6b922ea7a6fe1133427fb5a7a7a88a035b9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ada7e9f40d07c6d129c5d7dfb7d804a928eae470848852bdf97bc9d29e5cdd0 +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d17b2cd7ab9fb189dd52ab8f0d9dc12da6c1107e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95520dd3a6bbf81fdc5044f22c2292ac34168ab4c65164964abbc6eef016609b +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a845c679a1f7f88b016f0e86d46a69201beb3a0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:858b6385f7c843731ccb9ac852bfbb8c5cdc6f3108698ebc83da28ec48c1714f +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8d10b334e0c8df9dc4dada166ca3c83cbaaa37d5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64f62d586e2ab8ad3d81c96065a568346a69de2f0cb36b3c420263d7cc625c2f +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..13ed155d557c4524b40c94cae02e3dccbf3a6cb4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70f60a84d703176654eba8dfb1d9650ce38e515804942e15aa7fb3f0b61ad8e0 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5280131802002de126b2a96746a420dc6c2a1512 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:379087eace8735f6f87b08041f0d2b226e8a0cc523c75ef050ba6ef8485866a3 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4d7736f3aa8d063104e0633bd216db07fbd8b4e9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a97967cd034d306898795a015e53f5dfa025e36b7fe4643285aa6837f1819269 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5083f5918a5ea0efbaa0261aaf0035c0bfc618af --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18ce9b80a908c63e94a1f9ed76262058a8ef2d10942946fa12773aecbf4798e9 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d3517bdbdb45ad891b11bd0854b2021cc0b95a21 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c69e913d4d928d0b43e6ff7617d4b4a3a1f9fa4a7cb09431d0e8d7af581ae9d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ee7a5649916c5cabe3c1f72d328855157a835e4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ceb34cf5fde9c6206ba0c89c7e069e817594e3605d9a5e93084715eef98ba3b +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cb2fae72dd9367cf40edbaa4bb224fea45d8f22e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ff6877c2407e7e0e1261efd4129512fed77bb5f2d1d4995e52d7e51ef5314e4 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff2771ab788b57404b923c3f78ab2db2b6c1a956 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c744ff437fb8af12cdf66f0667c76c18dce9f8d1a01e020c34e6466fd5cf811 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0fbcf9522cb17c982abda57efc64a916191d3210 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65f720611326359f08e4e9450479fb403a1e35789449ebe7dce544b3c731fbd4 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eaa4e8cf12987450d516262a8ef9e7e668a7d47a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5241b9918ab7ee90540d80b92740e820c56b1dc7c80295676dc4f577c69c903b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6e86f42275fb5ed5fb0e7d270856fdbe47a7c33 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ad506c1d2f3eb6bd14bab7a2378756677a46d3a2c546bdb32becb0306830f7 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..92f1310047ec96b3eeca4d3ee70b6491997dd641 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44399cacd64dc83eb23160b58cfa94652e1e754832523cf56e770e3550b3d2d4 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7fac459bfc8ea3fb1ed93949878209983037011b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc3eb8c054bbf208b4e1c4254dd8f45f3e460cd67e09d384886014a4cf8c7a4b +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d43f2ba996c001120e64d57a40f2138d440735c7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a94a13b244630aa02baa3193dc2e76219c2e62dcf2498cbec447f57d7b7e5e53 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..303bedee17d69f41a3c4183f6453d17209a8333e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e67faf6553fb5cb75303442ff10f3265c64fb662741b1b8c7653846b2ccac85 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..59a4c0ebb621613f329786f005df7b8d4df681a8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:904f8bd953172f48dcff34cec3176868795a2313a6bb22a7b99b76a062cd7c28 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..69d6dac1b6399b94f0d60898869ead10c965c21c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e0a70d3da5b893cc0f75e89ec2cb8beb0a6a9d1e7bd4541b97e78c7c7495b84 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..31d7d20157f6a7ed829aa7b7605970ea0db8f5ba --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfa1b94668e9bc3dc74e4894b5e49a3261825a1c2f3717556e4c84a31f6ca073 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2b179c201489dcbedce2110f662f561fa5942809 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ac33f6b185057b0b1d6edc422385901f412741e655514757042b6bb9310f5b6 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c27d64de3359576665d601b860e1a6a291750920 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b02e1b20a46b67ec3079e1244f0e0bf4e05ae1dcbd826aa1fb72c9ec746110f1 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..59fe352953fb29faef5a14834db43296fa7d910b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22c5917ed291d703f6468565626a46c99b93cc45dce2c10b0e9bf1c102b929b4 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c540d63458bb67d9f15d9e4a89927d3bf8a81fc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3a43221ccffe5872c48dc9c94bbbb956ff916c162dcae29112636470f4767c3 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1342fa6589dbde2295a434f3e5ef182d92474247 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86469a49909f9eebc7d3a7f1b6bd32bd04e76b5e133a79c95821356ed534d42a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1378d38547fde84ab00d4c96e16fc4748df34b62 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2841bf3282231a33fd127d0af33cde39fe41327f7710cd5aabccce0235711573 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a7dda0be6a328d41ed0688258859a4b724862030 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:325274b67b642532ec2ffb19c3b675d667c39218f3174defd5a5b8162794f7f7 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..92b6080e65807e1cf1fefd8f8b7f7a9cd2fc4d1b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8faa609218852df75fc8985cec7cc962c8941edb69727e1b21efc6c31d3c7e6b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c46fbfddc8fd09080e4f20112a63c00f9d6c0a0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e74bb4d87f919a4aaa7d2ea61174b7c4ed898e7c925b806f00b450c3785dfb60 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ac3df0c4d9e21a2825f5d3bf10891eb6b2b9fe24 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78614f8f0eb9ef4f01133bd37382a73724254a37678f651acb904a346c41593d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bde0e9f4f212e38f0077f364d98a3d629dda2154 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:494d1060a8e9c7de2b20f45f07dab9fd2fdd4b1f9ea67c266f0a16fb21c05f78 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f6e3cabbc8a7e32ab9e4296faa0d0b6a2fd9833a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13d9b29be39fb5fa5ce8b37104970bdad6232f5d7c6d33f29f93e0d706a279eb +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..caefd4f88ce839bf3886f06420c0791b9f2c0be5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76ad8862cb2d791dc50072e1a9dfff4645b1506dee99e28a851b5cebc897e281 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9606043cfe0e8d256613d2930bc18ab0ced8b4cd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8848dac9702f63db82fd02d24e03016add3dda9495f3bfd8abd72debe0c3c859 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1d84a24a0e451c322f93d3a0262ff7b5c52c338e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92d8240852f096b21a120e48c4b7dc8dcb98971d4435d7da14706e613e9407cf +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a50e21d3bcd9c2ec729d25c2d90cec88242315d9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b61632c41ae9c64afd632c787faae567996fbb2a9831ddbd646bf771d5e5ba25 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..79a39eeaba2939ed101216919fca79a55d7b1a54 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c35cbdc9cd2eaf1fa177cbdc1b1c43d2aa97c24a6c7db8e06a0bb52827917fe9 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..084f890703af392ccb6ff3ae479399b6f189ba34 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81e20dede81c58e5032f8c5f5f7f1627398159bd5ea0a40b02a28009ebd94388 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..19118623c519fd4c5c2699514fe216bd092a79f3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f5fd7d7db7f9efae1198e12d78b618d4b125fe1bbb6c4630aa7e570c6bef62b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..468adee399c49f8fcbaf7177bd7ad7badb306614 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3756a9b5b82c41f55f4d9814da9aca359909074385bf1422a6832b87dc17550e +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6ca510b177b2362b1fed33562841242a60d099c3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c61263f85aea06de17390c0a98979c7d7108ef951eee9b9e5637513aa7fc0796 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..516ecc4820dc486fd13a77cf6a0b8ac35eca0916 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a226a01fa2ac3af532bb638bfc67ac9f9a301e633c08fa976b3819459e755555 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0a16d90f8816cf2273346503fb58aa3f8f7eb179 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9a7822a48944db08e5f6d799902248c5cb6d6d30395d4b462d4e99d91768850 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6d0debb1b1b74a93517578ce03f2c2f5dc942a4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f21416fb9c89e8b37dbde5c9fa6ae5aa681eacd050fa8afa1dd23fa60085723 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..be12150ec6a91e93894dfd9305ca33397a6a0147 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f4ca684ab5ed02b6381a8f6a7b9405aaf71cce2cdd87ac223c77a370d5d2ee5 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bb234cea406cb7df15e4d85634a77dd6827acddd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc5fe7fc2ab3acd7516c821ab82dde50b11601da816b479cd19a0a516c4159da +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..24e788949596352957c1d5ec554af3fbdea5a8aa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd7923fbef0854d64faa3ec010e1f09aec443a6883f0b3b1745bb31d0bbdbfda +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..494f1631f31df130cde0c113f1c76b5c09d2fc37 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11fbc1d7f2f47408cb2c4d09f4f11d61744503dca793cdcabe9bd3d831171f93 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..64bf38527c486eb6a8f0c0af0ed19c04cd1f1669 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7be888bfee4db608ff66054c2fe40a5e5d122ad1057700bfd3527c3ddc1138e +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fc3bb592eedf74cdedc8d78a663b9a8b7bf565a7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02eb86402e57a9276e8b51e4b0b960e466dfcaf33bcfee21d07410ec68f97e51 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2777f8952094fa801c23067b0ceb277e0eacefb2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:159d2256181c728e69435ba276e78417c3f711590628cee0891b614df7360c74 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..67aba48ada3231fa575b789bebbd564cec06c3bc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:153592c5c2ff6cc77565f8f7827c3637503743334198e9c533048c5ae4c4b1c5 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..88c87740c37861b08b61d7d5ece46ed918660c29 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af0485232c3556d7fd3992f4d6ff6c81bf14cf14c32fbad988eab61de727b662 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f048a439328511d0affa4c062b993fc1f7b99303 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9e70bfee0dd96648ce6ce12281dc63d4f2f83aa6fb4472f19cda5df404ace14 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e55cf44bfccfc40a18d3936906cd84e59a98f8a6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55acf03bf4bc73964f67d066565f5c3d21821b2ed52ce1a8eeecb59c0185bb95 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..017216e2d85c326af39f520b7226e5137543c4da --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a20f36e56c9009dc10de531f6fb25650be021f11af90a6c95dad75424880ae4 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..015a23e00a6d3586dd198e8f1bb4720b93b4623f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae36ab47915b80adaa4c8894fd9b20c18649008604b2f09f687b153358809b7f +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2d6200a682d999cb0ad7d8ed0eb13f005054289 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9be9662b16d097e55878f46aa930d051ac0cd131cbb28577671aa810b2300b0d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b650fb5271115b196c4682148c02f6132b97ab23 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94e43ffd600d8095511c1b03eadf096e74093324fd0faef463e0628c3f9659ec +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..848c5bfa3734f95fcb2d207493d826326157c2ac --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69d0fc356e15230386f4202d14310164fe585c855649fde6e623934df42b98dc +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3d57bfb93e681862229da48585abbe10a5d4a0bf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bd6e860a98f059b96d03a1b5a8a357b0e42f115440ed06e86b254613fdd1c3c +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bc1b91c28206957bcf4906531bd8cf597789d459 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4c580dcc8ef8f86a7c20dfab310d726dccf7b16908cd156fc8601931e6deae5 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..631aa63f7a58db4f82b197f2e7ac9420e610b71e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f46dd0e70a141bb826d3585d4b1fe44c5f1f6ccd83d73a9276fd18a098b70b90 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c1ad858308b60929a38bb750dd19baa76da7aa73 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8b7383f40f5191476ac538fa999d56a9d957cc27a358b286fe59785bacc030e +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5e9567cd8b29726341b7372e43f16b922479d15a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d30a634ab3eed716071cb838f7052ce8f0da16e775aa6ad6c75b3948b70bd42a +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..868bb2af9fa1d11099d8d1aded492d0c01bc6079 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:364dddcbdc2078a25058b260783415aaef8c891eae7d6c162666c7e33f89ef48 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..80f039bda0e05919f519e28d68af58e24cba2b54 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53f92f1e213a6c21605b4161fe0c7ab69daab8dca776b8f25fb0059691d48a20 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f15ffd705c555b1e1c3ddb14a58d93280c32670 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7763bad52f8498f979a9ad0ecf398879e0e8bc0fe7df8c8577c256fe17f0517d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..488184c97672fcc4da137ba1b7bcc228375f37ed --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48e3b8aa12ee78e17dda961acf744191be2838db9d93359c8c0958781d3960e7 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..01711851425b80fe256a2a4ab6d08cc9186efd13 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:677860da63264adea3bee2158885aaa9f0e16a65fd711e3a516b1f5a33bfe63e +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..33b48107b62a5bb09c7d50687c6874e052430db2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8f0370b4d075c2322e204820345fef9e937cc93f0f1aad97051627d8d4ae4c7 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e58a7b122fe2bdf8619021476037d2a8884ecee4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:545b80aca62583a77df5336e2170b3b5944322da7c0b086c1b757e38fe9cfbb0 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2986458421405e67b3b8f35e7fa80acfdc38f17d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07e7dbbbbb954653b0371570e2e353286f0985c7079a10c81d2a2dcafc0a4aef +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..65f16a2682349fdee7a0cc1ee75b0a8bb5b4a15d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b688a824d0213e2c6ac2d51299457bfdff1ccee59e697b0f1c50c80db430671d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..17e48db8b3ac58dd43f63c1bcab606ea7bb42a43 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ce4dae64664c4814d4e430dc54dc70ea7884e86e5ce9f62a89c78e2ac3fbb3e +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..076ee611cc5d28236a418d288e4c0d3aee5827a0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d06c893eab0fa7174d8f65415c317f1e20fbe2fb19d1d73405d945c954ae567 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..61207e80bd00d2112c7d8bb9f0e4e5057a17f6ce --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46b806b935ca8ff12b0294262dacff1e8056a849afa3944f30ce4215b721c8da +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..81afe1dca6e76449cb08efc68ff3a4872a7a5e91 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53b16734de92d839e154163e1e83dea4aed29cafc3a3e30373d5205141c19a33 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c25df43f4510e408678d88359d20995fab6e9cbe --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d39cb8403ce82d07ecca0d3285102b09e34a05e31735f20d13f6b16c4dbcebac +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..74470923ac2df966ad97d7a4c4aaccf4d6d83833 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0af2eb04e71e66911ac9114458a4707a4771432155ba2de30408c1ffa5e7bc50 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d6154d6f2fbce0e3c4af33753c65b941c0bc9c6e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d44ab8531f09633a1585a72bc2de7612b48b25915978ce5e4f6d98b4e892692 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..05315d07aa83334aa5621d48ea4fd75f17d2d61b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71b9fa098c1a12a475a8c2e347c21a91d99df3ef085c27ba77934895524e0280 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..17daa506ee160cbb09c28d61b15e5524b18538e7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bd62a081854aa63138d1b1ab22648424c5035a97910fbca57074dbf76474c38 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..04d60e42156d972538a131cac5a51625019033ce --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39be9aac543bfb874c4755d6214dcf9835e847ed6896b1154e7a8a8b5292d067 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..da4539011dff653c5045ec5005e1b95a4a9dda82 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9a68ad7295c5b6ffbebea89b498a0a59a53792c07ccf81e30cdac985bf30931 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e92845fa91cc53011a1d924ca46b48d56f57443 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5cd6c5efcf06236642028537957b4413188f9ec26a7433f68016884380fca36 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c20c1153613192ea8e6b48c35c5702c1ac9e6996 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d24820d21c3776e678e41b8ad3dc439cb95609b21379455a55bdc2fdfbd375d7 +size 167872599 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c24b9aa5ed852f5067310f0d5c8205ba8dd656f2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ffc3201f3a168bc52dc0c8b9708b5ff23bed69b0f9487dd965fa0e32b1109ee +size 167872599 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a8cd1368c6493d8952d2e08c0b62b007ae373b17 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccbc69987090cdb2e4c4df70eee26824eacd7bed84924f3f6625eec40adc722f +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d8f7c041e2b929f815db844e224cd280f30c0e31 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9507386533d6a0c992358bbd3c8fc9c9ac378a66d74e16e07734cd567e32467e +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ba38fe400702687553d8d5cc162960622afb5e8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd6be969d721429f3fff882c912ca8bc03d99742118dd59cf4e944f29cfce496 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a120f7d6568c4bac4e872eeaf4f3381af2b01607 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dc22ca1048a82766ff68bbe274a12b3a07fd05b600d53e1f6a6d6798c39740c +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a497744b2e8d919dc18dfc74177900af21485d8e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd3f8d66bb51e69fdf3ab480638592b83285f019e8c45f479abad012d1e257dc +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..db6c090b850a67099ee8aa0165e27426745e1672 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8372daa00028f08eaa893b8bcf0150b15ac832a9a4f2eec46a9b7376bf74d6e +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..752e340fb05416352db36419d5be68f405ea921a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b852de6331067864f09c8a6c92ea43dc34b34fb41bc7baac1d2a668fa9225bb +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..54c557d942a0154e6de3a9dc90e0e0f6069d04e8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4140abdfcc014a41f1313f8480e5416ff9a99332e66f438f66d00a6f7c8025c8 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1654b988c9dbe7619392ce6dd813fa39df15eda3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98d056ef887156caaaddcc93200693d5523a7960aeb0342167564667fb214849 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc2c277497ed18db23cc56eef2ab0184dea9465e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40f672d3e4893f63a175cdff1218a3b38a615a4484733383a1065b38c6738121 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..286b1829a57609aadfdebd0f2dd5e06f78707092 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0775b164c588da380bc8443c5f5cf21b04fd99dfe67bd406689d3912d7c256f +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b4bc830d45a9acacf16ce22cc3c191eaa0a7a58e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b37687217c296493c159bccb4fde17125f7ccf5c57a95c3663a6101f4e8280bd +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d796a031e89eda9bd6a7e42dac4c7ed6f5c7f8e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b32a2b96ec85ba92068f0afbd60fd2f590365d8769a9ff5b7b7402d895ce87bd +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2c853bc9a0b1a5e82fab113e1639673345cbbe8b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc45e4dc8db119a4cad7d1dc3415852cbb9ced969fc105c3b453107c2d51a27b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1dc8ce47e7a67438f41b47ba133bedbab5bdd01c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ef6ca5aba19a3baaa66ce5b0f4c40393bdc9422151d013709a322661dbd8fad +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d233b1f61c5f0c6551409022f103ee146f9ff03 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8d49be1e99b43c880234b6c3733976628be11f785be48e715137b60e96475bf +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..95a72d469ed62871b99fd387607f9459e65513ea --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9dc3658a523a77fb2d55dde873ffadaaffe0d3fd26870972330656713738b515 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..27abebdab8afc3fe4a31960806aafd8311c8ff78 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c19da293659c6f567634eb56ed746a644441b6ce8c8e96389a20ccdb145a076 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c64da65362f6072e9647b5660a1f503d101a853f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:298c4d5807b3a7a1690e18ff960d7d50b5ed66fdcb53088c18431f2ff2b4a79d +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f6926c3db48c73d4ee651ee4edb41541e981b1f6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cb311cd61380a41a56798b7cc1540316c6aea705ebbc24d491317845ae042c7 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ed154102292a3418cc0d279b51850daa5ce9bf0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9baebabbe2f291984a498945dcc0ef7c30851eb0f84e501745239eb70e588e7 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dee9492fa2971e83635ed1804fdb451ecf0bd341 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ff33f337808625dbf15b1e190522a9eb6148e3327cbf122394bbc9051d2cc93 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..99e0b853ae0af27a1cd85ec06f7d23574138aeb6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d90422a0b86a2706ff54786c7501660718e0d635395a69c2585beb8a35537bee +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..baa48ad82724b1ad797f8b56dd0e71d721a11f9c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30de9d013d470670876b77602cc655b44368dc14b3f871d8c31b274a7894eb9d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1f20eef045a99bbf267c267d9ed9d68f8821a70a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bed6e8d6b4d7e44483c4743e2216c5d1b8da968cd5efc73115223acbbf10ead +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..919a8edb9083ee312e9538775101fc6720fcfab0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cba26fb6ffc2c3af6e0e0c7aa99bd5aefa974c69ddbd79eabfb8d5d0cde3c913 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e9360cd24c9fd5dba1deb395e337c900790ee25b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19664a24c40100de226bd17dea0d5fd15afc452d4711746d5bf974755613cd6a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2238b16e99e96254a1697cb76e01efe25bc3892b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee4357e6518d3e479eda3c97f37a3fff463209ccaa1dfc38e2d9893363b54e80 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8efcf63f550d2479bbb7b98998fda2343b929a69 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:741e0066dca5a5743d65880c316283109a6d7b3cfc37f3c66efa74300dd5f463 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7bbd65af9f35d51bbb8872dbe33bc5c115acb5e9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84769b926c3e7ed01045e41f9c22e6d03aa748b11b089dd839754ce8ae48dd3a +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a7b21c2f9ad52da484054248dbe44fd5e4a8d890 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd3421af7adc96ae7634bdd86935f79d5bf121ce402c266b5ca92e68d15f1fe9 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..88efe3b0ca65baa5042c37106c6b7061e246a5a4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2035ef20cb3d3dbd90c82a42765c7b651b06ea3caa6d1abfda605445b19a1245 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fded5093523c685c5dd33be464cfa5dab3e28273 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0aefbe03ebfd9eb59b2091ddd75bcfb5f4c53a0ca80fdb5ef88abe89f129f5f5 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..95e407f99a27e63cf28568291ef2a6c3224a50c0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c3e5c392a9759e8768194fade761d84474e7aab735feea1437b6f9c3bbd1f9b +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..29514f8febf567872a304dddc57b1ea75dbd2700 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbda2a29aee7d4621f173bf4522e8a078ac14e979fbe922bc439026a1e5ee389 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..abd022e5ac9565188e725ebb1275916f063161a6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d28609f5548a250e97662f7dae1ca5832d1e5fd51e38ec2c8d72662e8a8ca915 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cfde6ce3740fee035dba1d562a6e0b9cec0de2a6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35fea5f4c50081da97e6e084ff5401c26b97458afceddcc8739719af5f62cace +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..87093452a885543a543828059ce9047c83e81541 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e04370c5bee73dbbf0ee0cb53290cb51cd58f4cb36dd8ac2495e5719724d186a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..854b8ef23103ad464fa6f5097defc19844559380 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3aab055013ce75f5fc224276c18932496df7ad3e9f9c0f48a917bb46c8ed042f +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..315b4d9993b6d5bee15b1b4f6c4d0c8f401fabac --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c78d00d20b7167c75af6634841b71dfb0f0c17d54d69bd0d9261d1160811d1cc +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..acc44523041bba4412c107a7bd3011407f32dc42 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b35ba59613dd343161d658c03cf7d4c9641874b7fa18220f58d91cbaec69383 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dcf2aa95d3b178302a1b27cc1f84c8ed651d81c5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbbddc9339dedda20b1689ac6c71cef7806dbe842ba9f6a378c8bdf9b8ced293 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..56c5aea59c23b12bb7cb758d41fe92f61cb19765 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b484178fe4cd1f2b209aed26d463d1bb059867cf0a0d9d04d8fc4afad663f1f7 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ed61b9465876660ec1dc5d3d3a185f7402be9c4b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8233626b6ee07127226ffe9be13ad727fc1645d3a496da0f83c1fd5f8ad0b7e6 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..87f0806c18d7198cbbda69cc6ea192b2e385ac60 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5474cd4cbc9127a7532ef7d96dea8069f08ef9bd1691440700beec020d2cf277 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..738ddf970f66c590dbf6078fbb0d50197c510291 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2e623c97bae0d6a537f1bd763f74f74d1097d93ab73af49f3590c8f70908618 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c93f779a9b77d75e3581268dcb62a1f1262efb6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11092baa61ebc4638ea042a90e17a60f73586de79ac163f75572830a636bd2ae +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c4b9b68b676bd7e2c981c6c0aa6755829bb34122 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:270d85b3c6214434dac7487502ee947ab030cc53948de9cab3fbbdc13c8bfe91 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b4fbab84f49f141b539d37172ad2495641e1a49 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f375c4782c8a86c60beb8bbf47aba604ab442351670aace5dd013c668fb132d +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..19f6386cc1c18fabc4e68b77e138f6276695b1ae --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70cff42077b0e66505f359a3db30992aea7c35315c4ae6e5033a5a3132a7b41e +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..332bef1d3f3e156386ef35c615c7c4344bfd527a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d40679e2303e265091913bc65371928076149f1732da6939bab32d1d723f6479 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e3336c6f62e2e681bf4b5ed4add2f0dbeada03c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33e668576c119f68e3694d3c726b83851a3cfd3b2ae2c4e6b3147cc6d01401be +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ded527130dfd576ff43b7200c4beedf3e67f8582 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de5c65841378ecbbde669214aa77e68cff7272ff47a55aff073c07a72da1a10a +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9646ff4b19945bdb7dd2121891bd2e8af91167e6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:884e7b9c303ba4dc462fdc94c8aa84c9e3c20e04ee11bd08d612efe631a25dc0 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0f4a991eadd0dab9165c183a0d471f214d1d474e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47d611de95e74ab6bb8157de9114ac5ca4468773a286577244e8dccde51eea03 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7fcb139e54d1a3c3dde1a37ef49f8722423c86aa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4637af3fff8b9737c8ece97572da5b6d7a5b111381c355f6fe4717b83c1acb8b +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..964ae7915521d54e0073e56429d34a0fb6f4b61a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d0a1f113f299db1db2879a4f4d334c76b0185f84a20ea994d0a761868d02009 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2e22c9bf5788b6fb0768d6a325a58d6a1557dab --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59073fb7649a2f70fc4aff5847261b408d6ba88b1b22513a5be33beaee37c1cc +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c266c64c7b92357fb6c220dad16675584d1675f1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6106907edcc4aa46c074bf4addc7b960e4775c87553f7843c6bd0c7400464e0 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..74ff2000cb77817574f1a71b822f5747506dc058 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de5866e6ca2f16b432def625bf4718048c764464cce27d9ea1f4dbea66bd7d5e +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d927c3f73e16dba1357a452373277b92d7a3ea44 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9547e1c41eceb6cc70f3bb44513f7a698420899d3bcc1078420ba87d00c0055 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c38af0e87cf92b196df3e2c3dda35bdd80e50963 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be62b871d485b88ddacf4f3526d9bc1cb04cd2c40296325980f5f8e91e66b327 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a88564f33dc40010b6f36ab3ae13b7d6b381dfd4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2ca8b443ac6fa76568afbe3f1728b01bb9e0f8be26df3202eed80d59dc44e8a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fc36b04622f581361e1b609c7ba3e5c0a40f9576 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbbd136f5936abc687d77c4b827a7ab9bce3ed45d3fcee678714a18a16071787 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0549ad86fe67a96fef2356bfbfc36ea6358f8c17 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a7e2d2b1bd2c34fb9390c89868afb70e3760dcc8c7cff09e3170b379c76f199 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bdf194693423eeb322adca584b24393e89b6a1c7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91b3b084bf355921ceaaea69d531f8c68937981123b1a96d1ab18b73a65bd4bd +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_01-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3dd32c86ff92a831e8c4f335b3e3060c4f8d762 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:999dbc4ee6b605a9fd71d63962cc146195b42e305707ca62ae06007c03843db3 +size 153552131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_01-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_01-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a54ad653cc4ef5d84397e07cc6b1e897e1f87033 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_01-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ffe4455fe48bd1d2a3c424b00ef53dafbd1cc64f495a4ea0075266bdd3e1617 +size 153552131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_03-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..81fff3b66f908c500a9120bccb40c70f563672c4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dffb2264ecc1ede357a07bcf855d8fa79922ec66951ac52d4c5af5538ed1641f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_03-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_03-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f81461ac73bc560d6a70e1711d0b87c37fede25f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_03-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa21408ed444ba3b851766e80acc53c5185f540ba560283fe4333558d00c3b12 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_04-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..53989b6cab194f90ea8fd65587595f93581837bb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d64af563b057db95ff449713868fb8d90f8f61051ea1a818cec69288fc3a7ae1 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_04-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_04-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f391f16bc7ab8b2941691d854c486dfcff60043f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_04-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf09cf6e1846fd173e47d45f00a6661c3301caab1a9a61f811e623f043e6109b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_05-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..55a2e832200a5507a6fc0ae69bff84bd431ed296 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c0dac5a4b666df14c43ebadbd98908532c14b0c6834248518f73c7a4d72ba00 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_05-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_05-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b7d50fb1e8fa5676bbacd10d892bc8df4a319c32 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_05-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4916621d9ffa8116efde5b26277de8a36484b1bb6e412ac63858afc7f37a2437 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_06-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d6005eb3d7d42cb63a1d5a459c67b58d9b5eaa4e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6aeeb6663c532058264cf4d1c85d743d475d988d7e1beffd903f40b997621733 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_06-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_06-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f0fcb646cc4cf251010b7768afc4279cfd4bd9bd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_06-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a6ba4498fd4fe345426035cb22ff976772c083654095399ab5c868e8f141d48 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_07-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b16c1c65c6fa372e3fa89c50fa9384b134727897 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fffbc85ed3ac901228b89d0bc31ae7d6f407ce5f8204eebbb4f4c4845b20aa4 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_07-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_07-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7fe1cb9310cc5ef1da74450b6982104832b77b6d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_07-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2facd1ebda977d142172fff6895c10be0af51b509b2b8391936c3829cf2fe8b3 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_08-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..21a271aac38a2492802d19e11ad479d5503064ce --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcef35bedd6cc5fa062b41815676a98d005c84d0f28d6634573435e12bcd955b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_08-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_08-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..14b89d2d478d9914f32643c22146a269adc628fe --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_08-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad7f9e0e70a56020103b880482185ed1e390c2d3781037e95ab4d6aa8aeb6cb4 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_09-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..355cdc980e2b2f94f78db96b00618a65e0259fb3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a97f19b2d0906a64cb0b1e118833724564782558900dcf551d1d9583f676983b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_09-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_09-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d2764a95a6474ae36cf7a77f06133dc281253b78 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_09-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4da82288fbf06a34f047a359b07f9d2a1042b77012677cf70292e7c28a6076a7 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_10-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d7fb1950e325ea2076fa67bef35f5635bc3cb8ba --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bbce599cadd4d0cd3f6eaba7ae9e53c63bc7e248c329d04c05cdf924f85c3a3 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_10-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_10-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..46df71f8885fb6d06969c1bbf56e79be89c90c76 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_10-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c80e3d5d43c5138268d7bc9d0f37f3eeb467924302e2e7abedd1202006286649 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_11-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..17b56397600c0c91c11e1af864d31b9d627c6f7f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7769ebc5f7b0dcdc1e3b15c0678896d6dc151b89acf30dedd00839b74ec45a22 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_11-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_11-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8de75e06b8ec5e4357630f4fd9aa83a8b0dfbc3c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_11-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cb5ad9b77191e44b6c2c16a8ec5408508ff07d08a3ff25276189c4c7a338290 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_12-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb1f869b374be11409c5a568b8bfb325f0176ea7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:328c4893cb1c303ac9c038beff4017b9d53008952e5ab1f6784a887aee4a361f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_12-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_12-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b07cfa2f0c60c233a5573fd0eca09e10000e7b0e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_12-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:693240f7fbb70014b787f023f4572a6a18a640b86e072f16ec842245dae371bc +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_13-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eef02498f56951d0fd9a8030e5d61e62217b746c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:171131953f5be3c44469e303be770769a3017da9ba3c49d4c29dd70577aa119d +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_13-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_13-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7a5f13e95f143b003abe5043cdba7d57147c3896 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_13-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba642a5e62abf487c9222fe678a7f9b3d4a340b6587d9f647454725cb4921b90 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_14-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d7014b35e81fc0e52f0316f38a23a4f95764fbda --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:310a39cae67c733c00a11ee86300effe532022c9abc53a0208ff357e2bb0e494 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_14-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_14-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f2672432e63c8d40c8fc5654ca729e6fa55c43d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_14-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b87585bae8b46f6cc11b0e621e6e9ffcaf48119412030cfc71f99c9fd243aa5 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_15-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2532a1cde1dfa00af54848bece418052de66b40 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f3e1c5d00abec46ec6a634435a99e0489e7f323553eaad8397082db6837ece8 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_15-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_15-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7e024b22deaf3b059410803739bbcc5696b01373 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_15-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7712b685eac1e368fed67df793c3679d5d3f3e0b26e835f0384ba7e79be8e4b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_16-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6085be40e8a36db7002fa29a499f19159a1f41cb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e852308450b19a262e87cfd328486682287216730b3e7568b608c79e1c32edc +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_16-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_16-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..842a5d02ad6419ed18da24d13ea2874cbbd09f83 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_16-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:010a6cc5137184c900b8b82a5fdb3b1b3e896228b167775748f8100009a5aa8a +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_17-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b79ed4681cef597009a3de8ee41eb4ea0efb1d51 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bd29783e3396abade7c811ef91f2ef9909d785939616c2a2325b2479007dfce +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_17-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_17-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..14cebfedd1e5d151304a9d4408f7b4a6aecd6862 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_17-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75442f6a692c1c4f6425c4bb0e2f75c72596ac031de3bb231ddb74ade0eb0845 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_18-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5eda35c71051f347c90ff2e2cacc5528746c1f62 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f0bfe7d7c1922da6216bd97804ea39e5de781711420ec6dccffe0ccb63c9143 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_18-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_18-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..201d62d91bd1718c10880455df33f01a2566ae39 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_18-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b887f6dc96c7b2c136f57770b43bb62cc2b856030b36f769d6ec1d8f5f1255e0 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_19-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..13161fbd8d4dba1e5c223c38f8880e5f508fcfeb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:935e85617b48157698a477f41017e1aa9d5aecb3b8cc3c09518d2a92f3a2466e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_19-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_19-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..080f7f22bb6f6147d05cb5d7a0eb735f583784de --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_19-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f9a41c545973fe9ebf81a7f7fadd9ff83072450f005b7dca40d572b5760c3d8 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_20-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..59ab1bff0546805ee0065ecebf0f1c4f2660a952 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b01c7534b11c5ef60d2902d14a339e9ddf0191e726eaae764b8db3fa46f42655 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_20-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_20-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a9eb8992e303fd2a99a72804451ce560bba7fd4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_20-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:602534674e9c7ec6fe42447be8487485e598cf79bc98f0470964b9d6631de8aa +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_21-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3e87e46a81d14dfd813d69f02be927a703d9f4fd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea0352383bab8bad48177c8d128be3877650c91a5ddbf13a51e06d92e8d13d4a +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_21-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_21-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5607f6fba012245e311c370cf71bf5db7704ea90 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_21-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3bc48a23d6d35b3078e99c6f1f4816b8d4cd8046aeb1081c8094f8fcfd3dcc0 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_22-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_22-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8bab6f2ce425309b94a2ccb69ef9dea4bf4fe140 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_22-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:352512c6a72064a2bc6987aaac7e2df112a6506f9b21318e83b1bf531c021aa9 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_22-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_22-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb7ab556dc809f12ab8cda65e121d69e8ec130ee --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_22-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95d69058940dd6666be7af7828a4e64ef28b1f281b7ea4db35ef2eab98bb1cb3 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_23-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1904c81d15399052745108d4232d78a2dcdd22e1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11ad8b414060b536835e29090f7581dd003d4c8dd14fb525fdf92227673ef754 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_23-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_23-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dcd9fdb81733eaf070dc981b41bc50d7d6dbb936 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_23-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d97a3db46c2a249854a24ce6dfebf5b863bfd726211b7123f253db96a7bb8f4 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_24-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_24-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..650f952e9ff27afbbee265754c4e76f79b59b701 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_24-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b91501b1ff0cde52d858dffa4d6c3e68bd86fc8893c467f6f10afdf3e169487f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_24-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_24-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0488efdd02f9eeb8b9be45a279cd7c1514519e66 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_24-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:654a15d8abf34e1250e79155ee58be90c57538cabfad1ea8f2d031bfdd212aa2 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_25-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_25-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bcab1a3392250030d6cec3d4ff80b6842337b12b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_25-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4093cc39b831105c30f3c29fa56fd5602837882e8b874593cb2237d8d282db4d +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_25-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_25-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..631bc30bcb38b2bd7800fe4e75430c7cf25eeb91 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_25-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e611aa35d258c530fd31952b9a9e50f832fd70291c9f6872176730d6d9e6c44 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_26-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_26-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3dc2ebaa0739d69725fb3be0da516b1bafd07f12 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_26-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12d48d13d0918ca73585c3c0591672682d6bf62c5381f5a99f94e5f8ebbb6a00 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_26-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_26-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..31088173765accead261981327c2bf2ee03c95d6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_26-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a99f56f41b6bf24eadd0e2ea9ad91adb350f02fc2df039db74035e4f92ffa6d0 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_27-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_27-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..885610c5c6f777fb0e64b09d62d3764c81d95f14 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_27-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e5908ac9520cedb9858ed943157d9ec4cc48c47bf9cf7e333df75393936c6b1 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_27-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_27-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c0aa5942b4f1b756e235a6243c457d4e44733e4d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_27-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:844dbbf9743ea08df561f39b08410670273df7b82b8122426e372f24820323a6 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_28-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_28-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e137987665724ef4f55b75739414a63c6be21542 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_28-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b115555f7aebe0873841bde3c7c0554a7fc62886aeb6bfd6a54a65a4eacb19bc +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_28-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_28-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1bf1e14a34b34ac70c0ab84cfb3cd52ddd49e4a2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_28-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82879a9defa59fcb93e7dc660876e77759d585569f034caa8fdabf1e36d6128c +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_29-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_29-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3593925eb0cd86cb77de7553b93d54a6829e8ee6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_29-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9929e0e09e003baff0c5030ff39667103520eb518d724a815b29da1932e5972 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_29-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_29-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..82ff93ad85fdf1103b207e868ef749e7244c835d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_29-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43a4f375878d68c7fe0a5c7dda2aa833de3ab6cdc3bb1420cd3889a138658d89 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_30-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_30-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e6925cdd7d22d06cf9308dbdb2f846d9c4ba1634 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_30-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07712c42744cd6056a6a52f14b2ec8fac366b72810193b692b05a46cb7188270 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_30-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_30-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ef0892600cd5297ad084bc369537f52a8ab55ab8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_30-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77de9784c2e4acbb1aaf847c2b173a125d6ec1fc94a70baf89802cca58f23f03 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_31-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_31-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..00b7d1a56603ea1f816ee8e80450722366715c62 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_31-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c9ae5ca87e0a5c0853bef4ffa98ce050e457c90d3e8a00ae63e383c4df3e29a +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_31-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_31-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4da34f9bf6ed36f1ebf5ddca295a57bc72ed908c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_31-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:034cd472d452ae447b331cabc4d4330c747acb93a1579b6c459377e91b930d8b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_32-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_32-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..96c0d1ed6fcf1306169cd781397a444156fc0681 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_32-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:896b018c610bd60aafc755d6189080794223da485ef69cfed430504cb70acf13 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_32-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_32-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9494966b84549d0cd03208964156f65ae80a4d44 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_32-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:314f5ac1b0b2d593f278d3ec83cb86ce70d99915c83e2a35c2b56edab9db7e68 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_33-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_33-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c9599d974d9abd2d1561d664c923b17fc1a4240 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_33-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cdb83fe4b13a6300dd0eaa6afa69e477f3a0d7dc79f94b40cc5ebba9e8b292c +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_33-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_33-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..86fa413ecbfbb001a6607b9fa8c7c77c618c7168 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_33-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc3c9e9870de88a17fb6b82521aa2224c21000114faa34c8e0852429caa05cf3 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_34-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_34-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b35e111f89a4f4998da851bfb3286695d854101f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_34-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d07982813581cf42c1938c1860570011ebe52149a75af15b70b6ee2b7b8599cf +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_34-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_34-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..99a4ccb2e3f0a23d279ce6522c05d93eaea9a4bd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_34-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98d29feac0fd0894e722dd1e256348247aca0ca5cebfc76f8aa70584b8415488 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_35-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_35-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1b9f2ef1d8a37c4380fdcee1683825682171e26 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_35-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:042b379977c8872e4459587879b63767b583758153d250f355df70bb93fc91d8 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_35-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_35-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e39e858800e880b4a5095c157f4c56b7aedf2982 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_35-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:295a420f46a7f09efc7fa909a25fd185f99fc1c3349516cfff8f21da24896d78 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_36-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_36-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8afcb588a961940069cf26988b5aafeac62ce875 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_36-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37b4e6e875dd594a0ed909d8e87a44080c2d56176d52b9244503acd7f47284d4 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_36-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_36-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d8ba16de14fbbe0e8c14f18179cc2f11d55bd121 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_36-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcfacd413448a50428a635e3d0d36616cb3d61d4e0ab39cd9a75c9d9c840fc29 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_37-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_37-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0a2e861a00c6681c17410a76337e6bf5ebe26a1b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_37-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96cdc08d101414369ed4f7223202b11262afc37e602e1b7e7eb2848ea76a89ec +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_37-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_37-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0b0a82fd7215a00bec9fbcaab672f8c222219af7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_37-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e20158b92715f9943b82f20d8b01c60868a70d98525bbb11c12d3058072bbfb8 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_38-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_38-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c9f97748dc3088c91826752779caa13eee462878 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_38-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ae51be15f51290498531c1355a6ae615b7b1726babb936c0ef5ea2e18a7198e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_38-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_38-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eeb9a2ca13a4bb4eeb17cc52bead44fe4cab3f44 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_38-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c554f1297581f58519616360b00af28f10aba460641584a9dc54e4acaaac442d +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_40-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_40-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..14c4b9916625ed5dc0458291a9a040e9100552b4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_40-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68a4c2489b34f74430f1815008f8ed241e86235666e0dde1e02710048a286bfa +size 12483 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_40-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_40-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0baa18b77b49016238c8f69d6f3354d5f783dbfb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/layer_40-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2aceeff4c7113530456b38d3ec6bff7a6ff292b97376126bd52811bdf2caca4c +size 12483 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/mp_rank_00_model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8d75205557515f17dd720c2d0252529894429159 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d081059466feb7aa0db19c4108f8a2beeac79f14db41ede598056d95056c55c9 +size 51443 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/mp_rank_01_model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/mp_rank_01_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b72b0ed3e058a5fe80a0b237ac0c4e7b9bf0a27f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step50000/mp_rank_01_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a230c59f75047ad4958e97e5575c9b887665b42f5e5d95c4a7537186bf7cd54 +size 51443 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8e64a9896169c83f6ef2b086af11705cdd53b0b7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c125d67fbd77804a3c5a25691e4b3d3bf26f13877c4d96f0cc53734fd2dbb6f +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0910563e619a251f33c85bb81919126666b6cb09 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b57f7f2eee57313b43554ae3f94ccfae1064f7a0ae917d8af06fd4cdaf3a247 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..23c00961e608d24eec3cd3a4fa4903b8d3f327a4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:749eda17772fb87517c99c181fa0f08075ae2b5c944f2cb3fb9059a43c8fd9a5 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f98d5186778bba80b4b3c7e2b5eca1ced3d50e89 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:491a5fd7a28ed1d257d92fdb8df0b043b960f0e8b10f1bc58051c505cf32f387 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc774bafdb8d2713caf887a6aee44d2a88998c34 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b89875b317f9cfc022f8064d55e3299815e232eb7433bec302351f699574b7c +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..30431f6a700d2993904a9b259c2f250be3991f08 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2382ec6fef53a195734e56434f55b7b21e189c48c742e7c3359e99f015908492 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..65b406f067f46114c167109cb4b97af5ff060c62 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c99bed88875e6a5c32ee7bc539fa9ab5627fd41c17fb167e21a5a9485b164434 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..539b579440d0ae462e3bda9abdb08ba22ba924af --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d67d14ccaeb640f57afbbb605c0029bfde1999aefd1205f1ed1355e21985141f +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1208f2d0d2b27b020c9d4127bcf56f29f7fe1d6b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1ebeea7e9b812a1d480044bbbe5b377852f5fbe7c445d447b0abf1b5e6a7aa0 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2121bedcfcebca0de959df34aa24d89401f776a2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea5b7f4b63a917b5b9d715bc13dc256a686b8abb8e2a43cf0f9c54ac96c18640 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..21584971f2de8c2a79875c07a445ec1fd49fb5e6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0aca8ea2af444cbe2bb664cf44692a205230274990837af9b8149bad73bd6bc +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4e00c70617ed01a8f802d9a02533aa24733bb823 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af71a910624f36f54d8eeea1740e226f3fad75a2643e91258f7627d113440e95 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d93102772ca107e73d7f640cec0052717fdb82d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47c5382d7f89640b208b209d42446650fa315db155020c44146d95dc81527c55 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3654474c183db62c87a5f1d4c62bfaaa068461fb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db2a1e7e830bd51d82c854ec09ca2a396a589f3cb7ec0b7bf47bdbbc1f8dffb7 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..34da3819b8dde9275ea7c7034f7fbdbfbddc2d41 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1a5208d1d716be3cbce49e800ad3b4d0dcbecf84c7e7fae52676b49709c0a32 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2c49b4596d67ba0f2dc462bd1153fa0a13ae566 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d7310ee61e5244b6db3e31cfee6ddc578c915c4842c469349f90396ac444dea +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..365b0ad950d022a5e4b393ea118a45996b4fec67 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afee6604059cd0a188198e3cf4b87606fc5bffea4a5b865ec209bf67d4568c14 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b2c431831632658227f2f748ca17e3064679c050 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45e234ce1e770a373f81721be405f35c23a72fc3696c1bccedf829c204e4e8fa +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7912c955cd4234263ddccf131967f0e3d8de1207 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2754b8f8db1ae7746340843c742d4e3d042efb4890911fb8dacb7cc3f57b1770 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..21ff21d3a94fba1d99d47511b86ff470cd8cdec3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71399e7888566db08a9d9e775a5d75964ec694236063e0fe8379c95b6a3549a8 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f9ac961c069849f5d4e3589694365a70dcacd542 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c2a120107a959b312f980dd4b398cb3ce4a3505ea01a94eeab921f3ccd72226 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ea0679961440e51aec0277115b2a91d2c4bcca2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d03014b731a509056367e06d26f3002c07338d3bf396c671dfbb4d4aa03cd980 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..be22a5ee4fe0836b74c8b968de981945f531de63 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:817331fa7889fd36d1b46c40f291eaed0a60588abe8a825b894a302c73c03b41 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..349652b5ba7b4ea7ecf841a2bb04a82ee18f18f5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5b34235f3fff04d388ead4af4049034b1f2cedf989c449646d98590e1d81476 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2c66add1479238e31ac61b729181b46bd4642b2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66764343ec58afad57d89baffa4b854be3cf1366f613ff01b5fa9e6435153ac8 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..da736695aa751cc3636400d74d7474018924fef2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5692186f15426021d592e8109d6c7029fb39d622f5f619c57b0b0846d58ac6a8 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d00357c9d713dd94f5d65e026f20e66ce465ed07 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd0e71bcc056c26ddfadabee469e8f8b69e3e4c52d524e6d343b14faea302f1f +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0442909039d77c8e2ee4488945935e3a342b5145 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ee53ba5f08ecf06dbb2a3cfa71bdbcd6073aa8c601cd072622554af05a4cb63 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ac0d7c4ff73ab3fa3bc75695111e7c1e9bb22482 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ec4297a9d460e83bbf4e41d7b8158ba1b6d6c1ccbe83f0ef40611884cb3c603 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7e7f62e1685a53f14c1df45cdcd6831b4f75e78e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee20cd6d6419f228317c6269fb8e559a29a527090e91b37e7fc5f698151aa9cd +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5a44eadc5e8e88e9045f34a89cb81e58e63ab50f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20b27396c02cdcf57e138722158df7cc33f089094fdd94bb50e2e8f9fee12218 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fd58d43c03122b3838c2d1e1399cc74572e5cdbf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24d21ff625dd832afac73a3900ecac126dc90f3e3bf87707a2209688d1717ed1 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a66d8013b99fb2e5c3a9391beb98a0ab15c35f7a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8eaadbcf9ffcc5574afb416ea2bc73b8e3b1ab9cde33c087c4ada6525215741 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2beedcd6ae254604319eeed3d8bbb0f2f348f308 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3afc3cdbe71536635430060ddb553aed88c84421b0efc1560c9f9e849c770f7c +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9fce3f43b4761a225c4ea6547d7c2b5b38a1e147 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eafbc2709d8c578a435aff0274cacad39f89369a958403d1ed18fd68cf46f514 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..99adda022f7de6c42d22877a70251f11493bb731 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f6e877ed006fbdb56f9f6bd912517e4dc95358368aa5ba05e5e3964654ce867 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..62ce323ca489be3b8ae5469ddd2de668e2176e1a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad8e324731db3743972111020be38385f42b2d76f3977a9f557be776ee46a10e +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bb5a298db1bcc910a9ad7275508a0f1738cff0ab --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8b663067705528aedbb988f414091bf35dc661038804cc0bf276b0e768b394d +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1f0cc6cfb5becf0945f96540946ef021de0b6b82 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aaf896108f190c5487b34fbc7f49700020b4b163274622b9c22f21ddacd59b0a +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d938a09ea9ca40c529c2bed880573f974f489bec --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5e3d2144d5a8b653404fc04c803ab8beaf9f977166919bf10aa17eb5ee8bf5c +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cdbea45c0d4bbc4fc3f89b487b60f507e46bc95a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81f44048d9749203fe773fc05f48b6a504409775cd18cb5f7019d4a5b143ce56 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6acb0524fcb6ebb9711d624a048410c619669f2f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a0c5a9ec8fc705deb05387c788b962e492ac391da759409acc7dc394e4b7df7 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d3b28bad5530c62f970b9d54c987fbbafbc1b41 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31955ac1a6c44d8dc248cddfe5cd7a3108a30476dac528f6c75b385650c3a440 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4cb19221843b6eba52337b37316f158bcf65b3c7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3a021d1fddda46de77f86756fe37735d1a320d94b06050c052fa8a5d37ae385 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..344316218c9a4ddc94e6faed064cb8522e0bad7e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:409d23203bc75c5f304728c3faeb3328ee706edd0e5468c9bd3bf9773f6ffb42 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1617573f09f3306992224a3d763e37e0981010c4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8609dec937f692c16774bcdca4c128ce03a5a396234912606700f04e45963c47 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2aa894e03b15927c81d8c02477de8c8b6eb41fe6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d2be4de8ce70778781f6139a713612bcf6fae6db015eff9c23f91f33ca421aa +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..32c01aa19bd2c4ea8a0bd322eae822bf6f7d98be --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40b680709639d0ba65fa8720fc662b7a3114d45ad3354add7b97c59f12161414 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..16b437e46d97a7aaf6083a20c7ff64577292293a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79468f775d18f32c3c4a053e994a364b9d43f45a513d3eb82dccd4e88322cabf +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b56e25f1f397d7edaf5329b17a960a4c6e24ad97 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d449bd266ea6726f6c211bea16bf0d25da11713daac85da1e23f236c42c862a1 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..444b52d0800eb1526b1248e397b777081787cd91 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:047878aff23f881bc6a189512fa3e4baff05a0e03415edb354e0022b5b4363d3 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0ed408ef53a143c37449af2090db24cd909967e5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5c2bc96f4333ad30d262c27da3e02e94c0609802b74df3afc76cc84fdfa84f5 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c2481b82f40b36bbb6dc97dfeadb8a1b8a06a6b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:477c31adc273d66e01750296d546ec9d158874629215d9e8daefa8e0da24c85f +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a7a0f48924eefee41f2756d78e6aad4b22c9bce7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3600531596a6d95898ec246dd2a8b6550997467be1c74c68a6c2a8563fb24fd1 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b8fb820e1c2502a934a8cdb73b01dbd56d3d45b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddbb732675e28c606bf3b272d1c4aac98a7ebdf3c843e4491f78c6612e1faedc +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..69588b0941899a8601b167262ff98458c7733002 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:555378bb2a6be3c2f477665de19b648e39666ce8ab19620fcddfd53e10ba54db +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..83e4efa394efcb79a8ccaa9ccba8ad3b747d1390 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f67639d8af377a3deb2b63ea62cebbaac5db7a40bed29c2ebf3eae762fb7766 +size 167872941 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e02cc33bb6a7ebf93cc9d868b1ed0951f0d5a2c7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5536f42dca1b51ee90f443889edd9f4a238c7c03dfc13a4207480369053fac8 +size 167872941 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..87656061b55c34b58c4d205efb1e71dbc94100df --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fed95aef28bccdee53fbb5e7075ae3b687886fc8dfa7f4d584c2c90094fae031 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7044ff658ea03c83d3a08c78e9f284a8e2872fa6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8898de970dc7fe5eda5ccc0cd5db4aa85a258f890c88697f6f35f2a6150107fd +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..797bcfb3a928f804807ef49d38e84ea6effcfd71 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61bc3a9a9e8e42931e7578c528e73fada8739398a0424d4ce89c942af3866766 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..43220cc03fe5f7a8c5f35d8e01e7c54995239440 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3766b24f484e6d8f46fcd5c1fa365d105592a3d3884f3477500bbe61ed3d3fcc +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7eca30151b3fc62e4c49cbcc48193089b83a8abf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6c6783a7b73bd4c84e6f22f0b28fd06e69ed99e0d22cdd897b06aa2b73e8567 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..45f7c04502cb1816365542de07d765e997759a0b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:144dfdd0732a59699d493bcb67e3ace835c81a994a36b670d408216adb1ced76 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..abd03c6be85987e27ed02b8dcea65b961976ccd3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f8e1e9a300f57ff86c71e5549c2607552b151c879945a8d69856dc6756d2170 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e76064c18a4a719b927565b0f5ad4d877611b568 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3abb8899281914714791c8055a7f2a77acf94eb0f93581633092e884948b0214 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b579a805d6c6d720f30ba65d2a9eefe59e3a696 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2157939631c484c7ba8422f20643d3b2d53dbe11dd9a9c767abf1885f3e3273a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5667199d59ae3ea54009728d81378eeeb6729b89 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afa6f7bbf18f16fd13f0ee5ae55bf7d46a774250490dfeec4251adde3ac6bd44 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..046cdb9bb4f110af412b3df2762ebf6878ee2f80 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f86759aac3fa9dd477264868bc96e78bc5783f7c2819c7dc4a40d53265641d54 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a8cd05f705963ed76d4e67454a6d69ddf42abc96 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:217cef3af935b435faf9c4eee99a5eca3030f88d35cae68bf2f7f45385ae92d0 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d218bbc261c6e54b4c777df41e136d86643c750 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b93ec663474bf899b244dc2daef1f81096711b9b6b5ce745fd0e39effbdc109d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc620cd67431fdd72b3aa359404c06c4ccb560f2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a39d9fcfa551eae7190d802506d91f87272c49378de42e887f6d619c978fa21 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2dafd479f415428fa2aa6e053719415a78cceba5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4aaec7b5303d55a196fa750ef9db037e0e1969592cf7c2946704666708875959 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..44c371a24e80835cc57abeddbc381cd20f06eb8a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8706d4dffa28edb4ec3c30cae7f282521fd1d1aaeb869b41ef6f6d8af0cfbaf9 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..48e6d747d1650ade8282d7667a4c4fbc89d11d5e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c58a692bc878fb9953dbc29d24a2785c15c3d8156c919595731265a895a150a2 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0bdb118b7287d5f50dae2bfd32955f537eb5fe52 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a489fe233cde9f155312b830ebafc7db9457ec44ccb6ba8cd9ad3e10af62f114 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5536db30095abafd941a525d3399c524a251d8ac --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad150aa4c746e77f96a7b4500491958120d474dbacf7f9625c41fbd8e603e336 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0e7093cfc1bc53e41ec7e9bbf5d7faabd2d175cb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1734bf04016bb63acb30b9115d1e69c47530e355ae6904896fe891053494653b +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fbed3711821cfff1ab33f0b20d6e663cd18fafad --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfc389fb9011f0736e3c423572bda74f0c6affe0f9827f35dc211c5a703379b6 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5e5c4b57c87610d64b90d82ebde8c33650783b2f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:586bd60839cbabf512215f302f8cb415fa1ff30cbfe9c25f1f5def7ed5a9c33b +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2b410f9cc01092bc2437f94a8cd1cdcf16920639 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:548153ba811f83964c139db9bb18a1df7858e5924c81bf1a3647f0d859b8f4b6 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ba2e4f669b819a28bc98be7b15c1d9c0705bdf8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b8829515c7b0c7752d8ef674c88f45884abf94e0551910ef469e46a5268da9d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bdf8ff7df85a1aae6a65b77a57edfd5d53f72f33 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:063d12efe94188180cd2e7bc3cede4ff1a45f204d2e53eb195345ccf62471e54 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7684c8fa46402f8de4523d3f6fc6b8d3bb9bbe3e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a535791a0e584f8dbdbb1fa520707e1cf0067affd4d397a8e48414dda35d6f1d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bb45002bb81d8c5a928f094657b1400d8c91148e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:024286775cb36cdd158a8482f8934442ac67296d7c23a90c4df7cd8f87bb809e +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad03325a6e64e1620b08700fd9dcde7b93d1ed6f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bda7e42cbd51ef0cbca81c31577f6a72c652c6fd1311891816b3d23a06988473 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a10f67e0d0635230ccf3ea3ce437b6968533ede --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bf4a4bff2cf38efb5548c2090b3e32e31735ec0f5b6a2feaf894f87aea881af +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9ad3e9884b1f17ae165a66268a5e2ed841282268 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:555f9dc1ace8ca217b14336506d0febf094e2b6c6a8e6ac8ce67a140c5f1d083 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e4cd642d42ffe18684c00831eee560ba6a380ee6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c895ef373825e209ab18e3e3816347ab5b246871733b9f3f2f7678efafe63b43 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..823ee99d3a2f5b5092dd57c443cb4e6905d2ae83 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1067a333e672af19448cacb2bc20b6e2dad723828e66472f36885cc8f660c86 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad7d419ed80258f87ce6d599213925ab71ea7f84 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cba464afa63f62ddda58a8755ecf62d84a373ec5cbaafeadbbabd6cde209026a +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a86e1fe0229ba2f566bbad006b9604653c57c91d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20e047dd356aae00a44d918580991460a2648dddfcb420a22329641c73c97a33 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c99a644564040dacf35dca94b8f2c04d75caa599 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:756acb4541baf1d10b1d7e4c1bfa5b5b4b498429f294487399ef6f412875e163 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4863241d7b8db449e10ac232b92e0e8298bb87d7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25c1e8712abe6a1b09a01e77e2e93834f37200b17805643b26edc4fe2c45ad60 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6856ca24d70d611834335febc37b2dcc02dbb78d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ab81b7f043e3327b68060cde59231e0bde2cc72d8ebfc6672d8cd9252f677fe +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..07caa095aa3ddf45e3db3b7ed54472eda5405dab --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8178d8f6fcd221f12c6abc80878f2c216e04d72b6f1fcdec80dabe68fca3cf4 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..10c89908b5f6f2bbdb7a54781d0280b8a6fe06ce --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0de222767efd4584f99334b76bf2b9def507555a46f9d2e26710ba19b2a68864 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9fc64f3c9beeb327824300a8cb2a5ce53b71dbb1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa97234d17495c18a0ec90ef2e86c285de69c685f6fa618d74fb6f671f25fdbd +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6694df61884729ad2174f1e494eab0f07ae02c1e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:507fde62e8ca14c1f1d88355fd47d3b15fde18350df86476be6e9d51f32e49ed +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aeb90c649ff734161ce8d9f92b35cefafcc2aace --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57f7d2dc41b13a1c5ca558fb152fa30b58193d6ab64999c482b183270c315e93 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..15812b1c387b6118ab5a01fd0afe979c7a49381e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06f424e8e8bb1e9bb0bd658c06959cd165316ad1af83915679913621cf633aef +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..12a731fedc4e8cacd63015c69d1ef2f866ae319b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64042e65f31eae537126b5d24a834176cc2e0709269be9cf57b233ea71a379d6 +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bec26ae0b34f81aef7f093740040d482f38d17f2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc6abe443c1daf518303155209f0ad3371222d01c2e4c47e8289c9ca60df0765 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..73c6b72e7690acca42d912cc6c763480c0956c4a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb9885182e0753daf59aa913074cf4cfb3f89a43d32baeef1d86efba99ef80f0 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..00e6abdba59eff4788a9cc678b211ad4dc6b2c71 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6890d6f4d3a5d74c4fe806b64392cd4279e8331903adb691405a305a123429b3 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4295fcd9afc3a3a8a75d7cf577bde34e50cca9cb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e44494bdeee019d9353a7b7899a5e7832ca3b64b591149131e3a5f7ce9211f9 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..22429c530445f20159824b4fafd0a6789f04f062 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87c8f8f64e15d1fa1be45e37992f44e12dc6cab83745b7cbde0e520451165fcf +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c47e1a695237a73be3d80d71553bce8656f22172 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:299483a115ac69c5a094abdb0d2ce03ea501e73079c387f00622ea1502ce1a25 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c48f325d982f7755d14fb2f209bd180d901bb24 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28736915ec8dd91165aa8880431436dc90492cdc8abc5642759822e70d2567a8 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f12a61b8ba4a99f5d6ba4e2c19f8ad56c67fef87 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b33511565113beb0676429879d52da8a872efd954abb707ae18f17244825e622 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2358f8920cecf792b71a2bbf8f8b7409a0895bc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d75b250e9f86fa2da0b7cbf553ce45eb1b03b678c9cd3eb23fb5b2e868b4c0c +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe01f382935da6404ebb750f8d0a7c5ee3d3b4ef --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aba9628af7a4ec1e22cd9cc53bf7f8f47b44734f71718d345627045671ee6868 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..00c858b58193ecc84c8759bab494dba1ede64b61 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22a2723a99028536624dbc03a3b719d97bffef498141bea647cdaa11458f2d16 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2010db7d703512bcaca5f50f5d18a49136eb730 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a29153e1a8955b56594ad07f11496d566a0cf6139f41905e9bf5a55af8212219 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b3d70e5dd50a8266e21e8c6a6cdbc90de1100e33 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d3e941b489ce02a606409f95bf647be86166cf74337b89eb399262d7d663340 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5bb237237d897e2ea5a1046df82a837107e7d613 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05f703104c477acd98d94d93e956c7e4b39c681281acb29a16191746051a6030 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..635050b93719401f91868437cf8f7d989b3737e8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c67e71ee48134b8073a3e5ad7c6d8a7668a99c342a362319f6322e20160169c +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eea8e53f5860928e7b552b1f2ea0653a9a100de3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df3342dd319e0b5b99d9d392d4278293fa6ea06c4d6f909341f96991e7a6d992 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ea69f1eff45e5cf3958a141c994099777c0c94e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3e9aae0f4546334063d7be51ef1663b7c0930867ad154dbfbd5de6b5440f8c8 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a3adf835be3f8eb6f8f5cbb081793d27eafdb77a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bbf1eb21432554e6383d9f33582be9a8f2a65616fa4579a0c91c67a22c41077 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..024d59c228783f306a9684250abd49b205a703d8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10ee64abe08112a875d735e31d6a795d9dd4abae8b66408c8db288bc9e2eb0e5 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f3caf0a51475aea6b4254fd1c6475c996d919e03 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a3f948f5ce53f6b7726456bc4b8856a2910890ebe4e41d355cfbaf177ec1d4d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bd3ad086f70541a06e7ad401ac6a2a5c8ed3de05 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:755c12bb1ec844faf0f860f5888e071fac37564dddb3b74369fe245be643e4a4 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6697682ed48a20c14a228e0e0a8e65dfe23ba1ee --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45b98bf4dddd3f44faaa69ba98acc89ac3bf7914406a319a69672a5fd1c0d90e +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..26716d4b385d059722e602261e03137e1670e72b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c328f15347bf02471b2b7c189edfada7eb82f8d3858f5dc0a94ed3ce4738741 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..43979a64c63adb0ed41f061f5bae56b0d398cb01 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e7212ecf297d40c5e89678d593d90e1976573dd1c9d72a196632227976975bd +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c5ea1cbe5ae535df6353040b12ac1080cfa5444 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36884034b0a9b096609962318f91556dd5e77b15a26f35ee54e86ab263ee256f +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d69bf0f7095c8b5fac15b4830641f9160fb49203 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06ed2132f5e58e4ed3c8437d7d745871b1ed33e551bebf3a5ff9526956b8d30a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..21644cdfc748a5b2ddf3198c29218c37674a24f4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa1c84f6a4bce7ac85364d69c246b30d1f67b07b7dc1b45512870b36f9166fbc +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..154138a4e2979842c417b82048eae89bc7bde0bf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bdf1664e9c6f6914e67df0f620bcf38df643c88344faac9fc4ffb30a23571d6 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dd48f8d2fa1528dcb56c830e87453c14dda5ce5e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35bb818e6674cdae118d7ab8b80c660505b471f9d8c408005a7a1d231c92b73e +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6884768fb51faa5f41813c63a7ecc3e083e2365f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0b9d69c9b13f11e9ba0b595267414b14d5a489dc337beffb7771f6baef489fc +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0897e15a80fefd79ac0bdfc85d00bfbb955d14b7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4288b6d746b6cdbc05888c0bc3bebb819b63ea86b4e4b94a34daa46b65053032 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..677a5f57cc19a995e4ee72ea6c0d21560ebf9361 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffab30f3b03482d914ffbde2d951858b73771ee24f9126baabbd111bdede65d3 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d9dfe03660910c8587910e57dfcb745e1b92dc55 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be376e2e0a7d83dfe570d32f0c211ea873804409e99d1c2965bfeabd6d7b5a69 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3460dc1c2674fd67621c972e260178c4c07e48e1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85f115215d2fdda860338209214e08e1495787a370a3eea4546c621966111474 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..574bb17e267198fa7cd42a034b3fa2273b1dc7a4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3e2d71f51a7fb40368b7d9785db6dbad577406c32f741c46cbcdf0151b7363a +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dbbc9d636243671e71f70712c93ab31b0e2688af --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:337b6a4de52e96fc322ba74305b8832e7bb95c9510823c918037edf13aa4f59b +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2abf98766def84646e319538e27c28ef28ec5d64 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2be5009c578247ac8b99d3a04c29dc81970094f9224ab79c7389c78a3f42c9b2 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..49d9871c3278e87c92082c6231da625007ed9d1f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad79556ae3b269db5b329f7039dde202767b7ce45728ea4de7926e383cd7d2c7 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2a20837778fd69e04f5dbc6dd5a791fb2a88958e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11384db08ebd83c8ae4276edf8814d8f386c8c40b2b8fa1756e020db6ee2d6e8 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8051d3a0ee350df14446da799260b6beb8d51fb6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:189da38331b1b86953fa3a7796b764b1ef4514cf09dd253a4c40d4a088555cb4 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..80aa01439e81a76fac5566f07c2d1f3b4c0e82b8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:051e789acee988a920130f05e22be36cc59b4dff77d696b2466c2159573a8ef6 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c5bf8376670b3e4c76fed360b3bc0e8c8b2d52f1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac1cf42fb2634228c1fb5a2d0cc7a8f2de283823e6b9c18ad84ec057100fe9b1 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6b57883984545ffe42fdf5e197cd1573e6794e0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd77be5393efea7386679929b32c18e1a3b9e46a943f455ea83cd3b5c0307455 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..36be4b11157c845cd8fd7603c3b7edf3dd87c893 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fce6afa7652c5fcb6a0e1ada4e63110a8c5106b9a7e4d04ad3331c18bcecf64 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d9938ecc1f5d10acbb742a48900735b52c947683 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03098886d70ed8c31465218565780d7c591bfbc2d3bdbe316e921865ba3908bd +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8dcdf90c8c227d9c6c4c67aced571bdbb34dabf2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88624b7f55627ec9b6f0ccfcfbbe018717ae772a98ce83e2a39189b069efad6d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4499a14dc97b951b5bc0adc2c50f0f785c661a1b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac0d7b434640a64f01d1cea2da7ffc4eb68f8e3fddcdbc9364fc37ed9e098e8f +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c29daf67c9fdbf5a5463277d11dbeefeea7d2cad --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af27262637c8139388a8269e676b832b6e7f981096646c104fa5f7b012f8066c +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9914d711cb27e7ea01f91bdde0a9175d0a75d540 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7805ebf2405b733bd04805ecba67eebe5f67ab355f3286a01c878a4d163d72f0 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..479821df104df984853ff8efdf08e791f82b15f7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:692bf75643a69d4736e93f22e7ca610c0ee97ce52eac8138c0221f3aca08933c +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2493392387d07ab3cf9aa2cd8fa48539cf76c15c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f538b1aeb85e94a4805285b7ab78ebc3ccaf77d87d0b7dcdf4c65fcfc8128ce +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dcd8a4d40518814308b801b4dca5693acc9072fb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce69005487b688d5bcd5f9bd43564f2863774e5bf32d18e3823ec78dd5f82148 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..430e096c2db10b020c5ccf362a1a6574cc44fa2f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09327448a9a77fcab3cf6e01f404cf91439a327fbdc6efd7ecd3914c06d5f9b5 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..137e22dbf061f1cbeb6b4d5792a588497c6604e8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e480a051851bdbb98cca400b330d72c541bb8caa5cc82b132ad5f8f4fed570fa +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..80ade8a795210636570ef69d74dc0d84f393620a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1111c0b6547b24ba0a716d257d89b21922f883973d88f8c7eb191e454a0097cd +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1ade5a4c9936ed6c03b1150aa6ac9a439b781bd6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71e7bf71123447af30aaaaeabd6658634b86e06b612f3155d0dab1eb5b9bdd08 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..acfe71a38db008b3a5a80d56913794e5955f8071 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:277c953b49fefc35553724cd4c1fed14fbf0b4f6d2a43b708c0606d3cd716efa +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..19b4510d809ac1423974e3b076832aca04d1dc14 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30e45f90a7ef6f1b1004ba1beb03d7a396597bc73bef5ee76c1b09ea39cecad5 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..85323b2b89f7b62f2f862d416488bb7de492ab7b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dfd46a476958c890eef9f0ad5623bdad4f1f8690acdf8b6328435a405635a5a +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0228fd84e2ac237c91991b43cbdeb373a3fc0532 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd32dd54654ba676edc12af0c44091f13e1898b7a4843596d2e640ab307f4b16 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..de8bdaa1b10faa553d7c3c447cccc77287483087 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ababce7ed9cece435c36fb924725671d999d35ddd98720d614cffb7802a06e80 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a65be330fb098a7a2950be6ccbef148419297ccf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09c185ca1290b75873523fd76414a1ef094257ad59f920bb7a5fb659bcde2150 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6036c95e401f6e9c20a124b0f13936755f18cfb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3bd7445f73152ad9d8241ad571aeb1cba0cf5de525e2bb55349629058a44880 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..649004fcd9351007f5a2484a24837ee7ed1991fc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a37d634537f693c84706fbcd0f5acf6e567cbc5aaa5117a640b61cdfff93992 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f673a5d1204475546463a6b9e17b919f55fd8afa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74f0143791280179536b3fa681fcf5af5cc76e58dfbf6a67a307ee9ace5f57f4 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..824f79f6340cd1322e9a0b08797175c9d1f706e2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2efbef0201d2298037f3fc6356aac70279b0d8b9aa0d8d5ead2fad356cb7028a +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..973b8331f552d885220b68b1cec7b4c152170f2e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3a4673860ff654df0a8497a97af0f2e4d5dbebd4572ca1d8504b9d0f37102a2 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..645bc4f14b1c8d2c6eba2a93aedb0060c926c3fd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e61b2680d968bf94bea2195989c01212944179ed85c50b6b89d7fe3a03bf3ea0 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f647d6300e6c9f8c2e64d4e68e704d6c16366661 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8300dd0cb41ce99205ccf7ed96adcb1bc383ca2c37d679de5cc3ada87a2182d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..caa48c041162ac7e8da86bbc9b16df7839eb4cd6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94b38c58e33ff85ac5972b3cc7988b67a2e2e603b8b893471103999c0975150e +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..026f87b4bbb2cae2618a07869e5f34988c4e551e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a861e2b08b532c39f0b95917ebbd28347ac610cedc105e165f4454da6af79e6e +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..643db3021a6665994a9cf26c053d76d92c5d63e4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20218ca83272dbdf3072790cd78d514798d8fc57dfbd15cf0476aaf511350261 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..70ef49686545daa55c23ff3a3d98632456b3c407 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f7e38243c6d29fb83e5049882e1b68f360a281686bedff5aca41aabcb30c90a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f631d6e0492c742d60445b5b341d67b22663779d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72345a210508d854b4b99a358151ddaa4db194a2fc86bb0f4265526dbcd8f160 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1e638aa7ad0a1a8218a95561ae44678bfb2a9bf2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d04836b3b75803639fc0f6df6b716e3915c4f45e254d1fb51eb905c996e9e24a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..338924b823f7d66d3cc0d7b82a05b3affa3f95b6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a21631e835e17baa73d47446a646a72e6e509faba1a50e1533ca9984a4e9e8a7 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..18f8f5eaa2244267d3aa78da568e9a2bdb172953 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83f753e4fe9d9affa71881dfa175768ea9f39f07e2de94dfffdefaeac257d624 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd9adc4e4711d97507e1df9ff85d9a5fea1a2572 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cd6557a8571cf82d59287e5626aeef666117217bcdadabce66d3a6a8b47a654 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..50c253634c4dc9218d290689dcd4ab45c5676ab2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0811311f03acf15b84fc12d32b1abc42d017ae498d35493b1b1787730c534d3 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a59d01138e7d4f3c76098ca59cb8f38a22f88ac7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5df45ba429dad5452e2c498210f87926afd50a853d8639d8b07339df5763d842 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f8eca908b86c082bada4bb234f0c21322ba5c95 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6120d0a096abc45cc937a087c0043b72692bd4609bd31c96600f8bef006011c9 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7e34894f8b504bfe644cbb14d5527035fa002dc8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:849c9053a37f71c57f3186f75e4c3a6a18915539fdd5bef113e37e134d021d16 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..93857ca8b08c3d866dc1a88b20ab7381309cc87f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33427cda1b57b891dc3c6b5445c405bf15871475a1f1950cd93cd34371876fe8 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d4f195bb1fdac1e8321730cdfe0268c72b3a4477 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26915e0545d89ce520e565b9bad4c307336435968b99f4ea976d789f9aeabe62 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5355ab75ab747345c79014984af79e0b1d9d9329 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b4863aa0cdbcf6bef76649800a0b5cdaa5f401a2c9afe0c7480582e130def0c +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..94d40ed954ef53637912701e2ba3b8bc4d7b5621 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5eceebffcdd720afa44c0906ae1e2ad658383f2109491db873a14dbc26e2121 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..94bae4226752503d52a66fc5b35a7e87503fd303 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:513a760cdd64fd47cb2d1f6ba5db838ba6f654c8564cb956bce0ee201ccbb915 +size 167872599 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e2067cedeae80161f56f79fbd6d22c58750b600 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25633a2311fff081d98c1a85e290cc9e4feb7d71e6c67e6d89ad3ca0ddf04e2e +size 167872599 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a42a0fd49321f7732750362dbb75706e2d87bb34 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a58015dcf31a4b27b05cae481e719f28714e6db19b48a27d41a7edfeac28289c +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..49c635efb82d6c5591f644773b668f734dc83a86 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:370e8f1dc416fd1d9f7311de1110070d2996eee21056b3edcef9063fe3bd4dd2 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..faf3eeab257924c672be3ff7ed9be8104b2e117a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9d47466e64ae446d9279b6c840f1042f56d1d5c417f860d8d56d6776f0e80ab +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aeb9bc585b91dd1d6808b57d2aad0f71351203ab --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c964f2d4a2a5cfbdd6ed9fc3ba58cabf082b9d95db4fee5e0588b2ca8b026837 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b37b70579865dbbea2027d1e1e109ca46e46fb61 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e697cab1c25b08c8aae90d218844e62adea925423c8b26e1e32f6089bac8cd6d +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..438e1a5f99668c6d5f0382e5b59d4a826bd98df7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b9fd5b7f09fbed251877ad4818528243e20c3c2e71206d264c9c803b49567c4 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..94afb4d037f51b1a420978c132091afcabe3809b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35c65bac8f5ff97310b6c8b69dd9bc845ea416e0f3e76bafd425d576dd567b2e +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f4cf57a3e0b33650e6539f3cc77a38a77cc3031 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:123d81ded06eb7e32e66de39aefb32610201fceb5e0faa3e023eb605de8f9ba9 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..77c3f6615f2b0d08aafa9c0248e1b26a291cbad8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e63145c55d5fb6eecd1798eb49ca967edb1638489054fd86abf0d2691c1da23e +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ebd201b3747f5ef677f2dafa923056002cf3bdaf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:750f0072809e2632a095491c57514c295ab4940b43355b213e6e47c34903af88 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..324c08aeb7700b1f1f4762e4218590f32b1591b8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c7ce79cf47fa42c9a66495bc77303d4f36520471a56758b50a7f44d718a01ba +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d68d07fccbfab48f2345321a50edd3259e4c4a08 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3592c57ae78feade2fa5e2bcb5da4f892a8777faa6f7a04b1b55885809aa792a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..688b09ea5c4e20947f6673da073cc6864b6cff42 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9a0fd78d9507261b9e8eb3ef2600a446d130b71744387979c59e612bdfecb61 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b506019120ad7dcdbeb73ffa221b04699dbd5ffa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22dd13df0fdab5642096483e768243fa7135c28eb91eb4004b2a95fe8d7ed512 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3dbfe00e4c3e7eae414400e44703944f686496e3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e039e7fd281f2dac5ed26176bded4a32a9c2fb16af6043c7848d1a74499eedc7 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..533d87a2bedbfc3e31563b5f7b87f8cb9b9de93f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44a29bb834f5202f6f61f61ab95a07bfb5e4b98accfac48e953df6a6fa65170e +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..91fce18d4f1733cc39b51be52bdc584a0b9c3b6e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd60582088c4f105acda6ece96a694fc6f14ac6c5935852a6899438d6807df1a +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e716af3486d3679d9e1dd80e7adf73fdc94f0109 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65a97e55c0dc0e47460f50006d156e766ea49bcb9eeef5b9e3554c9570755226 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f7132492f1fe73d8d4d3bc7b4c6bf0d04d304b1d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4769adc5748185f9a4886f62e120373c5f2e35b3325600dc674d9b2b7fd8578 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4140463dbafe9f04e5dab91f62f5ef0e11302643 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a8e6a2da3e80b37c81ce66247713d4268e4919fd763d1be7f38318292fc6c5b +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8db1b80ab5f0f48cb7620728090a6092296bbcaa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd8ae8c3cb480c1e678aa2c447743400dafb7f88e7c9bbb803f75eb82a3a0e08 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8d45e05b706cb509fc0f1a35409e02508a4ec1b2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:519b4fba21cd347a527ce805b1ed68c6b53a05948b764b92ef272c44bb068106 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5dd368b79bc442bb7320ad3b4aeabf73dd7bc838 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfe3488cebea36c1abecda812bf6328d54f0a72b90088be8bdae68fec90d81da +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5ed888b136efcb82a2698a594a478fe2a391e49 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5caf2cc4ee2be716c76afafd5eb9298ec787ebf6bd755cea2edbf2468c0cd2ad +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d347409f61fbc92f2c9f29c4c9cf18cc82f2800e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2b3a65719d2c12a8fb65039658ab6ad531ff942b8966991dff8cd4667e74e71 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4352bbb8c47f7967f561fc0c9f0a518a5a2109b2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e5986f009e6e71e916aca2cb41fcfd21412dfebec54f756992784ef58fe8987 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6ea84e56bc0ba87a96c38568652117744ad97102 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0cfba756d74df5ba8adb693621f659f4ba23a43457b5c87e43a7bd930985631 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..20a2ca5b37582af953d5752c547306a15f839753 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22dff1aebed7b92fcd1cbc254db25b8ffa68b1eda79f1f9d575b45921b2f4e69 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a1ae7cbc12c1ceafc44f5ec8cc53d9bf5d92a4e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:489b8289dc370d27385aab647365ccf759a46998a6d5060eee9c4ba13f1ccfa3 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3006650ac4f9952eb637409e4bcd645c90cdd3ad --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e98d02b18a7b51f383d51c1c7ac533c999e114328945c2ba7d57f7310716147 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3720b37eeaa851dbbe6dbdc80a1141bc77bf05ec --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b0fa2d4f2150c85d88bf9f65e2028fa2fa73fa0408abe03ca68105ab80cdb52 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..db60cdd398eb948d3279dc73e7a7657eceba5b1b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:764dbda73162f6726a17f90d9ce049bcd87872ff8a4c1c3e24b326caf6b659c1 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..91f8f0deaadd0f0a402fb8dc3d5c8966a640f7ca --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa8910f9e21653a7384d1cdc5a4c9fff35219cdc07599b10b6cc37975d8ea5ee +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a2e91ee9e2b595b1f035c6de06d603d43325f0ae --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5802dda74e35f0e97912cb3d38092942d6f05d6f230fc43c40a925335abc93af +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..38e38abd0e8cbd3ca1c2374dcd1ef4b2321abf03 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9e183651053fdfa09845679ed4dcc31d669334dbaa688b4f4dc8f42c9f665c2 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c0b67ed877eb502357e36d26206da3ce4f7e5868 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e09638bade7704620b57ca6c59927535f8e650cd837b2404c824be094223a3d +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..10942331647e558acf1f721eddd358ba8cfd94ea --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42e99ebc20e08581fa5abaa32b7f83d65afc9606b7b704f5c075dccecb3b58a6 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e6a647851534b0b3c583ad4481278c5c414b1c73 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ef004aae8e821e8319b8aa194a15488020c395cc5a6cb3496cb9772cf41965f +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dd894b6b9b14137192984416ac3789dfb3b1cbd0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73e4302962897e0f6a702db0c0708ba259c9a2a38e73b0160dfd1bd77bbcd252 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7dc1d76b988ccbc5e65db15f7ef10d208661a569 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4936ea8bd70fb5fdb792d0522031f0572a2b0318dcd3e7e3342acb47f4068719 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..161fb7ef27d7befff20550e44b0e361c5cbd0db0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d4dbb82bbfc1eaa78435401030af0e1cc096d3bc88f634f91b73b5b5cddc8ad +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4926eed2b802b86b184811e7a7628ea387d69ef2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0506646c0d0b72b2e6fca3f93f3f8ee2668a41604c96b3d1a6eb982d01509c0 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a4d571bffae4876b73843d577960b267a73c860 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88a86c2c3bbd46d09add66cad076cfdf14c4e0aa1214d2f535b2f63d226c88f8 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c1f9e9daefc52178a29e64b3b2b3474e94c7e39 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:802b88cae397fbfa902d374456db66646d73213b6abe1564da234cf114a7ab74 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..316e4a48b8e9e0dbcf1839babeb797fea78b5c41 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a05984a51ade1a62ec9622b26a9c39f626f652e9e9b1e56838ac717321206bc +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8d1a386466bf3fa90b481ff7912fc9ae2ab178a9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe66391d2479333f8ee52cbad8a3e1302061f8c461a780cf45f9778f2ff60413 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6d3ab04a74deba3ab9284b13b4889b0334df3e76 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:305e5f6012a19bed1c10836bf1b7a971f13171798bf66ab72cb4e237ef4a53ec +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..991e34962e8cdf5b68e5b491d078960ce4c99f8a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5da1246e0ba91cc3cd0d8440e740c22236252eca7d19f67653f18033e0c379a6 +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e89b920e04487e8a813f357047cdedfb67c767db --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d24bcf704b69208e221b365ad3dc25f7811e65d54411d32e9b537f7259164f7e +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0aca925bfde6948c2752f3eb8196f9ada973a868 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:848069862980cdbe5473aaaef72e2195206aa0ce2a462f011442fffaefe2aca9 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1551a008b70d42383042e338d1e527b79d63db09 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33e42206f0529fdac8f8777d51b0f8525b5b881768f6589d5dbc2a887861bef6 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..88cba09aa33dea82e31070df416a96821cf62584 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09db5a1b44527e46f87e3b7eff2eabd02f97d56ad857e7d24ccbef13dbc123e5 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..02e9355caf5151942c09a90f93e7463dae1fdf6e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3807c7517babc6e5c72715ef9cef1152009bbb7d1c627807450fc57f7f1c98ed +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ea46a7e5144c08c94e22eaad603f1ac07d47964b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40464c03dc0d097486cfa9dfe4fde77e908311c6c86b228aa168262525ea2c5b +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a30dd2c02a2e4a214de1094a32fe5ee6fff54675 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dbfbdeeefb5c2081223147567963dfb1c14810fb47e27c063ef7fc4ae416fb1 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..de3815d48096398f5c6400b51d4c2eab5e8cfc60 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:340dcbc4467931d46e11ddadb1d47a3ba904dfc55b9c7f3e4be910f6f221ae8a +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..05459cd268976d75a1d4c7593d0aaebc4aa3b412 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74c3beb93bfebc2e13b2a929fbc1e4558d3a42d04cfe050f178656f075d9445b +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a924ba45b802f63088af6dd676fe34216634c878 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81a0d8ded15acf9ab9f02831836c263ae111aa1eded22fbb130fd5432876f21c +size 167872610 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d2386228a2d2ab5f25cb0182a8e5dc0bb60cfd4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc38c5d42e2643d23804a67c6cd1d56474d318dd865aaa099e2ffe3469a754aa +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9856ba44c479fb5b3eb73f823ab53b8098f1d481 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9be2b0bda6780b0adf69f288db1098468515a2fb22835dddaf15d6ccbd105b98 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1eb1468da68b2032a7b761cf3936b2e819ed837 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e756e6cd71282c5883ece3008f59f381ef52da54042388065e9c801b436e9408 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..07213d8daafa373456996d0249b549e1c06132ba --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81dbbc08638c908e4f633c20af60f2f9532f0d29a014b47720417691dca00534 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..000cd52c114f6ae22194282dac086b181aebb1f2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8520c7bd7cbb01d5d9cb2185d6a4745089c2dcda1e84b51b02806f7ce939dd1 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0874be17062e991cec9f42e58e9b86a028bbbd8a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3919ff4c8f0c171e3c52a7c16fe37d3bbb780d39d5753a89c236767a33abfc0a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a88ea555536b4e9882ab095d2316575bf7153f29 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8008a5ee9847e22899df7a61180372b127f7e799ea6839c5ff0f7e88e838c8b +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2fc2c7affbdf6d0ca825f10d417672dfdb2bd59a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f62e1eea4c4f44fcbac6ac590f6a656134d0c2a1b7004abd350a551f0e26d3c5 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_01-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d0c159cf07af9abaa9185fefdf1ccaf480b7e279 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de26b6af4281092f62b1c905325d63ec245875faf13c0f3a981ed663ffd63e4f +size 153552131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_01-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_01-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8d56f117cec92b7b72a64fcdedd8864e53f4f9f3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_01-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cd86e6564e2c6058491c5485e80e8b0e11f2ae8c8827117e3329e27e36eeb91 +size 153552131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_03-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c6746e644c49cc0dbf58a0bcc74318a72098c88 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6943654441dc75f13b00619fdb266077e2c1216b0c1d862686a96f0c44e3db3a +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_03-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_03-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..21df51296dddcdf3d1a9a73af3970b67cc4e65aa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_03-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:844e773de34516dcf91d565b560f4e7de691ade328cf5c3564dd2edb816f1cc3 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_04-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f43fa90fd9ae6a7a556c66abdb3a8c2b202b61be --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd89b736e70622018b69a258b2571a7f433038a3f767653201937dd564326a33 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_04-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_04-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..18a4ca1224b6798aee1944345f81bbe65e03b390 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_04-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2b560647912bee17d323d8c11dd730fcbae884f82af3e5d53da3f662aefc6ab +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_05-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..45c7032317cb94be43a75f141d3f7dfdd50e4d51 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54a893afe576aec7d490cb5398e8e51622a91b78688e462ee6dcedec73623060 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_05-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_05-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..74c2fda5b52700ea89a8d880e12d6dc245c26c53 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_05-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fce5bcf4c161e925161fd397ab21f5716ad461346be9178624632e03b052244 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_06-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..55e03242ad47c2dd3ea875e350d0df03dcdbbe0c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f175e5920dca72e868ac8db5ed978262e218ec9fd892bd8a584398119f8f767e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_06-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_06-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2927ecac01cac4b38262bd401aabeab1c25d5fd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_06-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:458ea1ed386c45678f25c2197fe99d370fe820be94cdea1e36e6e7e867974dbe +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_07-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e032dc727cb296cf334d3ba62c55ebc28fd952b0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88ded85e3e52e97f56b07a535894fb2a6cbef914f62e75f015a95c4c8bb12e1c +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_07-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_07-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..721f17ce803e0471edb20b6655d5f3ab41c0c141 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_07-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4565afddec38b4eee44063e615aedd39235c73020729b9326406709e37d53e03 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_08-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2dd738b2796555919e32f6bd6807c641585cdf40 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07f33ca84941202e23b1bf4f687d36d7c2c870e605013114a74681342e2f4398 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_08-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_08-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..590f5d68327f1c3a5105fdfe6ce85cc083d14eab --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_08-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f29594eb863fa138edd63ef667fadab037d7834d76a38746114530d99605a790 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_09-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3cc1e61e21a369453a09a0c56c3dd4455d6fd8ef --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa6149e9c3d5a38d07e4ead8d184c5ddc917c449c93fe91a6ab9583a881da26b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_09-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_09-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..08f4c13208695adddc980e0b65684f28e2d890e0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_09-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd05f50f621f850525a465d7f56f2c6f08ab7b91ef54cf97fbce45d1d3c966ac +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_10-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6807b45a09b8687669b9d018780b41fa446ea71 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bf51113926128ac8667ab21615cc729a8dde29258503462d28dffd6538afc10 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_10-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_10-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2fff408c83690296bc428fe2dbeb9fcc42c725ac --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_10-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dbc1ff66032be84006f6d73846e588411f92dbae596bd0f16c5f6cbbdb63412 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_11-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b9bfe844018a05a5a39c4dfb574a35568479f133 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:383a66bc98afb82e363274d4753a510c723f73bd0cf66c91c27f12d060a8fec5 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_11-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_11-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0de22591c35eb0498cce7a9ae57a4afdf05a8a9b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_11-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9382ae82ae88688c7e7c30f75bdfc1432b640f01bb6112d6dce4a40f9a89735c +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_12-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7677c2fcea147e014df1f09f9593c51ca4784a66 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21697db250aac998ad0791537594a067e629ad93abd76cbb3734ae585b7e67ef +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_12-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_12-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..84746c6c1e82faf846bbc354d692a186cd41e158 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_12-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b8b95d007f0f4ba78e0b1f41b3114dfbde1dbd390551cfca0cd3fae8cc8c4c3 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_13-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fad36b3740ea89a03ba2dbb98e1bfa92d50d537c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7aef6b97237e1372c9b2f0199005cca6d8fd286f5f82d12155d06592e402379b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_13-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_13-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..185703c82fc69f2542d99ac84531c993bacc41df --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_13-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56eaa1fa5cbf10145a760609ae7cca4812eed46da8103be60caeb410f003ae3a +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_14-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6d1ca82e51b628dce0c0d0a656368cfc74e615a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1abf44769a0769b1657195544288ce472e7cfc48c60e582f798166981d2f2b2 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_14-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_14-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..09a27192a7270669cf556cff0923049a1414f9f8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_14-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6ab7249bf1b58845a812bd0cf9671451f9c5b64212633b3db5e225d65b60d22 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_15-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3034e781f36231d26b4ecdbab31019ba48149c8e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d789f06fb645b6fa89f8e1d23ebe59f6fac03c222b82a57ef29b4b88c1f2223 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_15-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_15-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e15f2472618a6090809f613c75d4c663d431a424 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_15-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1975723b0b88a426ca0ce665f804f73e1add3215ff457994f802e4ee1b384807 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_16-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..386810afb41583cb2f881da47c1da48e54dccd7a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2b42117d98163d927b94ee9e77b7a986339c164200545068e6aa9e4220ecde9 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_16-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_16-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4340e0a6074256bea7ed156cb5a3ad7abb921f30 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_16-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4ca3987d900a5c48e8c51fb1cb9486061e949f1be1a81ad24285c4ddbae4cce +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_17-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e8b2f4d603024df03265b1e57b4bb0c32f740074 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1db145410a14620c7a7e08e9d45cde7d6e16232cf382ba0130def59895202ba +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_17-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_17-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4126e5093b1f3e405a35341d598edb088164f6f3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_17-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06d99508496715972cecc05c75107fe9eac1d4cb7c1979793e1d090dc5ba2d29 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_18-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..73db39bbe16715621ea0a9b07849f1acae6aabb7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26a509e07bb98e751e6b2c41b2df81d9ff3f0fe51f156d44f5854e93d3cb7153 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_18-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_18-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2436a75483a0aee3aa53da1530b41534a31b470b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_18-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ccd6adcec52f07611d56dfe48613bb1f21414bffc2adc13fcfb26c4f512aeeb +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_19-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2a94a8724b10a4c75cb723c3c09b555a4ed71760 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:454d4dbc41a896aa64956ddfad214c8a0b23c6f1ba77ca65cccf7cf85931032e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_19-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_19-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3729c706d22ce3c6fcce5909d2f983530d3f36c6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_19-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b2fc3f2bee316fdb580e00dcba43419fa12aecde82424cd6db2a3c0ab798a21 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_20-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ba37ff239b3c157eff74d23136b344b07c215936 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10ad6fe429caf4058560a0fb7ede8c37064b5f88d6143213eb49fe145bfb9882 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_20-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_20-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..23ab9e2ab038fce3fb87761188c42c046d5ee776 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_20-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e627c2e362f41a5de3e47565ea4e758e3559ce6c193dac4b6fdc88d0dc19d986 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_21-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a654fe8b9faae8697b1f980823467ef85ee73b84 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0824d285a31327f8b6b6e3766346e7ac8282dc3e8843b1916464a9a174e7478f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_21-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_21-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ef0ce495307a2e5662738482087d26aaa33298b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_21-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea5d86a48115a901be4ff2168859aa8004243c2a002e8215924c8469f26ea3bf +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_22-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_22-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..55ca2d52ffccda0b0caf4db47d38b7d873da583a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_22-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e14f8b521c7cc706bb0a410ca34866c5f542245eda6d19d4b5a5931f2922d4fa +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_22-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_22-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a4bbc55cd9b35e5df13fbb1e5d6ea38975d2e09e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_22-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24a69104e28ea83a9859df7af96cc0d77522aeb61792f4c1236c40ae8a0850fe +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_23-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f623efd3a205a759ed3459fa85613338c0ded1a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59b3826c14b0da9158403ee3d7c23c3a887d5105d8f6e099337f65ec51c98a6a +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_23-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_23-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c10d211f1e530e8ff90150bd55c53ebce4e3090 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_23-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f09ede2a1fa70cbb6eec27e7fa27a77d72ea7ad25287dfe7e687356bc5ffbf8 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_24-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_24-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c65298acafb2e113545f785c3ac5eb9397a2588 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_24-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6cd070befe7362f896d47f4f011807513660d9a9e59a30ec988b51fbc2aa317 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_24-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_24-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1b7c579d4c11797f18263b0d0161c2de71b22beb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_24-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21ee5fdc7898c51c5267a1591b3785a43daf6721a8a5ef2e5c920509668e9124 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_25-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_25-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..23a07f2fc25ef65ef5190d08dfe78e20da30ffa8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_25-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50e74d721d39da50b2541700b516de951041c1dbfb61d165617fb052a68bb1d2 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_25-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_25-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..799742444c147d5e8554a62d16caf91b3d4c9f89 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_25-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d18ccf7893ed5b384ad68f5e6d627b1799f05b28fe19c7565f8f27db38178cc3 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_26-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_26-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..efa21253012272e69383626684fc30c4f9531966 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_26-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b566cb6b5a5a4948c9ebfb5b0b4a610dae59bce72816a1e7f86d5cc87ab4703 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_26-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_26-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..264f923c12b1898e2d89b7017a8c4a17ec54a8fd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_26-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50fc9421309c44b04f43670f33a9385daa7c4ee11b4d66bc665cc3c4053af873 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_27-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_27-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..946f4efc445d23f475a8db7405a88822c6fbf882 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_27-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:611e3cdc5eda39ec5e646b12bf24ef51a371409aa86c12109173426c6e310eb9 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_27-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_27-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dafb2d8b5f42f2473e4a31765bfbb27eb10e7e74 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_27-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae279af5007559e80ed73e4c41b91c02301a4f972302502a08e1cbb8994ee000 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_28-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_28-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c7eca957dd73deb027ea456cf9f58b828ea62c3b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_28-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:411881debeef660f24513f9bf9a91d7c79064bf76b5be87cfa4ed4d180d800e3 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_28-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_28-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..903250e642c2ae99d5589d574c09e2babc26528d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_28-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50c860400b3e87f0f89c7fedf82cde6342f7efc03381bea1f58554ba9583b927 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_29-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_29-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f92ef85d4c4aab7f783348d11f6bcef082a54ef --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_29-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24427b7118487fd0aceb6bc39cdc215dabde3030d2f56da426e67b56b0d1345b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_29-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_29-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9ff8adcd3c81acb2b1fa1d44863a14ff31fe7812 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_29-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bfa840409100274b85319dfc8b653959c512767de8d12f523486a0aae84aef8 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_30-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_30-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..148c734f59811596ce6463c01e9ed127d75e296b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_30-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6bc667d6263ab556a7f6390134e5048a4e8390133626c9aa29076d2c377adc7 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_30-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_30-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f7e9adf5f16e14fb308cf37574e6c4bb8b7f885 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_30-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5da124b573ffab3ceebb588de6e2959eb076f200a7d3f4f5446200e5d958636d +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_31-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_31-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e71be408ac6c7a3ba1ccd05db4a2836e3cd776ab --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_31-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f2f2fc0f579b67fc656cb58fa739ea93e22ae79083b0ea3ea0018fd99939e1f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_31-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_31-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d9bdcc886dc12b0bdab0b661c49bff88073b08b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_31-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:868668299b9c7e976da5ec73bf803a58ed7c0372a84d908db467d0c799799a27 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_32-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_32-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3945980f527330378fb8cfb8360c075b839f904c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_32-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8f29eb71598e9babf6bec16913390dd81d3bdceb5ab22f742add798006ad4a4 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_32-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_32-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c84bbede074366a8fa1e0e374c986be3e9878a6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_32-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5142c4f4f184ea3169ed90cceb0103a3f2f9fe8fbda211b17df4b5239800a789 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_33-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_33-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..11df0650185f307be99697326dbe505faef8a029 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_33-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f1bf10e81365f7a500797a05d92545ca693411492e249dc0a4d741146e9656e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_33-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_33-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9510b304ae135be596e7973b6943b051193d2864 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_33-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61ae78ba4988b5c0c35a94318d02234e7cfd12576067d0c0390b65429549931f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_34-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_34-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a871d10bb9f09d4adfdfbfb36a0fb1d0f48853fc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_34-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:528c58bbcf8818577e1ec3d613fa92c5711515a747e7df26d8b63ad65a458451 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_34-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_34-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ec8db58b5aff6281849298e2146b3330ffd2375 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_34-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f521434fb8ceaa8af37d4d8131c57492fca13bbb26d3b68d8d26e01e92604a2 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_35-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_35-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a8a037b8ad314d49a48d01b255409c20d70a9a3f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_35-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cda3dbfcf6c5997fc266c9ad1cfafad2cbc43ca5b4b35ea55937d527ab9a3424 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_35-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_35-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f140308871d4d704fa0362cc8169e5d99b4c7f9d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_35-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9499a5149ed49180f8aca6e32377e7262ad54767861b188a231facaf9306bc66 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_36-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_36-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..de0d73afd136ce5507cccb807cb48f96a67c2854 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_36-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18cc5f2b82ee5d00c72524faa5e7ccc164fff263f9c4ec499b16850861fce0cd +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_36-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_36-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..79e231b8e3fc500e61afe20b2fd89cfa9f8a1efb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_36-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ad2dd3d983fb665ef6261defbbad4e12f7fada8a83730343e5285b9c9a29d79 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_37-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_37-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d1f3b22394604253e3be9120a964d441d29bf32c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_37-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62938e6176410eca2cb43c9d25e28534a63251270457ac053751fb5c5e53e995 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_37-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_37-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..944a73360cc10358823b810b89e7ec6fde591a4a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_37-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0416896e903a66e8ab72b52ed2bff7f8ecfc2030cc29baa1ae9807a38efb589b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_38-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_38-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5f8a012290f7354b2c350b52d9e7f6be0ea6c00 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_38-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58efaf58d742019fb0933e0782a5b79466b16fab9beadc9c0158fa2e406d5756 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_38-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_38-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..444a3b805e8d714fb9b13101a40fb987d292d0a9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_38-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45c555e9362c4ad65d5e60ac9ea95c9009bd91a01c8be3921f14017bced004f6 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_40-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_40-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f8d20fe2c5e208ed1f8bf2573280eaa68a8505a2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_40-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83ce7a83fa3171f827e5bd3d98f404bd4582239feafd9841bdd941fee93c3348 +size 12483 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_40-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_40-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7ddc9898fadd6953e29e15db790a13c9ae204cd0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/layer_40-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98afcbcaf9952e8375abc2f9df2dca3028d316150eb62fc071b460a6a6b058dc +size 12483 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/mp_rank_00_model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..26ecba6c1daf75cd52837038784e25c3a0e175ac --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc0f0a57ec3137e89f1fac37d83c1dcf830d5f9f26b0583216c1db1d8d112d44 +size 51443 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/mp_rank_01_model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/mp_rank_01_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..118a170a9c9a17a86ddeda1ed8fe50695a17dbee --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step60000/mp_rank_01_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b666c1dc66304618a4d6e9a90bf4d3ee5fcfb49221d4a9b1342aff47489072b4 +size 51443 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..63966911989a6599797fad4d80bd301e7ed585ab --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0df73862b47e515b97600d2eb982f03affb9bf80fc7222ea9013ab343631972 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1169683df06aa9c695f64032eb478d7a86baab7b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9194f7dfb8e72887a27616884e9755ab2589d45e19373c1d66da200d1e5f17ad +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a986798f71358b105fe3c16e15d1b719e0de6a80 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e38eb12bc56a4a19f388ae6bac2cd74ada9c23d70595eaedc1adab086e4f1e65 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2bc3d49580f55e530129aee9c405f33fc4241b34 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7253cf2c712b864ac4d1268c9b8b1934f905223863767a9e59e5bd546bb08632 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ab7187c367351437a27adce2d871c6ab9daadaee --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d05d7fb64c32900bb9d1dbbb52d336f9c556f301275e0f885c956fb8fccc9b5b +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f68b89c2ae722b3c3aafe09ff485579824bedb5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:572bc22101e57086a196e2a11f5a7349880ba8893a5f32fa54efeefcd331ecdb +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6bb5de265a6d939303e710bf1f243056456d0e25 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:361184b0cf26a2b4c7474daaf3437d515554126183adc5ff5ff20dcb2b0524af +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c8e9cba13bc88d0b3e82131358995834c88411d3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:924a61b9909d47ad2bfa17868ba79d49ace146b70aec2e2ab60728ecc0b8ecff +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a9728e277003fcb6e5444259e4f2172d768b6ee3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccca5a0f028f1f62a1fa4e0e26400a210be4d8a7c163752e3297ef02000b399c +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d70131220b4f6631e0bd4f9b1e90c188a870d5fe --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfd83a4d0b4da4191a52ca49c74a68e88eb9fe9de5da540fd2c53575c5fb0129 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7df2c694e367987f30c8e97ed2bd5411f1891151 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1e5af035fd5378425ad90172ed7e3d4ff91a6faed66e471f3159f5c58605add +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1db3969e21a8ac20dadcf96aeb88bf6a6c054642 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0d12ede67716a912ec140fa0d40458222757f78f0c51ff3f5e4434233d6074a +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ee7c44d02a211e9ba6403b5288a8fedbebe618d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ec8c80e022274712320542a73a2427a0b5703fd3a076888709b7c10ed776f04 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..afce2aef7537b64a7ac93def2bd1545737aeb179 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb54a8445a8af8c1292c4042a7e6c184584e01a51b2d0e715e57ecb5a14131f5 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5959424882e9b5ad2146613fcb0eb1f4a98e619 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e49ca033e5e1504be2a26278c02b1fe418ce7a5bdfe7a80da8fa13ecf79bbd26 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..be9b652cdbaefd4064678d470a12890e74a4dfcd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f15f05291f817167228dcad932daad7b8ddbf5553e198a15cfa74489cf660d4a +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff6a74e5b07c423459efd984a076040d4efb2b03 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a98afd98d6af03c3454b62fa4e7fe7a1dd13fed41aa40c78acf91f0a5cb9b25c +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..df5ba541149836c8e4f5360a0fb046bbe10f229c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83cc58e8441dd7b2965c28ab1edbc76b566710afaa668cfc1fd21bf18e267be6 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7cec7d7a61293cf0bb30f7d88e7e5c6877a86e81 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5f4efdca85ab36c82a98c1cb0e083cc0b34f3860a66381cbc6b8f954cdde815 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ae7f81582881acb444f6f96743543c8af1a6447 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dddc630cdb114c08398d9415942ffd7e2d9faec8c5fb94579e594670e3a68844 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d8f13d44562d55b3aceb9b9b945d5237fe534cdb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b802fb63f440f012016c0d8b0471b03b5ebc83c4aaa7daa62c4ef4a26f71e3b +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..83b74d023c677bc29083f3e99b94289e37436230 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6f7838a40db70fbeb572b855f9ab7d44c8e7af943fd901c47275aa2f5d1e509 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c91e641543667afe464f306aa380c4d711a70b96 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01cbfe4287df40ea54035633c24b43c0570adb3139c30c201e3b1c7de52be643 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4b128ced418e7a25a3196b42ea1c35d4adba2269 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05aaf64a2f602a222f41b7ed6c0fd68ce2e2aa897c9e7d12bf4cb7a17a8aee60 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..07ac3d8b3f5866591db97a64b9e2c9a004905f4d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0980c3702bd3e0ecce1984ae540588f91c348836c2140f970a55606c1b615640 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ee1a489b05d5eac2a014c88ce45f0f158182b29 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a94e833ff8759217d9f6cb47b7b1d741cbc188d590c0aa5046a37c47d61a00f +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1ebef55b598f746aff5ea3bf10eabdab8ce03550 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba1bc3ca44fc73e57cdab49581d756921fbd159ef12f3df6eb3769ddb87043f8 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ce4f99f77f18131c6df25076eec3f862b2761b82 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:574c331b63fd979a48803393a75523ea564ec5c3281bfca97be8486b5e634754 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..87113f1d58d3cf12f18fe293e1f0f2ddfdcc46b5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f13f2e9347d1a9a361879e8b6789c32b8dbf7a7b1b688c79c56cc7b90661992 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7d0afc245952c877310dd2b5c88a98ab30f8ad4b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff7740d6d116f769816fd2fc8aba8f271a664b55507308ec61f10b55c7aeab84 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c148d657fab23dc405b92cf1670c57d6202597ae --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f34c3b1afee32e15718a4c9d84c9df40448d14d3528fc3c5f83f9f5c9f671ae3 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6ea33573bebe23231d1ceb585a7f4598d55b020f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7f70885dfbda2c3930c76970740f0e95d4dd209ba7e2aae3f892f289d7c0a28 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..17ab7b76517897bc790a17037368a5fe0e9990c5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:544bf23c2fc00bfdb88038195ebbf1d5fd33fa2cd09290d8e0df9ad751fd64f4 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..daa6172d0f15a5cf568f81f108de35e56d5e197c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e1c00f6a6f2ebf82d08882ac643863771da71b856e2703a2e0fe7e668e9189a +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a5d94f5661c6dcb1d78dd0a5d2f0ec7b4770bffb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2784f3a91d7f859175c5968219ea1f57c937b1cd4be4a4c8f4f46633640a859 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f38d8b9e4e9c475f71c9af993bb8d04638c9d015 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc3360dccec63c0cc83872c5656689d90043e8da2f98366ff088ed428c5bba6e +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..38d55dd57c493960c467f54e0ab73d00999f8dcf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2b62c7af5d6461f359b26a685b4b0059fb2a45b87a2ff28fdfbb91cfff43258 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b696188e8287e75639f854136beb43ada35acdc7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1ea92a94dd6f92cc94acb2e119b8b05c0af44e8390440e7ff24dda441754612 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc0105b3c424231fb34378a6e3e7563559d23e2d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea6cca9a03cbf0e29f8babb459ad5b904d0177b423edfebe3dd5b0bdd01f9587 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7693b67c929ee2d7928c73ea020b68473756be54 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1db2e472065fdc5a52d0eb7b710b83659879960d1043242a47a20a3723af89ea +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aa133f7c8083aa5eaf10cb465d51c94b366535a6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74678e6b3545f81a6d6f03808cc96ac4ea644c78d8e97a2afc61b688ac828a2d +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..24b7547084e2c24354ac39e0d619a427e853a15c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4728ab7b1de49a390680fab64157275220ff75c2a343e616d6c1bd79e5c8bda5 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..acd296ed7553566003171e7b79421995f6204fc5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:851dcc1726a05061ca8298911fea099c9546a5cbf23210d472f049de11fa25fa +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1bec620df217b2fdaafda363535a8f22e1c4772 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a436a19f80494800b4673003491eee4d8d7042ff37e6b6984b390b80a0a661bd +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c00a6745843334ece33dc3ca78219c0dd1cc383b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35cce7e4af375047476d1d8720e7caa8a3cdde2176f03766352dcb30c6ef3532 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e7dd24b7699712d5314344abf68d3c4d35481e5b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88aa421de51ac937909edc0cdc0b1f48ecd699605bdbe35364bb5b854d201064 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9795f5dd07674c7b16a581de749edc2b70ff2d22 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fd1979c8e955e53a0a23c1074c5f5993074ac0459c48ea14ac51665162117f7 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9366b39bf8f7969452257df6dc09838729c8d043 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9549c12579236b4525ec872d6aa4404b513d142dc7cae9e329a5e97ae2192946 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2a16d68a130cefe12b54dcd770ea6b83d3c2322d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5022ddd51f5c6ad298b5201e5cf2501b8312f4786b57be1f3a416b9941d8d5dc +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2be85e2d2191ec9419268748c2510c8b123ee075 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ab09a02a2249661a49460ef2519a03dd1881fc108a054c5d652a53a5ab1d712 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..955ce1e57c5464a26432429bb7c4b1dc2c1cb2b4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87677b6316ce7bd22f75a2757df5b0bfa0608bf033207744960cf93c4027c507 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bbd1ee6a247580a4fe2c157b9da4bc6f94f2abef --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c60f2e36d6285484dae64bdb8856ac05179dddbde81a1069bb1659ea8d4d0f8c +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf278d870f9ba50c8f2c2f20af590fd14857249a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f617132f3287f235c76aac5690fb7528934f6b7a89194ee17ab940f44456f4a1 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4a52ec84a0421ff918d845a8ea70f8641f8a49a1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b04e4a02b71de626dd9e67cf9f118d61db1a7dfdefc3eb37c0bf12cc2112b51e +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c5ab69038f6faae28e15f281d8d901493c5969c6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c882f9592d699d533207327d2e40d0b1ad669b66c1d630d3220abbf0aa1aab5e +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f9fa5076dd88a3e967eeab305288b495b117c14 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1205a2227da696b454b0f1329599e641a5fda6167a6d668973cb2e707c6738d +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ca948e3227c4c08cc54f5abbdcfdb7cd52b88aef --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d53e0c04b73fe901918866124ddda5d72ce8a8dc5196aaf8f2848864c7a502e +size 167872941 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ae514fab3a75a876b6aa8c0d8b0ce41940445665 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4cd997a6a32a5daf24bc613dce11c68c8594e1c6f248c4086eddfe782c0aad2 +size 167872941 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cbd050724d1be029a3beb6768e28ef2267a15ef7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d71bc005ff65fe9f4ba9b3d8b695ac5da3ef0d9a3a2e9feddf68bc07dde86581 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c496008a9e135b7678d571b109f28522a9bb68d6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0b8ca854dae4b6c6a9ceb6f17d4e71999fc28aac7c5ba9f063dee95cedd5cb4 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5de9b7b65105ec3577403987e471c7648d7ff5e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f313abb81025977ca0dc7f38abaf99f1932334b479cfd6aeba9ba7f544b0ad8 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bcc2d200e0c1bd72b672a16c420edbd16c3bbba7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79bb7e59d376a6ad574e829677585c42c45c5f5126bb5d64dc6c6e122099e16c +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..409b361622c647da0c1668f282d2a9944a7629db --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a93805552bf719eeae8cdd8b7d9f7d4aa90499937f700b21f37423574affecd5 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..96b3442654d14b92cb05fdd284116391a9b71904 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3293d152c26f172bdf0411c54b5990c94eb8be85697414539cb49d697238c94 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..849f3d6286994c734160a67d39286267e221061e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e045d56916ed41e9a57695d1af555b6fe34749545aa47a4636c2cb64a64b2ea9 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..40f1297859f8c32d99bd4f3d73a1fd941a9b67fb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23c38fd1b953e6f8df5652ab357fe926983addc99cd0887db5bcf68e5313a6e4 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f75d35fc0ab28e738e5e95cb7e6738c9e531fdc9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57a50f37cee787e2329b6fbce4c3edec9cab68e13c9ac9d623e1c7ecfe776f1e +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9aa72d7fb76d89cf3a2debd73639badee45a978f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f46d6838582e5f5856b31c66d2dc2fa7996aafb78592e5ebb2740d08d6f46c7 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..787640be22f5652fc0d65a998a4952336c9c6121 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22f46b57a4487026d6427845f6724b4d271dc060eb8c0bbfe99cac25ad73ad65 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5bb0732df74610ffae47ce6f146964b6a4067ccc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b51e075f236887c808b9685290899169107c714223ce7a49229ec65e31fc89f +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..da0eccc22036ec9697744d417167abff42c323ab --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d17eaa13c1a056201cca2ca8db81396f3c0a714d17b260181b612b144d8a509 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f3f5558e2d00cfd94aefc9c45d6c52fe89ee6fc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:756d9f7a6f1e2be7883be0a6800393be4eddff00df372da9709b52ad38f1821e +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..60d67064cb204c16da49c3462b50197b51a41a42 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1d36e94ac3255147ae5e92d8428496a95efb9c5df974e716155333f3c6869b4 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..573105845265e452e59fb391a39692f02460ad95 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b57cae737e38c972cc8123f5f2fba5e8620da89f7d6ef95a896afcf408b4707b +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..de9fd594b38e16621fd57ad4b5895c6321a7fe32 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dbf7095b44d85ac2838e0a20f135f1fe20542d333303e7aabdf2bb1541061b2 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..23c34073d93809923375385aa480c0cd52de7a5d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e659b28192f17c3bc56e64f83215492cc884ba4fcd0cf5c7254938c4ee9b68ad +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f3a137b4d4d88a6ee415b14856c4ef370378db4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:beb9fcfc0a637322a047f4e00610091a72a4290f14132ee2944884fc02225ac2 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..434faa22b1fe021bd940a37c134f2c6b5a2c7603 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07c3cb65bef9f301c58e1c9589555943983f9eaa0584b7f974d5ff25aa3ff4d8 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..70a6776938590353d04b45ea6bbfc07ef6d51adf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6545b347de0d17b685548e085a2102582b6e4682f8984713be24dd3b43011c9c +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..94893c9ade4aec9f3f41b532897f8785d07c7b51 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e470d2285227c6d391dd65187b67a3dc2b50b47d22f37b18c30be3edd02693f5 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f7f9f485b8ddc4ff046308cc7c9e64ba4577136c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdb04b7db2ce6e37427fd365ba4075522e49f6962b47287e0b0b741ede884051 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..163cdee4f1b63ed55997c19a1ebbf914606413d9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5262dfaf13e220f2b01c291851f6e4a317559a707585c2791c883ffd8f7758d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..db98ea962e7c5725b1f558124504aa2cc954f29f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0be555375432cb9a7330dbb2fc7c83c397809c6126193c74a50c1ff501de7ec5 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fadbeb49fa891b8c2ce97593ab44528f997ed943 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:235e88eb0d3c1529c5acf86c3c497bb2d7a820e42de91feffb99f3551cf80fdf +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e0b876a50262dc2b54f511538b08a3512ca23717 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0cafea72be4903ff1be62493fdd04f9e5d29ec30820af91d3d6e79402ffd0ba +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d62446688168feadd7eee4f79cdbf59fbae2237 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:258b275a9b62c3def4d774de42a235e72ba11702464ea906dfe55c9b44502a35 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7a3e0f8ffd26bb0cf48a754e244d7e16087f5ae3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d04806780b9f85f633d013c94aae109b02fbd3209c7493303e5f6ebfb7cd1ef +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..56f87aeaf75d2ea2fd8d0f637a6dac95ce9e882c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04ba45a0a579af52bc9499a0f435fcfc358f8aef7338fdcc658eddf5f6b037e1 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ae7a965bce24b15b339ce2bf074271feb5401d43 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a760985cb4887320bbe348ac58d1dc38681259b865059524679889425242b64 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..38d7ad2b283c22d26c13dc6afb72bf233a280457 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae2fdcff44ced6f8139c20be2694ab629cc50bb18d903e3143cdd5e1d525416b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3b7e781f78ce395926f99134c77c20dd01c1924 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e98ca55100e66ea26ec7ff5a730aff3cbdc571a04a8e98e86980648eb5717ec1 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ede14d52c3adce3e59ecf93d26d89b71fdfcf72c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f555cad354e906d1edb5c2629a00a5efe7be875e1680472ae56b19fa4cd71c1d +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ad73c632a0bf3c07aaecfb470b195bc425d7cf1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e21965e544cbfc780d5daac33a1aa184bdf543ffc8312e76029fefc48cd67c06 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2b882e6ba6e1321f9ccb6f049deabc7643140a6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c2b047fe097eb821dfe30020e2208d5f7f22c141598416c23347b4480bb3cfe +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7731acc170ab52b7b0a5cccdb77a3d2694950b92 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:faa3885c8a1e283633dd0d3ca31ced7436753488004fe1722f663f65378201f4 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c025c5410e13cceae56e50d527ffa189ca4f5bf0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02b65fae503db58f02f2f92546480a4b9ed1a6ea66e1cca3af9d5b8c0c9c8341 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..874e8f5652ca3d50bd95b9882771f125ec01c6a3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:939e5731898e2725a41fc8a8f2a05ab931778a76ad4fb5dc972d240dca9a711b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1acff27c53ef678f35d6d40abe932a9003a9a766 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad95ab6ae0a051a3693acd03f9bae26d984c1d5242de66a51f9defe6951609e9 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..741a89ec5ff5f4e9e6057dc93f84dc1decb7a40e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58c9795eae9f75deeb32ac122c8ea150b74467c208712af65d576ff43dee8888 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b88716c6a0a65829b9de2e47ba592c433abf825f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3f1d7a98f5451521f0a725c3c866e590b5aefa58e355b39538bae70239c7b1c +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..104068b935f9bea3a1a7c8cb47267b7d5ce8db2f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:194f46ea1a10e7676c7d644df4c65136a26c9edba41275755dd7d79a0b0ee2b4 +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..23c0c6a336b994f49560c6c5aedd75c6fdf5f04e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60d1a57f310aa970a958bc198e5dc8eb2f9fcb2bfccc62c6fda732bbf26749cd +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2e9f5a7055f1d249d4f78d460b4443930796b2e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8a210c03b629c978202e1c027aa8e133e8491502c144a24d1ce460c03004d13 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5bcb5b4504ed22b20daac188e627a62b678e796d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:715daa6ff3a0e3c04f3a7d5237d92538bd0fc9ece606a8dfe3d221d0700bab5f +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a497351b0e5d14055c457284dae75b5e697f92aa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75662eaa9e9bfa2e7f62a53104c27451072290d3685a029171df1432652ed33f +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..212bdbd624cad0170407b96b6a368e633273ced7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:777310c13d2a1f2f14436c0d31c31b851c8f9ed5c4c5d31db04ec61c485b4593 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3749ad7edc468064c4eba8919990bc1cac8c87a0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e70f7b6eeee3bfcedd19fc6021ea9366914697b986abbb5718343118f63f8aa2 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b3cd9ff0853e2f172d1ae55f039ff0f564e58393 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5eb38caa87301807b7de147e1e16c8751cc61e71f17f5ba1fa1da31c907fd298 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..56495c514a8afeaa066de9e73eb5eb0d3b316acd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:983d2bdd477aacae18132e35e1934adba932b600221ff46139ada99ed0cbf273 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f767eeae2530092938d5909d0a022293d1bc6c8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57b6430de5faff9ca6534e0408ad7211964f62be1fec4f59e3348a850a671c15 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b1ffaf4594d4487fc8b7a18184c2eabd5b5349fe --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12345a5e92bb808dae48a4a68e6e5a94b100d59af1c0f6277d5e903c19388fa6 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b5586256bb40dd61a4bdf9eeb4d8032cfb8367f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dffbc545925382f03aa60ca34bbae91bf3838ad0f1689fd9ac9574924448e0f6 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4210150a96a4572c666ba845daf6a812399d1d85 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f709f7c0679ca816a3772ef5c36f2aae4d1d62fc646f7e8b3d115b86e443cdf +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5a5715307e3ae7d175786ee467a7b15b29ed1a26 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed042c63862041b4e0b8874022624b5bcdea85440c8c87d98deb252cf7fa95ac +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe32e74edefa0ce990cbb33a2249e5ae6ffefb7d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee3de1e32743a76e1a3bfb64987606c3c4847e4be81c9b4be16c154d41ca194e +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..956a683b757caa309a2ce41679dde5cda492c340 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5137c46579494a3de571414b81f7039009f0bfda54b3af8f71a4a210155d2224 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d6ea78d6b34c25c31095a071a4a813739a0f90a9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c4ccf2aed74f7d393d79a627c18a61c556b9ebbebd46595473ba403a39bdac5 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f12c2feef608b5e7ca3512e78db3e1a5fdec82cd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5b3752d087c136c9e582f1f8fd2dde1737c160fc7442aab4119761beec21eac +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c5e3c1b10cb96b73fe94f72e8f67b7939649fbbe --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94bb259386a1bf9c6257075d04a80c307500e36c61778daa1dc068e7a6639ddb +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b3c49ecf56ca9087f97cb31476f02dd736194ff --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5057162d55c010733b506d259c1f9c3302666d3e3bd54a2eaec7af228ee93d34 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..745424e7fc4849d9463c0de49767d3697e0f0913 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bc1cc0ade3d88df0f016df0dd0732e30153cf581e43dc149918f6e9e6100be6 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2a66f2449f60f20c77ccaa77722a8c7fe1c3caa6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4115b5bc6e6aa642188c7960ce1baa41e4997246d6acf13977ee059ca559a183 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d1c80a799c5304932c0f1ad661b7c3496d0e2a72 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ddfe26a9d447335683057474a480e74c8826f1f48c742e157068a6f1b7c6e01 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3356b738975db9232fa9c6c2c614044a16a23a8c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a681aedd3be8d6549bb0da1422bbf33d2b038d425cdf324195bb1591c47df64 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..076b28a48d402286d13b00d6efe4947bce9500f3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2363e2b70eabe5bf650addc98ea365438d4eb0e4a8d0d7828c11fa9d5e5c813b +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aade5df34a7dfaca20fbee8f4d2a53fef67ed52d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ba12d7a6ac5be4b60e9cfe50f0af9fb3ffed31875565a0404eaa82c40096a6b +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..03ddccefd08948a8a2e2258f0924ce47af8f1617 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60a3390f8f879f4527c29c0030526e1e21b27f4cadae387ce4c6930a0e55e8ef +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd2742257212c1157f8bb3e4da974f155d611fef --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:761eb99d75b1fb6e99e3171d7dfb34d468de84941d32cd8a14383354d57c158e +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6bca5f2cf67ca3a0f75f758801f54d797efaa2b9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:532c1f3f9a46eb09a26f01ecddb846da55f50aaa4d3ba428496885d5fcaa7e59 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e4f081df375a08eca2d05ea12bef15e9946e41ca --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0199613ebaceaaad18de4def717f3521e6c957abbdac3b3799c74bc95947317f +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..539a536948ee063871c83c7dca8fed4492029723 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb73462d265dddaeb8c2bcdd403f3652804592565cd8de1a32e8503f996edeb7 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b186531193324426abb7945cdd488a960d611df0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c71a829dab82b9eb308e3bdaa7dcca9503204d89c61dc8b9ed58a12ee0dafbc7 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7d921b8367374b09059751f0854faf4f563185ad --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acb9a76883479308170e30699ffaf3bc3321bd2197f2a378e42a68d5388ad0b2 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..64ea1364d07e44ac27380e6391998c9009ee0f71 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a83352c81681f4de411aede7cd5391d34e1d150f4c80cadfadc3ee94c9606186 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..17837d051f3c70e6957d5465ea4982e320acb738 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9e58f7d78b02de0754a81bb041c6e935f6c0edba20aa15447233e843a48a44a +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5ea6c8975bff0e6bbb72ff2e824aa646bbde513 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1812c54fe40e1007f2d3bf5a03ef123b234d886197095be2eb554107ffd95b6f +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..90b697e130ff98f63e4067f005ab4feb699df715 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a0f8a48be957294f570dd6aa7547d150e064e973f366004b0e7d5d9408e2a01 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2667186c415495b01a911971107cc05c9513127c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c74776566cbd29a3b4505edc0a14811ea52a64757aedbb7772894e547b39fdfb +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..915950b601f1c62612e002db8a5655059a8f541d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e81c80f2083da55c02ae5bcdccff5d4269f1b216a256784077390cfddd5d57c5 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb2c333e76182c442e4ad7d55b20cb43b7116e81 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:662acc3e2b8e8a56bddc7a4c451d4f51b8942ef21c0c48a164a4d9076fa4947d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d44390e0d6cd58c2ec539f8550733ccd6decbfad --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fd358993c54267ee01909bf93a01f46ac223a896e47d7cc71d258db794d01cb +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1661bd5f1c1a4714c6b70a763855eb0295b4eb1d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4458524bf9572e8de62bbba6ad8fcea2c39d30a3704ffccb715cdc1ff6d2b03c +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e9bc2c6d5bb427aecb60d9e2eefca4a84022a70 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9d2cfe2f4297b2059ec56d1fe43dbec6a56785d22fe2ee94e2a7d4e831969a1 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a079450a79d9e3d912c25f2a7f981842095b69ac --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0160dfbe1cb187185ddb745e90147481ec464eae918eb9fc1b57c87633761e75 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..db4f466c4369dd5f6f6fe5eb4c42c84d83b9ca92 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49b15490a58a2928e8ba201baf32273899e0c149e982ca7d836fe6c92f619619 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3eee0733dee602ed5f882b51d583a546f0a2028d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:525e4dc819245f0e77b12bb09234c16134fc5bb5517c06b1b512d7da9fe1dc71 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e132ce11add9351ea42230282c6dad8460983be --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c0f5614a1a85666beb0660752d803cbde9337c9fd32cf25cc13f8bee8c46f82 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3d979698a4183a3252996cdea8176f96296f9796 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98d11ea7fbe79fac93a295bbc2c940bbf8cfec742f470a9f17762fd6ef0756a1 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e398c536e01e4c76af9ff62e3170f5a191fbf40f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20905759bcf851bc3ea6615b46da257868712234bf7d2d07e774e8ea993711fa +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..41d744085186d3b0d332ed548562679af4dfb67c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29562e8ea8d103b7588e42bf17af190275a00f0574d36f0586de2b25c168ebbd +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4916bddb1628025cf3b5b149f4e5848ffc95c398 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:087eca7d304b1313d82fec22b514e2c58b56315331a2db44b67b4429f9bb80a8 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e8c5e1e5463c7ab09aa0af72a979549d03eccb81 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39b2a838f1fdb3e9e4499df3aaa58d96d813121e7de3d3e1a56bc826cdebb2f1 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f958cb761cbf683b55ea61a18ed1ec91013231a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83166733e36fbff9c97a4529ec664de55332dc7b8f24df16e1dd8bd6557b0f40 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f6dc815aec3f4e2af2226e34c1afc0cb285d9b6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7535d9aac674a2a244c1882fd1caf1f61e3b833f037a1215af8908e9925fbc65 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c52361b685bd3ee5473c6fcfa8a4313fbf6244d3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1807e8011b1796363cd44990e972eb5bde3508899e757263795590a6e600e3a6 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a8229f6a9e502ad87be857af6a034719ab358a4d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9aeb31155e0e4dc3be5254c867ce57e773b974c40b688115dbfa3fcd7a3038c +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1cd5ab0530450d501ebc3af650f2ef144232ca82 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdb07de2dab927cfdc3d72186e314db95eef651f6a57eb4e12f4251f9d8810d7 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ae0bfe45fc721edef92ed05fa76405f5ea7180a6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78766af7420028ac74da285d0da200de8f709c30fcb8f9fa7fa97e990bfe65fc +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7af3f7329888a0180f9d3777667a12b87a0978a7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ecd71378f49b929e2e5a21e3d527535c72d1b5cc85851608a186ec666b02779 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c602c5fc16928b15a13801d1786fba523ec2c51c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6586117d5defb95b272698eff046c7a4a5ac43df0e668fa37b71fd0c132d8470 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8fd7a389cccd34ce145ad3035b07d73e284ddf79 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10666e3003d59c4cfa866add699605620d6f177611ba04a53a7cb044cc5b3f8a +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bcab81fa05075466e49f6db12121da59a818cb8c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca0e1fee1ad333c708206e6558425c56ddde4a4d70c57d26d20978a838f706b9 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..22736dbeca8a9de255db6142f8343832e670dad3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d7aa9a3b2b22cdef7391820a906fb6291793f92dc7866f9998212e34d54e43d +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b37bee129242ca2d390c8af5ed0e471384e9dcc1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61513894a96eed9f331d3fff6023dd0f3aea0df55fed79ee90efac37ab5e82bc +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..30d09e4054797a6585fcdbef9ba50f72aaff67c8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:030ee63c70e9fde8c48ef9b3f53f8054065fd6716d013d80b008817ead94ed17 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e4378b75b99405b892e51c718c0f6e8764ba9433 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e908a2bc00694f0e8a99256137fae914e53bfa176b95e169c338d868710b511d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..826edd241822a1b393a47d3114375717ce500744 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af50e7f4137dc0dc9df832bedbce618870fad1fac5903715240319bc15e0d6c4 +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..468d198c2807692eb600c7ac2616ce259e24803b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71c1889df6839181c257c797321e98f9172ae54c4b9cc2d90565d1a02c6cd298 +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1f0ce6d1c8da7d5928569575314945cffcde496a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4a093d2d8a845ea3287ac96fe9d6e7d9315635a055140565fb57094051e2892 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9ef5c30b7098d336a54ef50e7dbf61d7854c3611 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be27500235fd6d67977808c985643d89fd94c73a7aff7a9cf35fe2c4d2c7e5ab +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5818363dafbf2bd3fb132e9212ed19bfbf26f0ce --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11ea992cf0a3f4a0e5ab4b4e982be258369486072c6cb1778f28aeb6e4010256 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a2f3dcc07cb0a845b9ddb6feb4aa267b3cc7f810 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7878ea3d734acdb5b6cb02206a547e8130f41c178cebdc4f4513695ec5c9c21 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aca85f706c726a34f235579e273fa1aa64aba4e0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13eaf6503e18480ce3a1ee8aa2944cb2df3e5761946ed735a67dc36dae4ca28b +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..71e56563f9ac599113837829944baa6fc1fa7f97 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29b7cfa3f8b4f16bfee65ebddc1ffdf635a79fe81c1d1a99096eb4d9ad54c279 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0ba55cfb98e9a4befbafa5b010b36b53ed6fde06 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7706c1e8f0fc208a7470ef44cad3beb34210e21e4a1fca63864f6e75346cfff7 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dcdbafd47d668474aa1a7b6444a331f0c3cce712 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa82197fb59c514aca289370185bf4b010caedfcd36efdca2ba30b47b832d5b3 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc9e8ae979e96c18411add7b7235a1ee430f4ca2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d4595a2c37b572528634e1c89d75951998d91a072305eb6e8df310c744a883d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d64fda3553ca94c2ddf87f5f0a9e3e24d9df5d1e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1862b1caa8384977d187dd816862529cd94287a80a9c848836cbffd5e08b960b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..324a88aad230bdfea31e4d9d6bc1ce6ab9a3dac4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92537938615b38580082e6de456a8bbca6bab266bbdeb457a538555f323970b5 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3675fa7f90350920babb760f8ffec4ca6454762c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3c4834994e61932652861b2f947fc0c7fe9ecbacb4317fff8584937147bce4a +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bf027c4c3b84ef4660542569e592c51efd22886f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:200674c6cdd31ef8a2972c9e5667a693fe986f86f21459b7834b317c2e4f14b8 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..83c4447ab0f9cead79dfb693e02f4d8c2b2fbd17 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d83e8d4bc4e2d81202ff8b93585e3015b7c55a443330539363fc5bcafd1d89a3 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..deb1513eb212d956291b15550a8f72c8bb2561cb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a933e66c07a5aa0287177e2e1fd0c2c5948f804c12a2bd34e674e94af0295c73 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c047da9e59506091994d51ff13bc771695b7e9b2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:046ae62231720e44462d6f696e2e003308d8465ee59fcd49c02510d00b997bb9 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2dec2ceb1b926e1a0d2f6b4fb1c4b961374fa737 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1163a6ecec1814b3be067bc548383b53a6ae6632828c6d8a1be7b3263328fc4 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..49ee852707b143d3b045b582dc478c476b2c9141 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79ad3278278165dd570e7f15942cb90ce7631d562650672387a76a12b155a6b8 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..846712b82f8f5a40c3019c6a7cbb6a331c73d8f7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e776e706d501668f3daad92810d19919c0a49139847a01a98f83f5c5468f85a5 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..169f475e0c3a3209eb37f93ade609b1214044505 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1076455b040a300b882082c853fcecfb68ef56d89144c41dae57509fe3830db +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..924782981548158d7f5c7b4f4f521ed84ab3c2f7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8762a0e53922754955cdc860a6b9ee82d0fee60f4bc2dbf885638b01300054a6 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d402a73552d19c3235a74b3382515bc56c3230a8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56105d7fecd801eb8a76a287fb7d025e39aed74acd5bd659b6d13d3c84c7006d +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6f479d845d9491c92ba265b2e2902a911bc649e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5515a6981d9ec239afa608b98a3c70d9677066693678ad5199703fc2d9308722 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..72bb20bd69561a1aa43e983a8e0efd1a4307ad02 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d2b4bcbf0e2064274a134f0d0efdce84ed45b61b718489019334d9cb9931ea5 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bcfe0208253bf40df044a89051049dc75b750f37 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2d94daf477a9b9ffef50980cee209d2d9f9f9ce825f069ab2b66ad8644ea80f +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe59efe2186f460f6be7de0a2259627e9a9b0311 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6ebf64e9f93321a0db9c4972d057ec9854b8af996f88aaefce6701c396a7f1e +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0f683a10d9668e75e4e7df3443396c6c8bad561f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d80c2e66131587a66a7e6b97a4b16ed659beed7340421178373a846308140d1 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..435324d0225d371d12a5c6d63e935040dc6d7f93 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0d7a012f0d782cbb4d2b95b72e06366db89ecdce3170acee7333cd3403663f5 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..686a8a9b00a776ce140c41063536fc4b6ace1753 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32f338e1f4d882ccecf35e1ca47f5633c33831f7fdd2b55f94ca6db73e1a2911 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..27f43baf10fa5ff9ed3b9bbe3bbb87f7892b8069 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50cccbeb4b9f4b138414b7c5137da55ae78f99a58dfbe707f221775790fef170 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..004dd534b909e718919d378d05cc4c02c4f0bd34 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5afc5195173b244c2c3922c636a559005136ca84a0aef327f1fa4a3e6d4f92fe +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1dedb6ec3282f4c2f5b632ebd17417d476cdc386 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23456aefea8b9d54298d5863fe3ee8372cd6ccdcd0c49fa48252fd5f4c225cfe +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..517f37959666f8c83189a7736fc37cbf271029f7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8d99e9acbae99218ede2b49142c46b414199e7f6257aa4125a0d549a2f775e0 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c0aee03b1fca0d2601a0817c3c39931f8e335d4d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7376b82b9047b1f016a034a287c4f6249cddd749e252d161eed5b6e024013e25 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a3fb2c496e96ce1b3c17d01d5fc59ce443c333c3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca9b8f5c6297358600900a187766746ff482508f6f4483a0ff3dcb66e211ab19 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f5f8513cc5cca37a410744faeab69363ee640690 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:204eb5c13ec25f66f04e1bd6f75c63b938cd93490ecabb51119234b486243029 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d693da9515fe9096adadc7af08955b20aa4bd7b9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac439a3c88aa5acb23bffdc9e9a47cc4894ea73b878322432c847a3f10ae7cea +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9cd549779ab4fc00a3c3d94d0b4be588b279ab2c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6153cf232331342c450379763f56c4e463581b14d8cd26d909d0796ba9edeb2f +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..449c5f073c102f23729e7edf5b52f069f6482e28 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f6f086dca38fee3277ecedbd7f7d6a45c6a36679d550c031ce869a2dbe9386c +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..89351166ba644cb773910f84d7e7ce6aa5d95b10 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c499bd565006acdf756d758f4a011b4057b4c5476f35458d42bb12cb04c3f6e4 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2b51aa818d0e2495938101d7e9ba1e59e3ff4e1e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45635670199e7ebace6aa85abbb0643375e85695431b94cc081f8c276694f177 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..494dd5877d9fc91822e2f735d13bdaf7ada69d2b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05fcad81fcfed7daf948ab1ac5293589f2c8c153f90ae3e520c098bd22b9d2ae +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..78e333f40fb6ba9e21601712887c70b24f21f427 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9744781af3446fdfb2e922f0cdcd63431ce0ed767f42ce874e5fcf6248a24d68 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7a1e88559a18cf14e8f968b8b45076e124d3546d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:820ec3b2beffba4b63784d8bb0611a36705fbe947c753a6c5b5d969c7ae55211 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..11b3061818ef8c5a3dcd35d420910b7edf3ed836 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbfe3811b4d1877c8263abf787819080b5e7b757082f143bb22116de1e7048e7 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a39282b28e22e654ada4e1ce2ce90bbef7c4e3dd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7584a302bbbaee6462dbd86502c834d0f0b5674f3d2f847d974f485091b19176 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a704e3b68e878006b1a24c74db47bf63d522c2bf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12ee32764927ededef04f5ed9e910824fa7b314e45e52acf1a20398a3424c007 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..008eb6122e9340f3d81d00eb4bb7144a669acf71 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20e4f8ac5fcbc77c09b52e353ec5a15047ec0865d6118d8a99c5d3e6a908bf6b +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e6e0ae2ecb529e22be43d64130d11e94b134d379 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7446bd75c5ee91f50213ef1e0c48b5b1473b074e6e2573392776d7fe25c4c7a5 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..012cf2f00476179fb238cc013a56e6ab52156fa7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b43b6840f41b1cb95edc665ec32c163b4ab2d874289825f971529c8177861ea +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb31abe3bf00677d0d5100560ef7fa0e4ba32b0b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57a9443d9b684bc9a5a7a53f80d841faba233f57250361cdf63bb6fcbe06a030 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b69a2a656e2f0e963b6759c9d25f6d6d529a7b36 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c47cfe659aeeb8b19515e4a918e1392ba93a7f9415d94613c3420fe06d292bf +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..21b12725888ab9c874b0a4572635a8e542fe44f8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab3d1a16967101b72fc99fb5f31b338d7a4953c91f788744e7508a9e733817a2 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..229570f030d8bca4d8bf3d8343c21b03b52724b6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf5a4357ca9148e9e96a866642b12d7dbae1a106f83ea58f5a79e2fdb6f01b45 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c582ee5b5e1d3f1635ff2e36457e8787000158b9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbc2678b29be8b19d0962edc4d825f74dfa828821fe5d66192f7ce50f223b39b +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe1ce333ef65479d26e9532e17ef629ba7999227 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce63d6fd3de2ed490cc0ba4ac0836bdb98f1fedd55809ed33532085a7aae8dbb +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..16291f8cd54a14ef70b77b2b796af2e7720dbc92 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b9a79456713ccdc43529a97dd3e168c838d6c6eb9291be3142f4581fd985d49 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..84f488c02e5b8854976799e7972721da489094d1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98202e0588a9b141d0e1b68e1bea217a38d5b7deda22d6e138538414df0afcd3 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b9eecbb1b2703f746cdbf42d5c5b7f4ccc6c1208 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9e588519f7f8a23833fc159f2ead24d9b1e35b43ab6aba9e5c128f3a7d7d3f5 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..44cca3369a2c8d6c6ff334b5633653ac300d5550 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:473b641f2941264d2c1eb7730bad0a52c269f23f6842f5856a6fddd1b1110763 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..01b95b95ad846c4d059339de44c51740faf27ccf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2925a000ab6d85c73a2ab3dde315935301df9576b51b1e5a9e162132b3cfccbf +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8bd1ffa5b8fc4f789df2dfc6cf99444bdb6e0b3f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f2cd8f5474800daf21bebf5fd3167a3cd0e1cda065b2bda7f5dce57d0d243fc +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..af1d7151e01e242c02dad815c2e6b3e0eea121da --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0465e03e9523de02283d1191e71eb42149a6177b17936e2d9404198d3d108d0f +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..caa771479a994769004646ac3cfa87ffe5a131d3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f275e205dbc87bfb6a56c79f192e1ab6b6c13757dbe6d3d0d006f8576b71ed4e +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a4e84a61d2cbd964b257a9f99d566b5557ac942f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7508e9cf4c32891263044dbc7ce32e32c467d5fc15b2ebced81a2744a8ca7084 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b8a8fd04e7bb3d562c02bc9e190e89d80673b98 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc3977ad487d7f93663e1f965c99981e3e549026c9d341905d01270c3a5d2d48 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..498cd7616d7a5ed777c29f8b4922d82c601ec793 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a988a1f7fd12bc73500aa01d6ced75e324d97a117e27799554d61d37edda877 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f7078578b45942d8068462a25fbcc7cba3e3bbd2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f44cd55303fcda4dc0606920bc8cc958389c8a49392412bae6b9eac3c9e9c99 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9370a4e01ffffc63da82d3c18664ec2c949e00f6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13a021abce3b1afe73f2c13c8a027a7b987f433de4b174ff68ba04c0bc0d207b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f4e9b84b60fffffa67dd27a79b48e91fd234ca51 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2193e6bc961ff2866487420c97d0ec7c360eb056e509d8d260d3593bdc299b32 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..87555152d8e6c9b01c40bc82a5f6c282bfacd957 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5228c891bbcb0905ca9dbd081b1b847e49f03ed963e1899c4356ef531a617ca2 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe514f9ea2f876110b1b98b336fd436f4de61990 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:558e99453db0b0c044544df45cd499b731fe6922f90bd0064e6fa81df3cbebbc +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc4c51a4ae064dbc4ec85f6ff58e2f7b911eca4a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b74a0bfec93287b0b22f7ce617e5d8733c037f2679076a9950f6546598894c8f +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f16c9497d50388d6493d9955408d8150fab045a9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9384e276e070d2f6fd40b9302fc3b6f1a1898907c6a0b8250a08516e1507d5b4 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f6b5ea07f5e25eb6d47562b387b1fb7fb81eb1a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13cf948b593d9d29adc8d444bb36a2d93fa3d3b7c155ab25b7dac1ca5872636e +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..22bbf1395ad265a1bc476471ee6f217768c62c19 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78b4ef60a75177e2376a78e8e902503f64844bf4662e05d812c54d3a4637edc0 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3295c2ee711ff5f85c6e5f587b8ce5271e646f05 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fa53fdb117a0872dd270f7398bfe74c54db2091d0b3d777db436dc878ab6654 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..52df33adff4cd8a79b01bb76133ccf32d75b4198 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5dbb7bf204633ee7c2e4392e7343de4d5f8f57c8cca58a4cb5a1f33898c2a06d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..17cd8f0d113dce20b9ee1f476af12ce491349d62 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a7e7f900bb8faf1ff8896c0a1f4ce5e9130775054b5724c58b2c6896cc24376 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0fbd7e8addbb6a66ff264393ac008027ef2f56d6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4acbf5cc069782c87a735a5ccd3b7c8af5012778911c6775f4d1edbe32f2962f +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b4d1e5b925681ca515d356d9036195905fde4fe --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ff97c4e064fdc5188691681dd90d501612324b769de05af56aa7bc2180c905b +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f284a28d01506c6a367502a14a20a6a74fc1fef --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea6504abff1ddae25da6648af78dd49651b8dd962be4036b9f7c162250dfa67b +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a3ca03506f2b8dfa43552d1d639535a18965a803 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac391597d0783ef42650dbde4b54714c21debdba89c7a7fd1e43aaa26303d057 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..76b6e793b22be690f026458cd35ec15577c6433d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a4cd9528bace95165a725f6bf39fe70dcda532a9df5c43dab9545ae84c28837 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d63b73e7bd851d278138dab1e9faf343b04d76a3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11505cc9ff6035118e8c0d3b44c02b1f2d209fec7439a6114b1e825c8278bca9 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9dc13d812b7ab2171654c18419d0a2797421db7b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52714490f860008b2c22cbaf3368e9f42bbc73dc8f67b3788d66ba6da90ff4ca +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3232c05fd1064adedd596157e092751ac45b50f1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d91c57aa7f6a89e206ae8c1090102a7cd7baf45fffda6a93f839cbd1c2190bc2 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a776f999afa2b25a200785667a85d7fd5eefc5e6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:195dee4064bc6c5f5d556d79fa89c3b09710a15de598c0d98c4e99fb78af6409 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_01-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..119f6031733d0294b79148f78ef1e84c239d0926 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f9e250f3b7258a7b7047dac19da16c5d767791b5c23c1078ebce7c8107787db +size 153552131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_01-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_01-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc88d9b013ae8408db9c4194a449ed0791c8c479 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_01-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ef8e4241802f65c78882bb031c08bd6ccf80fd29edfbaf6ba63f8eeecc45f9a +size 153552131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_03-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..335f8694e174e1ff3f95b4567ddfd98f4d7adc1d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dca5375da8c32029b24a9a3f69d2728af83ad5c08728c0dcee59d936152f9897 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_03-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_03-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4bcdb6a1b15e160c765b4a4416fae879fa0f3248 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_03-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:703579f981958c23fc32a24dcab88ec63ff0e69300fcf2c5b47bd00996df5483 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_04-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ca1fc8118bc15650fa1b052f06fd76bcc8af7949 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b088a679bef3e445d3f366478c9fa3fe1c8e9c84c8f8bad3c29b7ac3c85dbad3 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_04-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_04-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2296e60636f2fdfa0f47ddae7abb1d3e2b255bc4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_04-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9feb3ea6aa364b764d97c3dfb4d431e931871447e53373016e3881b4d03eb9aa +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_05-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..40a0e08d3426585dc60dbda452f5f168a5e81e86 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:353d01e4ab6c54751b3cc0e75d2a95cf4f8d68a2086a37657e555f79d7622c93 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_05-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_05-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4d66118edc4f12acbad6080b74278f4af3b56685 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_05-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7b01af49e9cdc3fc69f276413f2ad94277347be92844976acbe18b915f98400 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_06-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9e2757df7bd3c7c9d6153d4f1555ea97cadb2c1b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e2d8590f7eeafdf45adaf421fdb4d67548f0784cf5e46ab3ade5e81a8018752 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_06-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_06-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f5a33e496024183719ee3e5ab4d219a6995f36f9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_06-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c440f8dd19a39ff8243cd5a3bda9bd3f57bc6b514dd59712fb19e2dfd8e14acb +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_07-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4698f8c5f7709670901a602204067cddf3a863df --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1893aad242bf35c246992ceecb06e85f966d29cc8910acd174797a30ed750eca +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_07-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_07-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..612e9ca51d011ce90d7ea030f49f8b99805b3e72 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_07-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ede4a304116e08965e0a4a40c65febf77b8b9ad28de55acb34ba45f2c5681b34 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_08-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c952312ae6e4e788f9a57dc46520e9d0c771354 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a432b85280e04b6b7ab260ab2d68f4a042bfd8be51987a4b0320df7d391a8e8 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_08-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_08-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a668e38fb16444dc6b7ca937d255131e406b5a41 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_08-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76d84dcd6f73e7749aba9ed9aa1237027d198f955f520fba3417dc8d2ea65ca2 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_09-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..63a17ee2a40e78a008197d34a8e159110e0a0d20 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7015e04a875db4ea1f284868c9fedd8c7ae1dbb7be541e0f6d5ae58a0abed029 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_09-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_09-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ecac78dd3409d64249b23dd43baec7c8a0c4926d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_09-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:506ba15fce315027f9fb228a03e9eb435d758f7036522f302825b7892713e705 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_10-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f5f655d91fa7903d43edca43a0c9e542cf19d8d2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f0450f57f8116c55657cf00f07379f28b73ace02b3af13e68075b6cd5b44542 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_10-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_10-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b96e2ffe7b826dbfe0697fdc4ec23decb87128f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_10-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e523c35d10db78f7d76e72aa58ce0de4772df2b8fd5d0063b6a57e14aa09943 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_11-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e073fa36b4144f652759b8f861c66e1473e77e04 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:924918c818d6d4c3b5428f8da83054e157dc1a3459392568fd87e789e7aae78c +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_11-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_11-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..caf0d0bdfdda19681180c93d6cad21387efe4213 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_11-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3aed99f3c7ead0fc90dda190735bacff0e273cd1ebd77e743d1cf3579c3aa97 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_12-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9ea8058e277611ac36604d549e822b543ad9529b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d764ec91a07e42c6e3449f895bf12f7dc3762ccb1a1f436d780ce4a6eaf5722 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_12-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_12-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..937d82f10dba04359ed1973d1b2841bd06a499df --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_12-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a038433bf237113fca2e08f80daac06075882ac2ff49dc5b6d1ab5a427c3b58 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_13-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa8fc6af56224c175f7fea4988e291b495060b1b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f84913793260c72fda6b2bc8ff7d49d16b7dc0e66518a74cc378cbccfb373d1a +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_13-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_13-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b27f87b9e68c15de31045b690a5b0ec2dd47b299 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_13-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:246dbcb8c77c8ea7422f034af57f10b29bcefe37a9fb90d5ce7261c6b4af395d +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_14-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6469e20dc2c2ae96152c56b5eb8a5a9ea041d8a7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa18260805bbfb6ce2bf1c3c5859c293a10a296ed161f8e68bd0a5c9213bbaf2 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_14-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_14-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3144e0c87277c8393a3b199b1421a48a8538d9d9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_14-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d05b455119e5a1b20725e4937eb8a9882724c5d5b3cfc3b65f5c684c8485c32 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_15-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..590863a4cb61a1264ff6f8a38d5ebbf31526233a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ded26a8a931eab787b712f33f29d20e35defa5f95ab8b35e0cfbe15e33c0394 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_15-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_15-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d69f0ce05695fa917b65230b1f956965fa9fc247 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_15-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e167e312729e6efdc4053114107bdf3e7e403332e148d1a95a2190274b0ad9e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_16-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dbfcc2d365a26a924255e8d3acaa0e19bd924a84 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c92bf7babafba25d26763ab1270f0100f3dd942dd38f37143c895b4152b2a52f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_16-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_16-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..777a30933963b773f05975d2e5b6d2b962452153 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_16-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae6c0d38ebdb3151f00b67d35f836459a0c565a87ce67066ff7098e4ea7a30db +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_17-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3e09946d2856c4f8e915befd19a738307dc0195 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22a45b386df91b51a84004bcb9d6ffc263aab7d335c9a4a261a3af80aad0fd95 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_17-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_17-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f89b26353d3ebbe0ae95f36209c4e061d43fdd4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_17-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bd42ac78d8bcda2fc95de86729825427d4a9e0915042fa7688ade8117bdad1c +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_18-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..01fcfaa5cbd2403b9853add452885fde6fa5ee3a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f22ec207145f8ac0bd773f35b52f1140dffc1ff51ce9dae70bc9e1af31160b4 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_18-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_18-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3cce5145d79349aaef7a51c3d37f25303578d2ba --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_18-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28492b53d23aef4724f7ba7c46bae0e330e19a7ed22f60ee1ba2f4a00bab200e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_19-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8bf7477a8b25ebef8248d1cc4df70368debe66e4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b9ee84fdd76b253ff71e286cbaffdf01756673e64d6e5218af63de03a1c7311 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_19-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_19-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8331bff8b7399f6da31531eb7387ad0666b9eec4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_19-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80b450a90463de028d589fd057a11d007fe75b592f8f4942c8d710e716d85c73 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_20-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..95a3fad6353263b1945e3e45e94990b7284562b7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e22fff97fed398f1372945483a1d7ff8d7ade6c3cbb5ce4671d42bed27380222 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_20-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_20-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0e2c6e329435075d547225b67c473113baa03f36 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_20-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:febf853590f532b6802cf4368f487e30fbeb53645fd234268408000744efa2be +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_21-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..be04a0af3c5f287eb043881c24dccecfdff08d05 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bc7cce72006e965f57d4b92ba74baa65a87fe7684aa865130e5f622db8396e7 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_21-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_21-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..40e5c71b11421181ba3bc9b9ed0f43b2d787e22e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_21-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:247342ac49460bb100f1fe1fd9f836334fc7ffc759491026c0e1f179d7fd7b3b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_22-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_22-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..47d8b0e900b71c8e829120eb68debc57f2be856e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_22-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d2719637d3e8aaae3d82f77786b9e285fd2a76c7fbe65b522cd8009c82f09c3 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_22-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_22-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b705ec8f77135f3c2ce43aef1b18e0b6dd7315a2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_22-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7b9a390bf51496e5acc614c698f8790a94d774ab405289410562c68ef303b91 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_23-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d69a770eef59e3fdf01b570921378bf279891083 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59d1bc49980cd1bf4eb108f19acb4c601aab5c45b498080003ee08eb948dbb3c +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_23-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_23-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a074fce9ddab104c8714a9b02f17a6fc037dc6c3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_23-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9c7a170fc9b365578169b470c6093de482c95240bde2e00ae646757e8ed1d5a +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_24-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_24-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d4419e2df4abfb545f3368106017c592aeb8154 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_24-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a18a79a0d7bdc82915e5c29db9547fe0dd3df6907ea0055c81c105251d3dfee +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_24-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_24-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..637da7e6cfcc71c2c9482054035d2aa6d64c3b65 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_24-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c861e8c93e71f1df4d12b232c8db2cb3408969fa0ba15a66cf4e00cd200f6a77 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_25-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_25-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9ca137e95250344a47a5a6992b4640f8f8faa67e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_25-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3f3ff7575eb91c5d315d6393b4856844f60db32fc5f2d5d356c492546a823cb +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_25-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_25-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d330c72f27b84114143f699ffb339cea6d314117 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_25-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:554a8d4150050db50b9b8bf45d6de3b8a2b02e9692a02e0672a27415486e0ab5 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_26-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_26-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a606896b12376456ad25fe4de290f7e11da72a03 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_26-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f67967067b0ad6bb884205ed13820f35a295b9c0e3696cfcbfc5997380e3867 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_26-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_26-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1cb7d2ed869f92dc10b5c8a773043696136188c6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_26-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a78808a42162d23640ddc6b2d0484ab76ff44c76e74bb302c48e4ce0322bbddd +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_27-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_27-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3d2b2ad15634f0027962348fa9d24b0832df86e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_27-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b1e38f778e81e50578f88697684622e7591739665ea63503ffbedab44adbd70 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_27-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_27-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..127d79b3c8f4e93e7afda2fe797e8ef8a9816c8c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_27-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cfdd54f21b016b0c0435cc907ef8af25172e405a684b774b7068f84d0752c7e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_28-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_28-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ccc9b747d6e0758998dcd39d1130beaff63f508c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_28-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:967d3355aa3f34795d2682c74b402859e259d21ba9c3db5abe6ffcbcc35fb20e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_28-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_28-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..06c9787525922bce1bf3c8831929a9036c22eede --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_28-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e80ef215f869b79cebe4392e4df6247450e3d469aae02187cf6a5998ddcca891 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_29-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_29-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8040e76992dd987334d657986795c097e30da763 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_29-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2faccffce974c37e67ac000b4017c61620a718d6aade345892db3851b4adc95e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_29-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_29-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..59b3b783392265e10a74f21f70f81ca97be29cab --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_29-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0509dc792a69a80e7a15e8f5d3f654fefc2b4e00fa9cb921263afd576df922b1 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_30-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_30-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d3645a824b3d69134a56175bd65a916dfd016cd9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_30-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0164976d8947cc5e49e43a3e4753e315a1c5ffdaf784e771853aac48145566df +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_30-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_30-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4badc4d6466872205e584b4d624a9d0b3622e221 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_30-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f7f5ea5170037745945d47254a6a0011e285e4276ad23a1838adbe464affec8 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_31-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_31-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3440352b04eb80561f673eebb94f9904716a32d3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_31-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acc426151400d757dd609f0a1104a7ecc0044a9905edb4573d0d54b64acba5ec +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_31-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_31-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bddea889aae594d54bf6d85a50cf726d427aac7f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_31-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b0c6103d2323f7a710d44245a4600cd1591bcc53c0c32c04d9ffcbd2f5c9d25 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_32-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_32-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..99a57c7f9a7d1053f67da40db28d23aa37480d05 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_32-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0797b649ca91dd2806972bf157dd4771c3622b96918a31f65c3ade31ac41d64a +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_32-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_32-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf6627644aa1968d5f22c314161024c5e1b12fce --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_32-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:965b3b5efb9e89614a097a71ef06bb748c5e7eb1bb119d574f221dbc23839a67 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_33-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_33-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..40805efcd8d8062493b511f76424d7ad8863e432 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_33-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3737d31af99e979850ca4fbc233e13aecf26aa743ebdcb494de8bdf46223dcc +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_33-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_33-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0a1b3f250f3dc12f712173b85e296a4d3d56ce82 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_33-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bb7a893a3dca932f4170831d49b3c5820d603be5656ff59872930b7389faf4d +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_34-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_34-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4356e867be65b3c7a584fbeea1121e1a6188ff3c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_34-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ac436d860de94870819cfc8e2551c32f91312b7be549cd8ef1ad056233e035f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_34-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_34-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d3d6f7da4f0d4de95dd017d8bd2afd6140eb17d1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_34-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7ae43c855d8b0b65a0b7fdcf60b5fc9067249dba3f915c7a00600029b361a72 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_35-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_35-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c1e1cf32098a7cc3dcda1f30eecaf4160ff00897 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_35-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f6045142fc0c830b7e9bf58e783238f615920031aa620d8ff33e0f25899df3e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_35-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_35-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b99b3e4ca6157f44d38cb4c3bc641ebd0597a54 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_35-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ae809832902b08e240fb8ba07470e56eb707baf9c272c65f885c6d310bd33be +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_36-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_36-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4149f8e1d3aa5382c68a19a5daa8948abcbaaa83 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_36-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc4928a35860af89d889aae89d7dd77ebc8d032ae1622c0f2dc5f9bfb43075dd +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_36-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_36-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..363d89d610529d5460f66ed9f29e27ace5324b91 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_36-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9cbc3d39f93e1d32a90921fd2f37038aceec2d18553bd3adbef10fa74d612d5 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_37-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_37-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..206daa8a35596484db146ecd5e58d2d23aab585e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_37-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92c9a30a2924b3ba425b07b8dd312df5b8035064e25ab47af7fa145ef676cf51 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_37-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_37-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c00a79afab0b2b7588d0ba044ff3cb9c9260911 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_37-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c7efed324b9741c5076b10b49a12b6c1dd93599006e3644a96aaef845cfd6d2 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_38-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_38-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6c2629682e1e0e0fa5f972c72adab53d5f47f6a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_38-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:416b4a47ca9498e448b8eb56ebc76f4977b929e64ee27f7582027f1b078bf2e4 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_38-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_38-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7390415af833e22d5c7364784929c6a49c744806 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_38-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2021face5dc81b9382985d67a6b950ecc865c36485dc505aa8355a35dc23994a +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_40-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_40-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..367ba2cf6260458c526e66466268a73130fea280 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_40-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:556c66734b0919fd0c80d26fce09fe1ae5247c640207477ff4972f8f1b876be1 +size 12483 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_40-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_40-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b99cbcea489612a2c950900783807822c24251b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/layer_40-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57905266f515682312956c2466ef08a503d503b1c05be10344a1d434d920a4f6 +size 12483 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/mp_rank_00_model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..515dcceecc3f4524bc5e881eb1691dd003842814 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c2af9b65c53df7d6a6d5e3522ad79fe96aab995574b2dfd2eb1891eed252dab +size 51507 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/mp_rank_01_model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/mp_rank_01_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2845eb3c587288a9aedc1b5786991c4820d1c9c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step70000/mp_rank_01_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:165561c9e2c7b337fffc6c61048defa4fec1d4c6fd9a2b8bd2ca92c6ba1e83fd +size 51507 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..528aff3777a6e3ce045778bfed95198ff9b4ed60 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23e7f799ec0b3a08fd2c8532ae1a6276731a3871d5a653ac319cb662316e41cd +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d89d6f257113d6810cdfddd55d45c6998f7245b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4771c9db398906acf2210e34cf3eec407c3eec0194abfded9bc83f103567bdb +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..af6afd1272b598311931ad965934c2014ab2cc35 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dd3da7bd168f096a9a6f1739bf2a527838e08f737288c8d199817850e52556b +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..94f67926664a5450a01e22e6fb6fbe98c1715f5e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e335155165c108ebe0ee3ac98db3bee78e135e6168a640d9eb39fbecf587db27 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..962a0edb085ef657d719ff54e87c764e2a22534a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da142b1fd7d2039bd39b8347f4937205c89921eed27d366d45cb61e79e6a8e54 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8fde1cf07368594827860f2a883c7c75be892e1a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ddb5bb249eb575722d156da964cefa3d72b2df24d5ae3d79cc6c896e224754e +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6d11b5b06e6e39e2a31af3f28251e148cda683de --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3914f3a6aa4370aaa56a86ff4f4479e724e1bbd4d2cd65ed31e244e3dd1beb11 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf8a240f6ffbf62be0858ce3f6ea25f037acb819 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6a11206ac6bb41dc60316cb422e0f61a6f13496ad314c99ab4d4671a7bac344 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bed6867acb01a4bc7bfeea9403b0e91d57cd94b2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a88423d88ea769ebc1a6ca1bac73ad79508e45a2f3d51fce30dc12d1cfd104a +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..faaa18c5ed9932c71e278e157a2c3180a73c29db --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ff98bca9ba82e6ca12bbc6dbd399e64b2eaf5c464e539a062339a1386da4811 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a0ada330e2ecfd2779b3d516baf5904796a915ce --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2d38c58ccf884440d085faf25d79693ce0ccf2fb0d70a31e88688985c3889d4 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..77752fa5a1dfbecb970392facb8d9fdab2720243 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c9a8624999d55ac9c045759eac6a85e8b573ed9fade656ac06993b7a314e7f3 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e98689b6b8ae57d2ea562249c5ed6e3c4a75046 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4598a9edf20aced193b42174bc00f9db574df15b1fc0a24de156390f7a10cb0 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cff5f4f4d484fdf044c6cc1c356e27233b49b33e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97aa1c8da66f57bfc4ea95d8a1f9140766c6ee09b53ee8db1ae5ca583449f990 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..70584decaa4e3d76bcc7fe301890801382cc8516 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ceabfc3b378c2cb2b45e270568617d6b525e10900d5024cec8d7d3d63d7bda68 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bb23b75e7c04e2461d326ef970fddd132712ac4b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a703bf418fd18183e52374c658135d440a011dba545956a4ecfee4ca1a4984d +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0776e8fbf309b97a409edfaa3f50c3f7f8c27917 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb91971d825632c21abbb594d1b9416b50963cbe3f5b713645f24ac3011fac61 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5641b20045c8f842334178c063e9ab489fe95902 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e69d911ab8fb030eca46a815f5cfd1ff811f7b35a71951a3494f2f3d14261a34 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2eca0f50a9e2dddb3c9b188d4e20c11a3d529ec0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d695dff12a6926d867052a6c0f1281e657fb47bc6d3b0f4beb3f23e658e6cc7e +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aee337841c5d7011b8082675fe9394611fdbc67d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a366c2fc7d42f88014c547855f7beed7085c4dd02f92434d34e4424615e45f71 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1956568f61d3534ba71bfa7def7fe198c9cfb1bd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16fc69cb653fe55fe63d58f6dbcb30c4c496dbe70762b781ecca34472fe096c6 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..edb2ac7c0dc0b6bfd94008fd0b079dd8f7c58b7f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbfbe4ccb469303d4538fe4a0c3c52aeccc6ab55050cfa3f8fbb695dcb1cf0a0 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..814f6cdaff5829ee96d670c34c401408fc1fa4ee --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:972ec970870d17e0d1ce25e77e1039bc4b16d91175aa1a36f374e16d01de9bfc +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3a2d6e9312756ba5f181d3c92acb3a5dde49dd1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22c561414bf565f14aa21e998c566723d10c8544cf1fc6636f234be9d76de0f7 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4fd7148db8d6d35029cc48e13256ffd62a173817 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29f6aaf8975a338652ea95815432ee46c99527ef156e7f3646b7898902994e40 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..50c9e607d47f562f13f85d96bbfc64fb47e8da4a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d75bb6a2b6766e77b1a91ffd26bde17e9cedabd4326580d2207bd6c14a397444 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5e3b4ff1fa346519dc74842fb590b258b1f5194e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9c2b234d85aaae7d8d6183ce006796b6c731d14b69365f07329a10e411aa7d3 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3180cb79e8c04bb2858c8154da78eb1094b8efc0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff7ed920395711c33d8fdf9f2d216c0358b37f44f29a85752e468e420ac4cb5b +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6d9c521578cfdc075b2c694d7ca446f826dac6d2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9854a07f6f440ec2e8d042d07722f2fd7868ff2bfe5ae6f1e0146fd910683ff +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..42dbcfa3d2ef85a249f6419034b0ae0f1bcf6be8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4fa1250402e3c27849c1f94c9e68ed22785b6a4e96ad542c4aa13aca761b99e +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2b2c88867066296fbefd7fb68a64032ee2e03eb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a60545286c561fbc8843f891de2e173be81ebad1d2209eae4bcda0047ab4091 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..58674ac1947baebc5e997c5e1a21c2f963245de7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3ff5698b0e6999d83d7b1db7506c33b512f710401532a93ed53f4f71f74d041 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..11b373b3604cbad6da22e3925608fd640aa27d6f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9870627ba8945c7c9fe8cad36d3e6a5044bc648e631c064b8a3c1007a9827741 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..63656148ae6603887759a40117d08d4ad0ca7c58 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d89c9c48c24d811018429c0d1643891cf6df74dfed8600b3df9eea6b98e2a13 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..16aefb3cebbeb9357c6bb183b979fad98d5ee80b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:495ac05dac38f64d0eaf894454121fb2a83028ed9e5bf5374a5d8e6e8df0e539 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2fb1d5adc0efb44dfe2219064ab5d3c79481b1ac --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f46afb8357809f582ce98d0c70d87198c0ec413984cd9048ea047e66bb917da +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d53b6a9c7f19468b35c99bb2213825b4a7490e0b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ab9d331f5a0f4783684011829a4df044052bc7e6a7fa8713847e797df60aa81 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2b0abbc817bbaf83250b9b4bbc03635ad0cab38a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8dd10f86e96427b4c7e9390cb8a7f6dc0279067305468555bcef295bebd52fbf +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1e7118e05ed5fc11e60f999b7bc22b155d42072b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01a1d37a85711d96a43da1646e00373f10ea55193c8a212a749dcf67a21d4530 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3d06986778d18eaa01ba5c7502f420df07865856 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2288ea4bf63eb168fc817199f8c2f71107b71c5ec7d59625f226adb85f74cc9 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aaa34c7c41c5777cdc195acf4f15a54467119bab --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b74c50d123fbf7faaa9fe95a10722edbcf839f93db36996f7f3a91f7b601500 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..08938421d74d3a83f6944c1962205e5ee7b99f52 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15af2d35be451ef030c821b278f3fc999907744b5bec9de8a94f17015c3a8915 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff1fbbf20b715e9ce008905df208cc5f04489851 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05adfb2d182cebc15fef05bef849c283c9dd5e7874e8a2314ed3675dbbc24b90 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..88c8e3d4c334bf8ab6d1d34c8819d07f1948d208 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44447b60ccad205de0612363b05c77be2909d4c8c4685e9d9d383c6f348ea3b4 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ef494492565a99ad9e0eab87ff8e149ed7e858b0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37ceaf4f947d0362f8350c08c614df72c4fcbe5885b3c6276ad4b3be632ec82e +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f3b9282e1adda7500b2ca299b633cf15f4fa8554 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:882bd0f5a8736c976d1f253b3ffe1dcb7470ef68a8a1ceeb7f18380cbc868a2e +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bcf8e461cf89bf0acd46585ef96f0839d75f0e83 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21af35324ba16c12ec2f334d69d3b205022824aa5d8e34d1ea96ea7715a791dd +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..26af6399a251415d18a5a86d14303f10c76ddec5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4091444b66df9e51e7cad7c77d3fb71bd5e6efde4ad71896bc16056f29f7dfdb +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7b686225d9f9305c5f8442ee455ed8509a511e72 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:615efa8ec625d2e062083eac7670452005804f0a6471fc63345333dbd066f3fe +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e0c4255272f1705ae5032e5eb96a6706d83f8668 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:459f3cb0b2de27a925d05259c4d64ef3824bb2c693c79843e3fb9172fd063893 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..85502b813f4eda15c684eea37df0a6ee81ba3d1e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccc19ab3b4234379de43bf860c1de83894797796f1bb89a5d56160e2b3324015 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..42a21e46cd93dac3e0cdc2551fc8136c3c8e3199 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:210164496fe4d35e4d17224829b0833e8d963a7171ff2b68ff6fb7695fe97a0d +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1869d8f581bbf185d01eaa7b8f5ae41602f625e1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22762fb55111a9d61b2a5974990936b9be2a2ea770c0e212a8c43d2554af3d83 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1b9e937eb5c1da42bbe70a63f903e2a922f2ca35 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c28127afe3f8f6b3a25229b10528ca67b45919a31f8dddb0ee938e0c4ca54b23 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..84d0954b4a442038cb7f220b45694504e5319108 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf73b304fcb0f917b902ba18b6d8c673487384c9d9b18d6c1874c73f6f7cb616 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c7836f369abfe9bbfc62f0ca60daa2f97a61b085 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e8705ace784f43d0fc7394d9d5bf1534af7ee1e864cb2de2ab9d6d55cc1f57e +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..af701739082c80aa26e5881f86c47050b3ae4916 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cc50e125982e2634b7773d800190cf51b0e24241ea5f42871527941d97dce1a +size 167872941 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..37831200f99b833245ffaeecd05c7e73ad9906c3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6765435965fe6b61325548c9b1bc9895d48ae5e6b8130b67282c4d8afdac8c41 +size 167872941 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6ae50f418bfdd246f39cb00bd56a437ec540e997 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c0cc1926ac042e30c4cd948fcbfbce8b35784efa7609a294c5c9eae5f76ee37 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec2b1802e4f112bd3db07fdaacc45d4f4cad5617 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:776e3908c83fc1d7bbbc33e1717845d576a7a1416d3623af954729397f0f5e2d +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cb03b976762cbaa206c1658d00d654e16392950b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfb8fb75d31793d9c5f92cff5187b1f15c5a5596ec544827fa76512348e2c40c +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f581585009cd63b626526a6e386bd72d4ec40853 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87d7d4ebce49ab0f2d5de2ed5f60fc9cd2461a95bab48ca332bc0c0205adf77b +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0b44cb184e6538bd145befdc90396efd342f772f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d82e1d411b25001ee06bfc148af871140d613fa23d8d3acbf30c85f7ad13fd7f +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..361760e47d1ad60ea30121b27181033b78fdee0f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58512657d745b92499660fc954b9eaffd44b1d8a3c2b08b74deb373159d04887 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fabe64ba58e58bbe2cde41a451f0576bed1574a5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4314f7acb045b6fe4a2c204c050e0d9abb0d4e727d611958443d565ac8216b0e +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e81a3ccb51d24c53cedd4ee9d8103abcd1f6a314 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9b61dca7c44a393aea49ae2be776cfa1f4a9108f35fb690312fb1fdb42d0160 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d233b0f9ffa623c9d1bfb42fc2bbe866011bc92 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b072f451faacdb378835b307815efe72f42d7f4cfac65b5b339d9d4ac96ccf3 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b84b6c425180117abbdbf5c49ddac2b99ce6976c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:765ba2047f766ace26a4da1802627e5e1a6c64be0b475a99c446f875813943ea +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e60278cfd6f22e300f19b16a10ac78c4dfeeb396 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2b20746aed68896e7f4115b3be9f059e47753c638365fb0af40dab682e378f3 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1e09f6f7777c9b23f1dd57940911d1cbd88698a8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a2411c63d79c01c8fe8e133b479ad7df41ea164e72f428cce06fc5a9efbf9ea +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f5d1d630940b2d2945a62f2c4368e322c8d108e4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a3f106f4607071cec8f43d2940b5298a517ad92bd39d96af3e55874335ea2ec +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0f0e486057cf37ae2fcf3315f101b5471b8ec61c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5dc85927456e4c344da951e14cd450d7d8654847cc20694eddf848b81c224980 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..182a6921598261b32665e49ec29ba574fb594d93 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7df9a039a615693edb08f6026f05fe0aa46c02b4b69d35d7fcd1339f4900d19e +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cfafbc9f9df057a2130a6123821b09b8e4894b91 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8a0aa3a54478a348521e6fdb7d3cd81d4c6dff1a72b485cd62daff54215c960 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..231e8551ad7409920022787306e686f24846b86c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2ecea71dca43cca53b33836488b22202aa1c212486c2175195afc60669b19c7 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9559e2ddce8327677600cf3655a67990258f2f55 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db62911145728df4fcd9112baa72c911ad7aef516d6848cf328c7b17e05e17d8 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e845513056e5c4e7c64c69ddf31466ccc0460d0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1582b3729cfce6be4e84ffcdd6141d089f0a8be4baa5e6fc0d9cb9b432966819 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..357e37127a9c8690c5870e911c284963b9e081ad --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0a25f8459ef0c9a6b5ce7f45c60a5c5486c78c5ec0b94dc960a687649a70207 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f4d69b00b1b6c37357a7393749dc68122738185 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbc2cca546fb801451dee48ccf0437ad3d014e2d38a9ef86efad24b837384fe6 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2928f90caa94186f47623454d72112d2e4bfa6a0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f9b90df2cb2444c87ebd37f9b7701dd69f0a0abbdb625fcace702d943da34ce +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bffb230bf7809f6ef23f5bfb87ef59a0ca155bb7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6098622df26541036bb9327596028d8ad59eda2e7597f96640cbfb5333769a5a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c983d1a3885c726a9e991b1cd4860f15e8a359f0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fcb2db9acd410c7aceb018d46520d694a14875516ae3f6a503b871a31b9b79e +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e63672599fc9c90c61ecf4369f852a8c6d419c0f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53a90192b1cef445683a1a8a48e28610aa12186af0fdfb9bf340eacf87f3199a +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..89417696eb615f50179b89e6b81b9829fbf09734 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd42f446a1e339724ac52d24cbb231a95a56da9cf74b71f1f1cadb8df86824b9 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..65b5e477d33f7f93f1d17e79c39891b0a1c9d36e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6931916db975d984973b4eae2b575a4eaedf8957e4ffc97271d9f10e2b10fa36 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7678ff870453830ea1084dbff1729631a88df5f8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2be0275b105e5e982211b795a2b96e3825b879418ed4a4028e319feb676d0503 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b0a88e6e5a5131a13f4831e48883dd3025e72fbe --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99556d40386ee30f24bcb9d24d637d46c4aa6846e574f05f2aad4fbb58eccbe8 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..054ed983a867c5c722e47630b3624818ae02287b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e90d5f6940ac48a86a23a1de32fc3569695f5be346470585b9a638f04c131ba1 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..51e62e2a71d7a79485e08b4ca6fcee93dccba8c2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e339fd47e729f29d5c12e7a717b6abf41ab25dfdf1e2a7be98e6061856cd3ba7 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..56d5c4932c572114e5f888a42f2dbbdc973034c2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b021c4ab1ff255eab4273c5b1bbd72ceceb760b76f5d5cb6df8d9081ab20df06 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..57c906a633626dd387b28af4f24553b2d858363b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fdc380ee7dc45378c57fd14fd81d605b0b4d325081c87d7e0cc7ab698926dc41 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2965b922988b27d32e43dcf3feda50983d8987f5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81ff316d479dbaa6eb162212cd0e6f739db17fc01727fbf948c5633fc08fca80 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..29da218164a0566264e9cfd5ce64dbabc3d00df7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5faa3c941d3b8fff85719386e142830ff542b1cf7e94820484ffeb3f9ed8a755 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b592094b671b157e784fd0924ece69e9ded9091 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:349540cf7527a13e03ca746891d2a07ed2fab2859ec8163411f88c8758f4eb96 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf7679c4ad5dafd1c4819fe5de9bc38a2995180c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d3788da4c486aaa50526c716cb229cb337b47e821d20eae72b899af5c04af09 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f9559d2871c9896bce44c3d55ca1507dcb7ba66d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c92d9f2449eaa77b8a5f351b1a9d0d2f6ac9200e8b6167c708f55d9c4c2c69c2 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fbd725536c3ec3ecb37bd0c5d6024d2fb9cb5766 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b22215d10eda9cebf72742729e7f16556d0f5f3cdc6548b22886ada009c554e +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2ba59df3054c6c1538e5a0e4f7dbfcb487de753 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed81b525e0666621acdc08e69ba6a1af7c414d005c9c1c9b9368c4e2c4b19f4f +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1b47630227ad8a0ef74d0fd1ee2723c544060ec4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9eecabdd33230de182d3ea36577e2d2a33a42cd1c858e4c8d89ec79562577ee4 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e57ae5a58545939ca958ab9b4666c87510fa5c62 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88cfb8aeeaece06a7d7a6cd98baf054b90c95ddbf2634000d35ebf4d31cbb44b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2a99ba494c4a103cbbd8b470b8b06543626e6c90 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2e8e91a9c17a1c998f2180392ba3796de8f6d0b7617b38bf6fce872ef421540 +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..57c7b67eb5974c6685896a379f2ebf7c5c830df7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b18acdd95e4d9ddd19985c3db61e882c2e6df93ca694ba293f6bd25977e8284 +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b62d301c7af620ced36659a8545945637851ef75 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ed3f0ce8c28b1b1227ad8ee7d92bef508a4128a910b80a91b4807ee9a1a4a62 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..deaf0377019cbcb205cf15ca73518be6bf3763a5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5681ccc09b587389c880eed9f5f64dbf2b9fc88534adb6ef02e11028b622b55 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..86681376bd2ebd9de20da497919bba795884a4f1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd5c8b105b9321b697377486e334381f10ce5dc11d6283e57efd359e5ac45c2b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..286bab9733145edc22d5d02109bc4ca246dd6c95 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52944ed9bb63b29033eb0c1011ec43017cdcaf64353d079f9ad12fa06ac511b7 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..495aa2fde872797a697b2c11d2a891a909d64c4d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5a6d910bc064ecfc3cb7e9a6cde0ff01022d6bafc154a9920a790f9babd1dfa +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f07f7fdbe0e34a97f0249a63b1faa0b795993f09 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:483ba2960f3ea781150b691fadec80f56acd1ca5bdb721e2a432a3a91ffc7e20 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..08495d7c932d83201352af70f66513918920dc35 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4cf9e708cfd7aff2a8f55dc720f3688b483bcef3f49b757da6f8ea96d80924f +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c48e0b1f772596923e19de639b961bbd18fddd1b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5c17206a5715aaa2d7a89e0f9def09bc14c3a7744c5eb756180f44ebb5f7695 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1bd7bcddb68e4f4b1b7f76494fc135921f081247 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6869f97ec27c42c8187f220ef5c1f6e2173fc3d900e5c477f8fd829940c63c2 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2aa361879de9ddd42b598fa6e644efbabb2d1fa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6feb09049848bbd5ce823172a8110199843936b00c4ac9b8c5e8ea99846e6886 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6904bf8640fcee3621d88a0381bf0343a0e7f7cf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9337d953ac3946832d63f35552caf69365371ccd8c197d8cb4267a508e88f071 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3bb0de3bf3d4319769144407af14bb78f093e582 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1da98b5e19405e10b6d40ebb4a35643f39dcafff7db1d2f51c5d64fe578ea83a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fcbfd19d0a7ef6c2287cdd26a64c9f8b29ce2445 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2efbfec8ab730c3eb7349096d6c32b56a3e25a3e7f437497abaf6e9185297fa7 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bd851e5c4768ae3a16149af8618bda3373bbd129 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d21e88a11ccf7ab50c850adeec8c9ecf611eb7128e004af062f819fe248cd54f +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c148cd1ae7b3bcc3ff1f9245450ae10c1234e58 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69949c9a440aaf03031e0d7f145bf79d8a9c7cbdac56090fe266bf25ab01ccd6 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..092cba42e240b20c097f6dc35b8ed3773f41dfc0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d557fce097c77d7c73115f32f53c7139564bb6af9b91bb1d5952d1f55a68c257 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..55ea1af9ce4e7e6eb35dc966ecb5d2025dcaa79b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47d7d084c53534983ce0c793ec72ca543364900735efe6011958e3211f8e6df5 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9e54b9856c4684b6572adab11a255b6c3c01a081 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0afc0a490c4263b0682c426cda12a290a5c9c1d37ba47f9143ace209054ea67 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..10582e6c78562a18b032578f3f1ea29e4067a405 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b73199357e326d6184663e08b06cbf3598a3b473a676185c07e9cfcc2edc54ea +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..82ee54987263a4468c26e486e0d210b2aef5e7c8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:730422c7c66184d3bf4957cb1371c36b04e96e21b31641c2a7956560bff1b21f +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eec8fb6cca15415a88e3b651efcd8ed8e5595444 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3062d9f28477efc38b69d5aedd332537d12a90bd9676285dac34523fa946847 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..780f52af483578f2cc96a965e861d627a21335ca --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66c53a1484a70f98e92970a7b3170705c526cb54aeba2706246d25a77aac71e1 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3e630f7050a7551b3c1e83ffbb0017488f6b4935 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d3cf3ccf5a603a45dff4a559e83266093834f292511753fd8e1e26b210f2a31 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..46cfa5483ef3009b1b19439a2b09c67a2116dba5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b0ea06823bde649eb1336d83a7199fc853082e8b0d4874b24b1dd7687014218 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ed1564bea4295f39f831ab4e87fa9f5bf53b34b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0daecf2d5299d0b717fbcf13b24e659213addb1b36f45add973b57e9d6809826 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..45e5a3cbe5ef5531cc8e83b47891fac141172940 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61d0582a9539255c0e93eb45e85b0176a886a175489226ee8acf867808b20099 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e9998f969165675598e5db012950a70aaaf283f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:492be3fe58d5f013e5f93ecd1f0228a1b257daa2c83d270ed727fa6c2e05d942 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..be9ae4a8c8606ae7a19e12062ad3d9301fa706ec --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58aa1fd3698bbce5c8eb67d1d2611ac46eb88e8a9178fb3d674ff4d868f2f9dd +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..13ab9a0c0ba4327163cb81e65f4b58b3da0344b0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09d32f632b812e0b064cb0a7c8853b9f7c3b5fb50246a74c7c63fcc858b4ff19 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7bab5ae83c2cbfe83e871e35a74c96cb39e2f59a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b004fc97410578906c2fa29faa54e4d69327058a2cfd1952a0f504701b6940eb +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ce04054694a5ae788b4758e9e6970afaecc58f8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c1f99df0dceb827084f97fdff17a080dbcb0a3092d9e7b6d7c42ff545415e9e +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c88c9eb9b0f06c41c2544c8159abdbb96d0a77bf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05678ae0fc1d75d52ba65587c8eaabdfde7de7e980ca989178fdab27e3cfc24f +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..234883ef25ef37d23fadf2a4c706fc3be1e9d4db --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05249f58ed2d93a690de17cf8f4f96e33b5cd61f127b9a4989d6796b67e4ed5c +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..59ba271c31b1c3e80197269bae317f54b052763e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44cc075763f8259e9ccd6af3b20be936cdaf04bfda625543c5aee8a6eb139699 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0a182b91dedd21a42dab71ebfb3c8f248d7d9632 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fe7a39b0e11737866ffeb38bf257eeea69cc4ea07dada17f9872ceb429074dd +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d8baeab20f2dac16cb1ad5ac2766044a8d195ee3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25c8ff8db93d4b1bc18842b91199c6dcc7643e7ff175f5dccbaff52313fb6e92 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8318f1ac61f0b622ce07d6ca1d9156704571408d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bed601ff984b7ba724eab746d96ebcd829b24f8f2e7c7350b9b3ab03d3fa3313 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5578bd76120779a5a2df3a946a3c01ddac09a8d1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ceeee602c2543aa991f0e0dcbcedf447f7aa26899c2faa9cab028f81431efbce +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..537b42138d95f470d9482e79a6f3f56e318a4607 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93e2ca718c7bb89307a2b04685c0498cb15854ad8755cb5eba05ef4cd196b33d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..76962b5a71c092a2a7d61e24f6b0992d2bb938fb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffffae91141fe68e1f6ed9bef9040d6eade8b10dc6da818bbf1915aa0d27a8da +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9cd61f1ba78f14b9c3b842fca0f77c3fababbed4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c5a520c7f959ccaf8e6e7447b003a473b0926f41ef67e1c9a8bf8a03aabbc35 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..93127a8ca113a2579af38886a9950653ee7045b6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87af2fbf09f8cef8ca698ee4367345d4c5acdcd91f9e336ce4dbfe6acc7c672d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f668a0a4427edc2f46adc48788856bc58c4fd9e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73e977b5f6f5695b03eb1e502b154b9504359ec97755f82b3d0e7ed0731db1ac +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1ce12bbd7629cb589f540d4d98e1941c8e8dcca9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b8a09231a2607319e550afb0a941d2996a1ae9d17dee166f6e5630dfb4513c6 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7a4111c10bdbea5ac385b40a96c2cbc56af15b6b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b4f6c3f411e5505a9f09424141e72df57320a8893e8fbd96396f66bfc115f38 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3e544d1701c568ba4a4dec8e79586a0f7aa79169 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9667526cd25de0ac8b8bc5a053c23a44ebb208a70c14b6fb0bff1a7a7c0daa1a +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b18977c93aa95fe41558dc3bce47a1941e459ed --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0219720c28a981c7a76a31aca8d9b1640601de7b16c03f0158a7bdeb1d4b544d +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..068cf8676b5fd6643f82ae95cf41895a2204d106 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13add7152192b3d88fc8b7749d86df173e249f21a3d4727f7d3cb1660cca2cfb +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ffab225fbcbe87ff4d672b63a45c19453d3eb4c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e637176ad04c0e68399f68d9799eed8ca1e1fb4c30fa3208335729b230da4e03 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bae6bfa5f79cf8700ea2bac0a25cafd9faf6379d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc8ef5b1d44638c8939963bb9bf3c9ca38f2a7ea4f6c371f7dac46aca1d2b77b +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ceea7bef7f97918053a6bfce6c6fce6a9c340d5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fe59af4c3ba7ad999ef22dae9f91d4217972afac7752f07a1e5e9142d309bf6 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fede6dd014104d14679d244c8319619e38408ff3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce9343b80e1a560cd31447de8796020c433a1c1430a142e768db519bba6a3472 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b389217d6cd2429fafb677454903903a51cb893b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95905f94098e4139b19ab0ade1a836376dc1ef6da5ca20e3ae3c98b411ce1313 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..48bae4170313acae642869488193f4d2ddf66a61 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e503eb462e150b69927e7092c626ee6cdc130312dd740eaa05bb96c2b2227ff3 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..85d832e174da6503d68d56cded65adad5b04ce9b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dabd6455a59b45562c4e611d29ee2d02193e73dcee9e681140507e4d2ac9c265 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..af2fb3ee8f63827d69c33d451bf890a3dc34563b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48293e2237532a7e760ddd10e1b595723e45eeff7748e8eebcd7f4948237e971 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9744aefe928a4f08e7d803a4ab7de0814c49ceda --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f1da7ebf895a93be4202713173f6a9d511967c73b54e5c24232b10c2322a8a3 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0832df44e180fa996993092488233f7f230732be --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f9717967b25d46ed3c7ec60cb163761c0df0beea89f8918a79d6a454ab93e76 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..03ada87375334cf5950693c5f7178c98055e11f4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5008bd3be5cd8168b086b41344fe7fe82ff2fa8e17b558ac3b9eb3201739ca45 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d9f099df1bd07b43f27f0c720e3c01247ddd87cd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49cd3edd119dfe14c07f94ef986d63838c1ff50cad9a47d20d3d722697e5406a +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e61524bb77f95dc3d434efce226748ba5f0ca055 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2c639f1eea950abe5f0abef74feb88654c419fdcaf095e50d2b78431bfed4fb +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4cdc5434f57c6b09f6c203272f2fe9311b742fc8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df216f381269f88b2e7792215520f36390ca5194be2efc4edaa6560d1d9e49c7 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bfc84b4af89d884af7462bb156e6a44c479b5d24 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d0166b1f7b237356ad23f3abad1c8ccf8d76fa78bd3c7370dd072238cd67cb3 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2b7547e21e582322c214f1e9611475832f86f806 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:682d19163e6232fdf04978b768782a12d1f9f39468b57d9cd6378be346adf01d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..64daf83bc90b60a5bf6017b4690bacb76771c01c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1310613742321c3682cd278e2b83f9a100d0f5e52193be5bba031e5ef42d55ee +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1ae34065d76ca82f7e834e2cbe1efbaeae62b22b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7bc1d769c0ec8aa7adaeaedd2f24fb61c21c36e9b652f1a84af3f4a99fecf42 +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..381e918e2c80f44e8af055f4a091b2f6cd804e67 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0aadff01698cd170fa0003d0c9b273ccbcde2610cdebaf761efb93b6e067a59e +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc0be4f611f71ab035f827e7192be905967bff34 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d32c2bb616c8f8ff46c12d2105939125dce01fe0606941ba80d00593ba7c852 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..828093110f5b26f9f523e9162118ec7d2ed09834 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4d44cf86920201fbc0c913700d22bcf7bb03543b22a3d8e8d9adb500a310441 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ab9b69c30c538117eac173ccd33463ef302ddd8f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08ff93cbaa03a0ae2fded3ad33590e2ab866c312d56e9a8e2b4cfcf8b2d4adc3 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f65357d7c39ea4c398c99b62bad71dbe0cac38f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc342031bf001eaa5301a8de763ba8c3085d82741779cd4ef78c1d92f4ec84e1 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dbb90e5f6dbe0cd100e1ba3cbf5ea6f5179d267a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:483449dc31a4bf4fd2fb54bcc61abb69fbd8a1a76ad80699550a451897ca5444 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9af0c03d8055ff72a71c791a46eae863b5f0ac86 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91e26be18825d41194fcaff8f28203f22aba89ca3b3f34289ca0739717be6fd6 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f8269a7b724c7aa173beb2d4796ac878c2ec7c75 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f741722770eb766128d507b3344a064db2beeb67a6c4e12cefc0c17162d6f87 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..10d9473417fa9642e41140a8a5adff76a18cc348 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a158476d79e581720ef3da08b51f0824bec204d129c96b1db73981ccaed0808 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a386656b6d3b4244eb11cb8029a538ccb4499d3a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e7ec9f72e71bbfcb700fb3465ce1b78ee28f487d429e10f13d7c0206ef6eb65 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..993b226295f80f8b30dc9a392a720dba8b5a2898 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f835bd2ec832bdf93d2790583137df7964385f6ae3fd55733490ff18814312c +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..076b584a1e509a66eb12b909c53cf3255796865c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:142f72af6ce0163d222e539d396326894301f44e69552f6f17e618c37cd507ef +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d4c415a03970613693d6fdd9d843d8982a5793b5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f99dd4c1e76ad177218a81929b1077a849e1820eafea8d8c46d1b026fdef445 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f64a2a6dee6ec39185f0aa8641bb8926a659fab --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7deff0aed5d1f499f95994323d4a83d4ad9a24a67d2cb8d05c8dd357ab085cc7 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cddb173f289300af9b90800035712355fd7a14a3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64c1ac108eae8e9f0ddd3b9b67a7a41b638803f3ccf7bbfab83a14ad3de0a304 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b2a10d483a21dfb8a1e94b9a3a6adf97031712af --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97623fd07b9eda0ce7f751c4bdd9bf7cb859880c12ef70cb5d06e6d29bfccda4 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..45937fbef4e0fee6002132f4a4880acfe3023a48 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8516ed0d6ef83a948e2837244c984a3abdafcf741404bb533e33033604c7958 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0dae4b1e5e707377bbfb3deb880bc4dadd21f219 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3070851c7d9cddb51f9f8e91bede34948783bec59507195f47d866240ff67e32 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a3b0114eb72ed00a2652a1e95e20184b25eadd2d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ade482a9bab0aacfbd90981f928d31124e4e8071c7bf7b8889cdb466e8388656 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a612a6a9ae053ca8abc2846cfe00dab81d72261 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22f1c87b94c29d8f07ab59c149bbcd9e95368e4d24e52dbdf8f0f92146ab49f3 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2b1c41583c4c6ce5a8e9d7e22f6b72cd10f7602d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ac421f4ee92fa29adc1d6eef5d3a444b24f65ed7a00333878999f19a752868e +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6c82363d8999942f61cafb4717ac5ceee943adec --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c57bc7ad571f2043f692e3c50d3776bc26a3341b83603a691e026bb27182e5ac +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..079b27ef8200a9032f9b62b14f3e890bd189a94e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:701ddb98b47e9b3170c5b01b24e7d0bc1e72fadf5980ff590f0a84696edbbaa4 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d230bd34a6111e8f5c9e6569b7b97cf59778f64c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8bed8ac1b8f0164ad235192ba989657f781c940abd31239d99ebb193f9ee2fc +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a6f5d37b562eb9181e796fc83d6d3336186b12f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20bc0c389e0bec730e1f4eae59841d0604bfe5aa3000a5194c2dcb80db7ac672 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..76249ccc18991566ca11310c2c4b8fa6d080040b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97ead2823df4642361e538392e9176e22b9a2dfb46d2eeda4cc54536a914212d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..65a65615a857cbe6f18a367a41504038fef902a1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:436c8f119e9f9725f2552d584a4a424531910eef9c2225aaf3cd4d88d92cfdaa +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fd15e0eda96cfcb627218bb884bce53712cfce6b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d078a593dd509acf3ae0ee9a70b1d21a51db24946f954e9cddb89d6fa52e1f76 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..34811d81beac45fc56057a9f173371df47fa9c4b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0a827715aecabbfedb19d3b8ebde5d3f978bc1e383bab98e539a3b80f4986c8 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fdf95f455f7ab1bd499f54d29eabc1a2b3eb1ce9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b73f74289262ddaddfff3afe93da113a9ff810881594749f0d8cfa23d401a03 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1855ee6b944504d86bad13726893f87ef9f27cc3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d750215d93b5d3ebb14a9a7fea86a39dd2dadace57fe7774c2242467f81e5d6 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6bb96aeb48421f06ff06e3f86af954a4962540b3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b77e533757e1d8cb5713967f46178ee99a945724b4347672e17c38a6ad08fd1e +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fef1842542eda02123d9c5f6ae467e49ceee181a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbbc93f304cc3aabfeaf4aa5ab2b212188c01fb8825e1b53eeae985a00518657 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b57692174ec4d8d33df65980fc3017b0a731039 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af1c15c36d2ed82ded9877af9396c525e9cb27dcff68534fba51c90130043720 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..42b3671b79c94c4d8af3035b34bfb7a5690e96d7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e60c7f43e364c2cf0402dcf034d5660f451c65842d7db593c77fb09d861aff3 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f8365f6b7f9224c10b4f62d2a4d85149f568501 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c53b173bb28201091c1218891b46491f0fa6bd715fcd2062f0cdf70be0956ba +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b49e6f14b9127dbd7624c2a6ef44368cc7fb0e9b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97174672275d0216eebb5f28f5742d675d1d5e72d8563fadf96c83837a81ee50 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..106ddd4d2a9c7777eb96e14bc2616ab2adb900d0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b9534b04919a590ce423cd9d7f7bde1649c72d12a5abebdf7ce965d994909d7 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff561c69053a5a86b57051cb411ab3d1ac8f083d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4537f2c7ec03b11f52ac99011e8f8e78ea66b0fefc67d4280c1a22af8b537fb5 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..97339989038bd090c4c2fb35edfdcc9ca6d01bea --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e7365fc4db8697a5eb9840073d955642995bde9a9c84ac53aefe7ad2d9fbd74 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9fe7a7cdac1ea48b42f3696ca9e4856e6b3d1d5b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f0619e3d644db2b9597b067c748987a24597f68dee152c810952ebac226903f +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a9c7d41da42538158ebdbc8f9f004010cebfb9d5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58d51b26a0ece2192020f306edd47de64bc1add1d4c642e5c2fb03332c1fd643 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..776c78cffb1053975cd702a7f4f568c80e401e6b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee83d4da76fc8b6f352b816416163cd68734409fcc21858ade1489e3acd7ed64 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d1e1132b933c9aa04f4d74aa98abdcc86e0e315 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0695ff1d10d242fbff5f49bfceb651d387b5265d6e1d03c3365aa3dd13f47701 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bd306f11230c6a8bbd7433edd3784d73c8edc89a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b3611a4e265a42e12897b3cf7eb344bf5b592cb763d1259d96dc5536971734a +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..516a0c9d6f673f90164748ebb99e58cb2db88b82 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84e0de8a1414e355954cc427074c8a642b1adc7a0dfd9bda8d9723db88939295 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cfdfd2c13c7ca3252ab91783e641fd2fd2d343dd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07f4d2100a5d8b2a80e6270834b63d1daa9d3ddbfa8916aa0c80fbab80c9c828 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..821afb7fc32c82ec47672d2178f3f7ef2ecdb555 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ce01b44a965213bdd42e508e47dcf1d011e4d843e985c2ccbd6865fd0dfd3b0 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bb4a47974362bdcd8ebf007f5b9c2f9b7926aada --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9919ac2090353f55fedcf49b9d10b77826b18367fb5a46fae2ad3aef3220b644 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..57bdcd5cf5c62dd5e20aed0b8b6ef80a066ed239 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18ce61d55ec74b2c1b15027ee74ace938ce3efe0488520add0b3a1912b69e3bd +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d1802de2308a8ddd9e7eb288300b3256f03a6155 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71fb5c6e330a5c5f1ff0ea6a02e8e7aca89ea8de29997af1446e7376bc1e2771 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6201dbfe01efc3f55511376300bef528d820e42a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:145284969d76a585ca537672c4b2e416e4c1538fda19000c3754527a9de292a3 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b256ca767fc299f34cb7b82fb4ec23a41f01546e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9c381d908783880b5399a628ebd91e86a9dcd870165eaf976d3e3ca4f2e4a33 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f9df53a7337849ac9669724b1a8e1e339f67e0d2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80b54e04c33e05191e0686f4b450e8e733bbd679339ea3fca19c4272824b1b1d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..07f2c18931bbff207ed88f82ff08617b1bb68592 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1d4ea1d9095b087e267b6668e4a6d95c471f9b64f8e1f99e55e4cc956d60ab4 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6975da23f8a5306f8954c62e4cf1100d55e38835 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b1384d6742845110760072edb6bff62782bf3e3222ec444ec67f5497aaf84a2 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1de549310da0302be47e77cdac3f4ce0d9590995 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf0fc2e0a7ff2809a126a9c1cdeb43fcd9527a190c8993746ce3825c7537fe70 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d2069c7867553b7d9f4225af6c94ddec6f736cae --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26c6d1f76fd26b498618c21b89d9f51562116653841997ba3d2b128eaa6643ae +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ba0d9381c7a4a2334a2d7aa56c1f7432fa6b03f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:960cd1cec04fc04b5e9025a4b29e91f86b3dc684caf1b19896eb0f3d2603648a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..19e8aaad40a5c63ed96c20af69a89e848d4192ac --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aab0adf41da60796d050b5d6221c42c18db6fb6ed1340c8a9aa2a25131296b05 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2833b4ae470a5d2b8abc42e9bcca79a324507ab1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da29d37eaff5df7875a623c77cf532771d5a417fcfb8fc97bd6cd16fe1a440a9 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ba94b563805c3252e4ec5ca2d459a86a8bd7da6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c21c6288c19553df74c19cd2bb26eba7f72337bc2ffb96be488628fbefd9c92 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ca71a65282cc08b0385893c05148d85fab4baf48 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b5ccff92836d51c46caebbd851ffb1590b8ee86a20b7b7f6e5e397ad0a085d9 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..38ed5872a28095d7d447b181730bcd4180e01076 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b8b381125d8e27eaedc7c59299cb86677b7bdb0401f0fd8b6c842547283aec9 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b7a7b0a1aed070c7c470fbc578b6756f36f16f5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee9d6bc1b2bf4899b85b16838c30433d87f6aac57e952432ecd57f02d367665a +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..77e67c30dfbd807d9ee85780ba6d57810bbb2e00 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04646d621badde2111d1c8417b41606f372dfc76501d6bb49e1c24c4f9bf57c5 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..22855209e9c0af55fcb746b43a39f54e4d7257b3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5309c3fd64ea10f4bcf28f74d2419684ed8a607c4d51aa9b076d5863e12aafc4 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a4833463483610c9dd96bbc4ec8bd0635a16c5f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11086d712f3a97ee40c074ac41529ebb8647513a61fbf309b8635929f45f84c0 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2b425f2401ac5b7c92b54964d0682b867483587a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c709c10dd3e9f423d421aaac39d229b1ef1b0207859cd47eacdd43bf3d82c61 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f953e41882ac33cc4384c668df9610d0ffad832 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ea680d66beb1d563b70bf19db59557ce5d608be8eebedc6f55f577bee83f21f +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2aa785c089728c6866fb659f4271f002324e226a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1591bad50472d269b3512e0a8d4f2845b028c203430a97626079087271ddcbe9 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c405e39889369488c2aacd796fc4db646c112f3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11cf09b5f97f6bdae417f0b142870b248b57f30d09671f8ec75a80445dcce925 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..91abedc15759ef994a3a1864c28e182b68b26d80 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2c4ebb438c83fa929feadcb352b9f98b250dc1ab87f7078e9a948c8b077b824 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fde7bcc24ffc5e7638a8dbe022f7f5fa6918d012 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff758b346b6ce786a8924e30832a2a73a6dcbd107fd22d2aab4a7031d41571e6 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b919753206326949b0aa062ec0284a825bd7ff1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:765f6eeab1e5a939ded7670729d5f3371146b07f060106011ac299c518b09938 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..44c2f9f337d463280860cfa32d1fbf8a77f11b0b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97f42a5934650aaa2cc21e0ee6b8d3f77450a7819f17e8cc6bd0abe23e8af34f +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a663f3310c0b165fc5aa317872d9370ce1e21fcf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b3fdd390d2a748fa4fbb28c77a1c19df807520d6180ab1dd42906df8b9e09a4 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f08e3de2a6269600c66f4a211fefc6974e6ed32 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cb3b39a54bae1862edddaef732789c84530c07c28ff860907cfe79c841ac767 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fc3651b949f1d60ee0283b8d3a96bb7217c639c0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5142933f640f7cd66748898880bfa6ce6584389fa8a34778232df6409e4b5ea4 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b0175ad026f83ecdcb93e17324fb0ec4ee16be33 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02862f8a24c6ddb74df5cbc387bacf788efd904b56cc40f17641764eb53457af +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a2ae7ccbf63aa2944a0d40a3a39df02fe3b69cdd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a791c0bc66e608cab23dc083cff0829ef289ff74565f9b64c903a8e62b25ab8c +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b2d0615a54cb4d76623aa195bbf670757192f750 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:055f201a9ac5120023b2cd5ff0a99dcf8077971d1ce1f96bcfc74158a5fb904b +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dd66b7f04e671f2fb4594f734675fca0e883c20f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3dc1fe2bf95c3d342cb3b4b4be342e5b21dd326b4b347ba3a7d609ff3f59586c +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6bb9554ac6e4e6459aaa05bf4fefdb1faa702fd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae81bd6bf26e24a92b204da527ac76e294f3185196b6bbdf8e856e8e820b4cc2 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fceb5d0d14f4ee9be92e7a6b029777e4911fdf6a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7723b9b5900e64c8787d15eec148f51ebde8cd0ce0bf04f5bc498bfd017be3ee +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a35025c44846fe81b4aa2ead73c0cb2170e60bc3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56eb6b489b6880048d92841ebac044a3f499e319ab2bb1e507acc9e2f80a1b71 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b11b50b69847fe2ff3ed28c670b3465d716ea7cc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64683f7c43ba4a9892f347db60ef64d7a666534e1438189354d2ecfc8110ead6 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..033634f16b16ec7d4461dd44ffe04ddfde25be0f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e11b0e003d9735b8db9aae84c8daaf765c5fdb9c11b88d4383a5d0397698eda +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_01-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6c7a50390eeaa8a739523e4ff8258db44b57f80d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e97271818e704e7f754442ec5097215d7f1c57c9efafe7dd10702c965e15eba +size 153552131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_01-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_01-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..47d34bfdc63b2def2fb03eac8599e11b434a6c21 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_01-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a1ca1798a829fe081c2d64288e5c3cab661d07039a558504a2537da1afcea22 +size 153552131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_03-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..709c66fc08697e4e3dc62f9bed40d688e784fafe --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a064e00b2fc93eb9e08cdeb32878868487999c7fc77022568b1f4c99ea74137 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_03-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_03-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8d298455701e078c79fdf5a2b6634a8eded800aa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_03-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85a8e95246656aa3fa9474d6b24ecb2bf060490892aabe14c41c459e766f6b93 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_04-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c7698008729171f1862dcfb865895b731c8d877f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a9cd07e7ae1b8d8de1596e00e6564a46ad15eef8e63e8dd9d4e0eca28761f81 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_04-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_04-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..deb7c22101b44b88efb2b8d206379e7e652b4944 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_04-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36085c409c8c91377a0c234f82be8396f28e223d574d511e1e11172dbf8bb372 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_05-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ddccaaeb2c597a15ee327d1bebe4fc4e80e43db --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aff79fa5d8c4af7d957c688ded1753d079509f9738756e61eebec2044973c46b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_05-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_05-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ef7d5cabf7d51fd48e2f45a87789d3c974409357 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_05-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e34d6f0715db5a45ed27864181003a94f38190005db963ebdd22031f8b4dd3f8 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_06-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f367c108a144b97fbec36e17c9ad3e0677fe47d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd55eb3c7bd0e4a63e590c9efd89c2b21f935cf63bf61c7405ae521426f059c9 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_06-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_06-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d4abe0a3a5014e18ff7c1afad461be0c74e87970 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_06-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5dc8a29c9992f23f2cd799632cf228de2c34f8b27c7bc0e34db439899a901203 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_07-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f428d819d9d57a0d767ef157d72675c78d697e3d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ddc85a55b5798fc3ef21c22dc6c4ff0a87beafd639d1115b7e3aae6070de5d0 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_07-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_07-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..527d5815c252065eea367f0816ff6924004710c6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_07-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc8a9a718de59ffc5f4aa34257a717afebc567c7234f95da3cb0ee31d3604213 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_08-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..72fcd76406aebf418ed5efbde30469d2d9e82496 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:361f9f54e4c69ec39bd0de3361c0738c0bafaa6c133faf68af8ab634a59486bb +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_08-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_08-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..20b12e4a855b21a1b226fe354fc55239d1b33eaf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_08-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3aa6809105edc646e5c83e86dfaa0897b2d8728f259694cb3bd5f8ee7adc2ff2 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_09-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e5ca8dc84360ec6e98fd17175eab4d140568ae33 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cb1c5fa9580aab1713f821598ba3dbce22860872da5b250fdd27ccc4218aee4 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_09-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_09-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..936da5162dce54c3cc2fa4f930ef0a0b75f13bf7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_09-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17575717aaf62a1912e810637e3de11778af38f9b1e14daa80ac2bda4d526605 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_10-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a13645963b92cbddcc684a2e3da55b56cad0b00 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dff7c39a3cb4dec8b3c69739c6fa6511440e1459862466b1949f5d0d419ec04a +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_10-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_10-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f006d10e0e2cd89948bcd9619957a59c3293c06 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_10-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b436513af1ad2c36d2f75b0c462d7b5d0554cb62582449a57a96e6c3e4b0329 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_11-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..13b2b1e0d49c10d3d62717846396b124684ca4c9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b79d51b322e94d7cb03eea5ddf8dd2740b0f40028efbd7226bd12d647a8ad41 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_11-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_11-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..afa090fbd0bab12059aa70ccc0a60925c260997f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_11-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2347ab7e36000e3f47ca76184d181cf9c0474298d023d378442f2953cf7d944 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_12-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8274a5d84be17d382385105ca45fbd62fbbf3ee3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:333a6467824c5a1e76e57219abf0664567fee13f3b9632dcd4e5ffa05b5ba546 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_12-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_12-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dd20d80231da08cbf634e65c63b4ce1bff4d42ac --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_12-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed0220e5626cea22cd7df74912a5db1004bacaa8e9b01fb546e8dd8b1b2e0cdd +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_13-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a554ca1f354fae15aff06d00d35598755a57a31b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13e62e2a9dc5c0294764ae820007f9a369f2d12e122031dbebbb9debd9972764 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_13-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_13-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3238e329fcae7a1ece28a7ff90617aa81bc99c7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_13-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb1f6ca898875d686eb494a222c444cd30f23982396c1137fc22fd00307359e0 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_14-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9bf113d32b4f02275a95ce5430b72e83b6827f54 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a1b441cfcb167e37a8bcf43eab129b82b2bbea8d1d8a3013449c55beb157614 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_14-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_14-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e94047fc45ade575fb94631110ea6ddea77b572 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_14-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fdfb97bfe5f97b207d4f17962a38e65e19f6607c71836ffa9c637f36173141d +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_15-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a01d5b0e5dc40b651a0b7a7ee02e709316c391a1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae6e9fd0901f05bb5b3c605864dab46aff003a708fda6b3b4ccc20d67f8ea4e1 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_15-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_15-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..977e845891da91f4878cc11913c953a9833df9d9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_15-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:251a400cb1b8f5896ae3d43d05edf28782943da966155c5f2cf7fa14756fa65e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_16-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..39b8a96d21e96171aab9c59d37f9d95c8e39ce3b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c689dd5a6f54c7ee2e63a322c1b350f0bb0690926bd7963d4e0fc452d706d537 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_16-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_16-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..47a3f41b58cf39290d5789b6c6091265b5dbe3d7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_16-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2c449b8e658a0b6e539bc94318572f158708ad73f5877694afa3c2c30b4f574 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_17-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5a283b6f6a50528d64cbc41fb8c32778064a6b50 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:588975424eba4a4ffb73e5bc096922df32d2557e6db470b3b0edc38132802b31 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_17-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_17-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5409f08af03e18ff3bcfaa1e3f0e1d80dc5c8bf5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_17-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab5bb728e6938b2d1a9d74e7ff0660827beaf8dfe60f2e22b931cf3831e286b6 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_18-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..167e0e7d8f375822d740411f42ef83749d01c5f4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ca9ecd19d26554008710b54abfbe19c213bcdc3c5de7dc2af57e81f543c89d0 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_18-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_18-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9bf71cfcc83103329a8673021565b4df1822a033 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_18-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:926c776d893a15746dc0d564e52c8c71f1402e95da257e456e64722b1e2dabec +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_19-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..de2181b21d3f6f9f87efaa713376017826294398 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:587d62d0aeae9f2cebc443efaa8eaeda245e019463ef4eb307f1a215807b92e5 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_19-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_19-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0748580898e9c7970de7c98365797e0e760a27fb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_19-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22c0fc75c68012ea0930f6c265d3dc5a8d63bfcdccd78a4607d64e974f83311b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_20-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b41f77446fa11a6ebe8089b5d0c03fa724b9f59e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4c611b4a1b375b4719d6164712d90afb54e26a29435d166c9c2128f713af1dc +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_20-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_20-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8af415c6df416ca779cf014913b214fb3d3f426f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_20-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bd03320f9f38e6a2f1b8abbed9060a2b317bcd441b14b96aa34fb898b4c9f8d +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_21-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4c4626bb55d6c1ffc0e6ce0ef9e9a647204fdcc7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9eb6d4a0a0e54aba764be842b8aa06559e12eac88d1073a48f64b8d3f6359548 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_21-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_21-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..faee867d25d7c08de0f5aaca44a5c5220c89ffd5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_21-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d17ccd389dde2fbe9b856be099a058ea8b6ea483672666da517de1d9ee91d464 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_22-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_22-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5e1f7637f9c2964cdf99d4d5492cda7527ef9479 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_22-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9afefcb69e8cda5b7249076256eccaa3bf7a009fe0c2400bf06486264e591fed +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_22-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_22-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a55ae40cc8cb31ad16d5ed7cf9625676fa4ac919 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_22-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a18e75fe314058ae289f97e3ed3dde805852184bdbee25b2ed06f1c59b2c424 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_23-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fcbdee32f388621a3ed34c47bcaccd4e736c0cfa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46a17b799af692d26e08acf86f8ecc1243c896c44584523a4423293074ce21ca +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_23-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_23-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..422f3061b8ceb4f09325bd6ff3a679aefc09d5ae --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_23-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:131db214d9cc62dd5747b9b2be56e3cfc2906037a7adccad429081c8494486eb +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_24-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_24-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f56aa24a1b606768e825f12813c9e0779141d92 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_24-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:278ebc87157f71cc8347cf9c506da52d17ddcc184a586e0396752846a457567d +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_24-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_24-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3512e15be8b2c9e186b26311fcc369b2aa503791 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_24-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8df33e7897fa2b4e72d40cbe63c45b476c7171400a0b3d3a94ecd6a09c134975 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_25-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_25-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9aa53242ccc4fed8e0ecdc9240dc85d5670cd46f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_25-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9d5a0987be7ef5388de227268c1111611243a8f12318a57ed7bc8bcfd17cbc1 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_25-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_25-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6beee52501f5660e0c4f449ff901c8be81cd6b7a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_25-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1060169f553ff1bd1b4341d1bbe72a2b1aaf768f470c40b3f0481e868d445ad +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_26-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_26-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1aaa5ee20ac068ced3aacbdfce8a72ec1b68faf4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_26-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c9589c3e4c941cd743ea0f4be69c0703141b3bd85a6530939392661dcd07b6b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_26-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_26-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..265e51f2b32ebd6563687e86753a4316278adfa8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_26-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:253ef6a7a51c1b2eb24ac5f63d5d5b21834a0ce9118532abc83f5f5381d92ff2 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_27-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_27-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6ae330d799b02d47f95ba2c0a5b2f48cf2c2cfd9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_27-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56e784d3ee7909083ee52bf86e2252d71dd9a405aa7ac9725f8ae9ffc25a019f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_27-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_27-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..434b7566cd9cca9162cdce0da7db34508b6c9853 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_27-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c03d3f05a7c83dda9fd8df7bd48a5704f99a8428cbd1c7087c089d8cf55aef7 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_28-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_28-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6c0a6a9aa26d2109ab6bf6860a6bfe08bb4cc4d3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_28-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10aed5928c17a524ad57aa8ff8383e744bce28ae356cbb4872ab9fb436306ee5 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_28-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_28-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b8738da6c041e9fd8478ea890aea88af8109be31 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_28-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51412218d5bd7eab1ccbb8a1ef7fbefa2bdc86cb844abf6ee7ddf28d275c0c5f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_29-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_29-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0df307d08e6146cdd9e5bc4208e30f1588a075d7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_29-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4f06d83508e76f6b4c489e49be9cafb4783e3827731586540f281d15abd7dad +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_29-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_29-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..93a5453c7d3be5b4ac45df97e9b3bbb22eb26eeb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_29-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed0734b23b23349b62cc98931338201883f8e647746a3999bebd3ef6f1e42ffe +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_30-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_30-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4302fdc32df184286cb3977d3dc13f46bc7e4da5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_30-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c73c11864d70a0019ad78191e2bffe283eccc21e7df22d18a8c4f888388ba613 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_30-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_30-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e5f22e3f6d1e3cab9c0b6520350bc903a42a5f3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_30-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:922b4d494ead16a0a7db6af8457d6f5aa90cea8349f851d17a82a58323064464 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_31-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_31-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a46da0b047a01f633b77c181d7c3c1353a20b136 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_31-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3a8fa9e96c1487e9ea68570b088f77f6189fa61af3401798367b5072a4cc8bb +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_31-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_31-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a16dfe69420d2afd48a12f2ddcb5ca4ea964625e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_31-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc87d4f0156a25e84fb1174173ebd147a7ec27ec00d996be9194d7a9773eecd4 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_32-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_32-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b0d966defb5b9674d520137b9c264521ff5c119 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_32-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf7b972fa91af55774648f8fb92d05b30d065275885bf8bc014fbd30fdebceb6 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_32-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_32-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d01006b9281d5b17ffee3e76dd06dbd000b8562 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_32-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ab423e177803b43f064f81673b551f8625652635e8f06ccd66de95017325b20 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_33-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_33-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ebbed8b023afe1248c5f57e2ab38abd2e47351e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_33-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df3d2e6578331fe1ec07fce024af776d0f645917b08041db8ac8cc8c03a7afc3 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_33-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_33-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f1144650ef57ee35d49e4c44aacc3150304bd5e6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_33-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e8c02215f07097c4f6c890d1e8224599dd34f10431f8c4d6bb1b14b2cee6faf +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_34-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_34-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fafa3865e871ae292d07e3f20cf2c85997bb4abe --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_34-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3638d4124b8f1358e7c06651afc5e54f016a3ad391a2a5fc10a2ba701c7a95ea +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_34-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_34-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0831c52df915ef5beb29fc99f838357c7ddbf03f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_34-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7521c2d9cb632e90158b232850de33617a9ad26690898856eb7c1968fb1e16a +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_35-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_35-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a2d6ec18d11ebddd5d8221dd5de6a68ad6e511cf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_35-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e5c112cddd32c7c94b6c43d32f0cbfe3f18381d5915ac3e7f7a023a9c6836e2 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_35-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_35-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b2da42c3c7dd5a1692b50deb1583bac3370c239 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_35-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7914f6497ca18fe733da9d834e0498f954155dfe452bf485761d27b33eb10475 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_36-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_36-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b1129bace74646ffa524a12dc163de999845bb66 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_36-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75c34ec0db2f699d11ce3bb8976ed7133304df986a8e6719fb6b09703f19728a +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_36-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_36-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e5a5b4eb7711ed216f875fb531a8e9f44ca7506 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_36-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:441413f270f7ebcc0ca2397ad956b9663b75cc0f1141d1ada6620ba6f24b663f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_37-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_37-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a8d9846b23a5350f32220fad5f6255f89de6c0ff --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_37-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:843f9e22a060acc6ae5d70e7c0b59578ffd68b32f5c42a8c78be9726f54cc228 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_37-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_37-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fd10790456dac92c75d174fcb7b515063e94ff0a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_37-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eada30f7852fdeab9b27453836aebb6fd1bd6687e2cbdacd4b69fad26f061694 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_38-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_38-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2bd5f362caa3ba6951ed857b225e215d8b34857 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_38-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6660b559da266f5f5fc2227da8afbededd3baedc2b15088377a940b56ec624ae +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_38-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_38-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c5d28a9b5950f0cdfa7e9e4515a8b877a10fa6b3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_38-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e98c6390bcdbaef679f014d2e7f3ab34aaefc4f64d0c502050fe8a918d756806 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_40-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_40-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..14fe276107b05ecc2a4c993ca2cbacde71036563 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_40-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b215eda1c8a8491e0b9613539cba0a3833b75fe21b8edf510d0354d8a6157527 +size 12483 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_40-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_40-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b2262923c82d7e677bf2e109beff4f628ceadaf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/layer_40-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af066b8db200e8409de2dd1cabd3fd2778a157d87280c0597edec4616f07a96d +size 12483 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/mp_rank_00_model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6320f359de6391de23f70426ac57e5ab22e41261 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9b282b11ab8ba4b64e077e9fdff58bf07abd26ec0ac24d536d801635de3914f +size 51507 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/mp_rank_01_model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/mp_rank_01_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8e42aa93ece441c1baee74a850aa6fcc12c5d6b1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step80000/mp_rank_01_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5ccf30735c70adb9ca2cb7d8d3ea831022ff9e5d38f0dc401587a0aff01ec01 +size 51507 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e07abfe20a18e19a6fcfe13a8f9a53e18379199 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:507743113636562ec36b5aa0c6d0e227cd59a1e2a4fda4770de401fd77d9f8ec +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..034caa60527aef33a2afd87f39fd87916348b6a7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1d8751530c80e8c3a3fa9326222b9ca9feb8789bb9626e43da10cc4c043325b +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa18306930a4d90e2fb94ed926c22fe3394c18a5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0feb8fc011ad8144504a5ced3c421797d43cd05fce781f3d1791584005f1520 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e6dc5ac9d4e69da6e1c53fcc2929bb3b46beb152 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6665f8fe0a70b3e15f211be9692ade472f4975a4c972ab7e1456cbd347e95c48 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e54c6bf822bdab385a0d2f9114bec3d401600ba3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:012b229052a67627e7651770974f8c5b3660fa5101ad30d0282b2f560eea6b8f +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..88562a17a5e24119062cdfaad87f99edced4e7bb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b473c69837d31ebf910dc6877b2c6b3d2e3d10adb2c01e91acd37b245ffe851 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4092a88d1d94e904a5bba06053c79ca5ed6a9b22 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80a83315e559546922130d629c1659a44ebbd04237bd0526ba9bf3359343dbe8 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..403f27b26414a7e8e625e99447f93d14adf4e4c4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94f4024e68f1c530ba45f9305a35106fe67fe36eea9320f0e9eb8000c952653c +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aa0f13d18c72c6a6d44ed66db53d6b3625c65e40 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14d946eab4ce71407ed9c56cd07df258a14e74280977581fd92a159a190f90c3 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c79f4e7a7102ee3299924a4b8f34bb16dc93da6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9264898aab975ae117cc7ba29d326a52b39bd4c07457c44fd435ed2b1826c4a +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b390dd76a6b1b831d4f593ca8f3dac48766d421 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5485558cac13509f0c596a027517338f5586760eef57921368e70e262b723fcb +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f435f7f3ac75818cd8be501e815c5cdd718100d0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f48edff964e7d5e2cfcd340422f29a0f536afd75bee48de0298a0ecd35dc304 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ade1ea1fba76625112fa2d1c73ef40ddd123b31 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af053cf279b878f91b1c2f3fe14b6ac6d1d2648fa189b86e304bd0b3385b051c +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7ea72a6b236e77966d8588e0052470b0461b2dce --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b37358c5457be2e340102385ef2d7f9a9107f72ae09900abcc2320098a5ba761 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6ea0566d2af7bacf850f3fc9dcc538f2a0e17f76 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a64462c2c654f51e4237c40d8a912edba1233d535ee321915d045e37f3238a80 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..113afe4f3aea798f37ff567a9e97d69f4b69feed --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcc9dadc0fab09606b10b2795381ffea82f67e78eabfccd6b7d3a02848209298 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d19e99c67b1815fdff3a40e0d5e5b3bdcb41281e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:068a2b363b21e9fc5eb7f4dd58781bcfdd0094acac04041a4a73672be0a0d39f +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d4065617e4d5d53b8bcc148aeab4d82327509a7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:612d2f77e28279757b4dca23877dc7f1f24c3574985d2ca7fd2c7aca0a8017e6 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1bb9aa46dd203dbc43b21e5f3d9b2c5caf690bed --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbc8c1e289767ec07ecd176298038304e3bf9c603cc32424a9e99d0cce7dd8a1 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f94a5426bef7d0fef221d81c43b112261728869 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25a6df0e2464ae2e08e208b38ae28fec489f258929faeb05990121c1f172ceb5 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..752b50075fb0c763d3d2dd09e57cd93d2a1637b8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4215d3a623feabb9a17ebe8d4a2281ca8803e62c5fc0d570fdb660f754beae7f +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f28243d86dd53f5386afe861eed3a3e386b35d5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80c4a1ee2e4cda5a4186a341ed68b9765ee8010b38cca0f5be4f2f50a28961aa +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d3296a4bc8461e3f311056b0ae9bef307d45ec9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cf4c5782c8944c1037afdf14fa6537741c78b761c435e3082615eacba11d2d5 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bb00726bf3bff5ff4acfb4e611dcf8b875b1b2d7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30a3015963fcfff4a034b9b67ec3fbc8721b4d0a8b3a6a1fbe4c78d17c3867c4 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f1df4a5c121b5a58fa301bc9db0907cd859cc3f4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c37a93417e02772bf7ae6af872bdbab323880d4635b9ce620f35b0fa31229d8 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4be2e2487601232f3846553b50b9ead2a2391c7d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff9e03e110a0fb6985eec799f3f187a5bd7700a9a5dcc8968c2bb97fae0103ce +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ca2c9b4f0cc5f2fcf13f9c143e4c1f1b8f06ab4c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a297081f2c31e5bc654f4ae0b0dea08c46e1515a643fc5c588345b69d4cd0b99 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e8e9f38258266916dac864e9b4ff1c59bbefe48 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b458a8e841e6157968f97a552760d971a4d1de08f8351c8bef6147bd7d951ab +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dd29c42de6ad3fbd0e560e26dbd7159c1ffca638 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fea12aa669feb29d980777781675c906becc7f6f4df009dd3b77fa0aee0727b +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d87434543f45a7dc0151bed2004d9a3cda2c845e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec04eb724b8ffc943f3bb739f9e611b1e3809c754b86a174655c52488bcd767b +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..435d3eddd659174530b97f8c756a8e658db56327 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:833b3735f3f5977f79142908eb08bfda481f3114351b819c31992fa508b9ceb2 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2943a4a44957df4bb2af92b14880066d38574a13 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c589e8bd39019844e58ee3c6b8dfdf3bd4434c1d760121e1a3c72c26775c9347 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a42ffb49a5c7a35a67663c53e9705353a587058e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b77d72402e499e34985b6d3fb10859867673456196490939640fbdd15f8d327 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..455e486ecf0c9ca09c176b134790074556573b2b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e52a1c93a1503baf3a4d92c751fcb48e3dfaed1dea41c543c25a91a106cb68ef +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5e01cb54b7c8da6be85052d268365935855dd49a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c77ead04da77b6331c97aa9e81cc9764f77c1bbdf01c74a6ce38c8a601f304e +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f5c1d3e82d72b6f8fc40de1012135d2189aafd1a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69eaf8e6c41fec8beb9b1a94d7e648cf3edc4129468bc61404c0ec830f2f05e6 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..464f6be69615c9279fa697b3bd1b90f86669a5f8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14cc466bfd4253b8f5ad240e5673457a208f89b48a7427af16d5f651a42e6ac5 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..56dc9cbb1607ee0ae72c12900f6cc6e91f19573f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b10b3e0b581d3ee8387e3f27dcd238b7ef71698df47d62f8efee8fe60073bc1 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4581241b251bd75a404ad1491ce3b71abe90aedb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9976a112fccd53e3c038e94106785ed44471a101b6acb321f80b0e847b5e92d +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6cb378a44051e0aaec678dde7bd2b6c7c6ce63e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e49f38cdfee7d5575f0595d63581087b0dc047c10ce8a334acba154ff8aa57c2 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..14459911bd3875ad65938b01795acbb1f3617663 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfab0a5c7c63a9d610345d171fd3444200a6159529de4b1afb1bbbdd5d2ca390 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e09f77f1d7dda165f4fba114dfbdfb8a57c3017a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8111a62e4c4b227a169506c2e26876101ffb7d25b44d9789bb0aaa6270181587 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..086d891e997941fa78b297bd73e600ead127716f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ad211c074a441a354dffceb24cfbcc7979bac8c637e124d5ad1105bfa5257ee +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..222c763b21146b7423275dc4371ba52e3d5e9c48 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e057cee2ecec42fb34d0a79cd186a8c00ddde6542b1560ea6fcf707b802907a +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5e0de840bfa3c385994a0d3f2f96e014fbbefad --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:227086194d5c4b22df9fb0144fd7390c5a8ce2e51773055e3c1ed676c1a26585 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..02004e13945ad809af02d6779655b7f2d1212ebc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34e1d965922d9890aeeb5c58982543de03a9cfdbb93af65536b9d0ff8390b1ef +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9311448c52045cff2f3d3178871ee5fdaf1c4e68 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:715c3442f320c0e364f0a6a0fa0584475c30282b1e1871c8f3a05609bdef1406 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f59fa2d3a96e5f9b9190dc97b5d1054d383f181c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39684994545238e8ec5b142805afe65552b8fc9edc266f2477380565db6d3ef1 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f908c58c75c145645c9c2ec35fa2978fdeeb6cb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed4c87963a5ea9e10ff1477d69443b95231516c393d5583138b90c3519958f09 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e8a43b92c1293223c2184c1360dbe00f46242fbd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06c66da7648ad0fd4c3d9f2d70acf9795d3b1beb0925129e62e69957802995b4 +size 167872813 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb3dc11dde80441028577e670f3ccee1fdd9aa01 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16eaebf7ba5b0fabe4ef623fa88b1bba5e496be789c052e1fef4976e0ba9d0dc +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ef94a828d5d620e83c5693a416f2b4fa158d161d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5325d8072c4c92d71d178b4cc5da073083d9407696bde242393609ff038e3b94 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..feb5556dbc81dc4a41c7321c5db48b3adbbafb06 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5064236e945692f33ae338d317de12a9eadfadcb87e7a8c5a20830a7ec5ef01c +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..804213371865889f3d5a3dab66d76f85b1484fd9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e5466683458778cad85d63cbc7cb53b08f2022ddb8f31631eb70e3566143684 +size 167872749 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..414990dd1f547407cb9c909a863a4e89fd3e7ab0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a82b22136e056640f45b39ee17504352b8aba3664d3e472920574fcb40230a55 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0e512367f44bb29c87fd438a67f3e39e2cdda1b8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:202bd55469b54955a8f531ade80e7c83e01bc172b960a1c931fea0608d665fa4 +size 167872685 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf239050655744cc6dcfa2fb4b809e737bc9c065 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f86cd625ce0314ecd93cb46da50e570197f14978c3c19c1591cb5710598b5be +size 167872941 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f7ce84193b6f02d3e4474fa80c80ff8994dd6598 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c285962d2b7b5c59548e4518eaa2da933628cf660391850362c6a4afb3377ac +size 167872941 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ddb58ec6ce72800d26ca7b983fd5b8cbee86d0b1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f6491c1bae0738b3d015a0073b1066796d55898d07542ad4195cd32355f90bf +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b21afeca6703c361aa1bd42397880060850b21e9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46589fb800b2ae5868b534c9a5a8fa3e198a5d81ac1084388b83cf9bcf62aad8 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ee8ceae74f46f6da6b8ffa2bf018fb492ba3fa84 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce170f1d3c0f5f5c99878cacc03ae80f5596aab161c0d65c670471dab07fa908 +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a16cea20d318241c836aae331d2bdc2dfd516b6a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74c62d2c280d96b083f0e8659fda7fb49621809e3f40afd8acbe691264c2045c +size 167872621 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d762a553c1cb6b452f9d7ef941b744583801c504 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed6a08596b748b2f3584e4803ea9dbd52903472524d5497cd2712862cff446cc +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3a1f4bcd6cc6ce68af4a995a15a23c9170a1e3b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:844682b3f592f303699880e310cdc8088a9c5091027b83c7132ba2c8a07d45f1 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a5033c9556a0f7547e4ca989a17ade7b63873547 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c9300667a9169d2fc8a83cd8a115958797d072d8b980b369fc74828b8ec50cc +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2f97d478126bb9a51798088a672d3b0867f90fb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3270c156e295ef566433771e74a6184d829731eaf72c747a80ceb0c7aa824e2 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad81250ac74f530ec05e217d3f679c2bd1623705 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e5afd1f65c09cf19988e3efb0566733d20efcc83709e4ce833996324073c506 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5aa38a6d04b09a98b48765c4c08db05d0718f7f0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f00fc8b4e30b8f5b0075bfdda02da8f13f1920f5bfb8ea315c223ed43d26d69 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff48879dfd4fcb97aa8b3139ba1bebf55fd66416 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c94830eab26c6cb5c07a7df682d6dd06042eec21dbbb9e5584188b27b0ab0c9 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0ca378e2a463f813d5e368076d95c9298c62b7d9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa6744ac31183225529c420bc0670e583fd65d56c22b0d7d883d23224b04b50e +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6c530921e99598c18799b73737661964a4684587 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d92da14c1c4a0fd3b06de286f1e72607c1715284a1e03c44af4d5f178091036 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d8f5d48fd5e56531dd0a5bb8bca6f876b1d121b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:312e38d33f2b67ad0cbd48558e84815a766e1ec7c4f4d5adf3d90df3a3f38b1e +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..550f501f68545ea358d24b81bb63e34fb8c0b539 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:439749868d38f2c48b37144020cec63a32c7bf4f8bad7afd4c0385a57e1410f5 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6c4c32754164193dbc8b97a8a917446598d3f93 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0ad114e5ede222a41e1f0de03e97493eb9115942fc97043402089d4f736cddb +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4bee55a1629da3fd2f79b310bcde894e661a82c2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2480b39ed65c60159a718b95205d7c8d7da4725340fd8cd528f7e437754005ed +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb818e2c042e72e1e9c2c4a60bb16355d3149c3c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35e07d65e020c9139a19a70cbc5a5d7078cdef415b77a0552da6310d39411776 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f81c6a84f24b427e7bb675ef34d92520d971283e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d2cba2e8eb60a383877b5123e77c0788078338dc4e2db72ae107413428b6414 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a8e17c60675b6a53f46f83e2ada95fa45fc8c81 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ba13a3955fbff47a40f0fc32f9d9f5fea79fc473d2e62aa762954e17420fc59 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c685e352a9e2f67bf0fe6d3b7c1bee03bb09507 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fcd840226e6dc62651ec3a9ef8483729d3ffb0419e8e94b201e215d7165bf7c +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a3e96b16c4a8f398cd91001dc7155aaf362a9f3a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acf49e8e048f6dbef5be4b3f2e21e447cb50c492c02d0602f9a54b3bd3099df2 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e00f8f891d4c2efa0f0e5e43e8b2541ad7787a48 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1002a5798fd052e76334c6ff7061ebf905fc49b75b994fe111c33091415d75d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e666d8823e262c5559ace4626311631646c5f1f4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dab13e0cdfd796d74e97ed01f47cdbb851453feb681d4e2268251d55f6f00aab +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b3a9b4023a2dae06c41eb20da391b28e4438abdf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63ea10b7431b25fd0319edf7b7bda2c40b5d4b48e6e84b8582059435602ca51b +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5e20018884815214281d660496e259512fd9459 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bea3e7770ee3c32b4ba6b88c91c7985b43e388ba0280fe1b3347e280430aa7e +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4c11dd0c5d702977fe96a5cb33d774ce09dfa730 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:526337cf7641513aa12f4ab387209e6cee3a4aa5fcedacf33f192ad76e4df7cb +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8e84e54a521d6f91004e140bb4274d738540a56f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1a98a3e4ba7bb789bced94eeca4c4053b8125bf2c8c141e03a613cfb4e359f7 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4c9e8d41dbc5afe967784660e3e84ab8ebaf8e71 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37ab4e8f0fc3d1f56da9d76edd69642a3fd84eeada645ba9c7abef2a1485771d +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3f8b0fe1c2ad070269c0a41526e0754fbdb1875 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:190cf875e31031f4d0bc969528993eda296f9afb6f6581439be06d468c7aa690 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d6c7a69395a4e40160d3d71687f7634e0e08d061 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e1f95ace2f44a4f1387149a5cb5e3f2b52a5c93a520d33c77350d5cdef15c34 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ae07b3ffd95ae0525d2cabf6b088645d1e94a167 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39345791366643fc349e657bfd2e467cb16c2f8bcf85b2160b2eee6d2a6ea8c1 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..75bfb682296dbfb132d6716376f2fd74a49bd2e4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b366e2272646d54e9e2e60f2ff062a7dac4127daa59a8e46c096255db687e58 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b735b578608d9254d85211cb6659034da5af0a3d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3c19e568da8656cf1e7ff86dd28aa43b0586e0b7edae6d381e82d1dfb57291c +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6390cc5537abce8490d7145a11925e29d4fafabd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b6bb7b8b46859d6367cbe052fefac10642dcb6c4d7fa855dff55f0587fd2bb2 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa873ad1dd942fd096a1181a194e0ce825eb4b8e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e86d002e92d84c5892f6d55358368e8bb350b2b72a831122fc1d678ee76fde4 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..47ed2592515a31e7121603a54cbca33894e33cbb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:503ac06916b174fa85d0cfb99fdbb45a637011fcc157cd2f9e45518f787a5984 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2c1ca520c666ffb3740e337489c367ed36d41d32 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0625b9891f6022cd2a54849c1b28c6cedc52280032e063f954d30914c5f3684f +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e55a8c4e966db9eca6593da3abe16d4dcc8ed82a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:751b938ebe7154f54fd6ef645cfcedee8c882738863da3c150fd80d604e6081d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3cdd9e25c1c88c830d7acd01dcdfe73aedb2ed0a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e97be9493f6d7815b32ab05c13bfff416de7f20b8dbea617c3b79bb8cb261f8 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eebdf481fff138d26acdfa83afa46d8214be7f5a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6976dfb8524a1cd1fe9a9bdce7311f51403fb26ab5e3e1af446bf711196d544b +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7be71c30c7f64bc2bc5afa24e865cfd7da50c685 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bde168b40d29acb6ac26ee3f2209dde8b558e96e8fdaded3e7d44168cea4389 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..53c1537bd7336f6c3f28d64b41fad97073e6fc41 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ca6f961a11afb4b4f2c8bd56a038609f352d024371f65ba55b68a9ab137003d +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7ed6e6c9827cbd8e8d97d8d73bbfaa307a39c812 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23ce59f5340b5cd24d42b42b44d76e7985ec3b9ed1c2815bb92c1b5d4bd30132 +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..58dc4a5601140a8965dedb779e3cff60b017c0d6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79cb796e9375a6a2edf57ae8ebd4a090afddf8e5173866e34da865550ecfe896 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2763e93fe15a3935667dd55f5a6a7499d5bbd61 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e3488ae10551bcca99a00941b13e43fb21d9e54f723eb1f8bb5341640aed1cd +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..56f99e47f7450529ca7e8fff31bd51fd8a352926 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:695eb2fe9113c4e5d05141037e6ef49e1687746ff364dbd57c6362381c588da0 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc3d9f4c1bb13bd39504ca8a5777b093bee9d5ec --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c26a5b7ef140f51d42b74f6ce9a013cab76e37663dbee6875ffa82361031f6a1 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e9ffcd5eff55d00a092d7155af66030cdfd58cac --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da7e3c6e1460f0a309b756af1d624781392d2410abac755f32fcc0c0ad34b5dc +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aa862330ac2289e8834740ecc1a936c3d88242a5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82e45bf8edae7cd1db2d8d894a53e4f93e1a432e4f66b0319973d944f5bebc9e +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..273844e42e11d87120bb6191555ea71ae948ded5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8b8017db2280c1d2d7f17218821dd063a455e7e3b19e42759ef3e7007a29776 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c9372fca795e7b5295da875343b8aace24f47923 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71335350526bf9d8d9ab2786745522e144f60316907904d8017f47376409a46c +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a32e72fe03cb8e4b1479f9c5bb2410df58472ea8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59b95ebc4869d6835680919655dd64fbfc34b3d07a2a2fff4c175ce09fefcf49 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..104dbe37e9de5208aee01e23ffb4c1a17bed8f2d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87adf71d82f35cf404889b5cd09bc51dd323e794b8534cc827da8fb7d43cde61 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..98e4267e875cfbc2bb256951473c325182679ee0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:449923241c323b9a1bf9b7c85e99e52c5cb5bb114133aca55a46965fe3db7c15 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e162cf1178533a29d79f0f3f4f1fbf8312180c9b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3994899bf5c31d641c37fcdbc87889991d6284b0f89903f979773286bbb48f0a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..71c03836ab991b937091a7b6d25fab02e43021f9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d1216ead5db766f5b4a6969c7e74584bb7d74f24a3f122763a3d72aa4abbe6a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a3f9e6605280832282cb5c488b01515ea451f9a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:907cdc908ef52cdd36c912d8fc866272de608e72c9a515fc2334ffde373f7d82 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d3469c360bd08fa9c12f79897d9ee4fef440b8d1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b6bc26880ca3ad86f63a7e602009213a78c2efd4f703ff1c6dc38ce9333943b +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6d29b71d42bc784203cc2dd115fcc5738ec64e83 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:147a03f792f6a13187a154d53000cb5e958effdafbe2fedadb929abacb1b76eb +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fbe13d05fb3c7975996d558bbbc7c476c7ae94d4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78181d360698901a543177a4b6c0cc1a8dbdfc817853f0db61d867bd7a17bd5c +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..52c049ad8e53441301ebebddf31fb7528c36cce2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:838e6ffe99bc7f0a91b0dfae7c20a82db2d1f20f35f9ad4aaee10ad1f3c1bc0f +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6aa0642ae328839b4517c99f13d2a7aed73931d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1764c63ad2c3574acea680c99a729f47c585b0b68a752f467844d5000d0a78b5 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f42bea5888276eb8539f7d212d2c7ae6ba5a3849 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d865a92bd2f976438213f2cc8da2bff077b3fc5b58892670288266507b1654d8 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..40a3c522dfcc35dd176c9d2e42fb4447d58647c4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5e9ffeb10515cf7f31e7dc5bacffbe96cf80ee589cc104213571e39d4eebc8e +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..43d4465ff6e4c5705b06060a97b28076f74946a3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:756d4bcd0eec4ae9d399e6a69976ab5cebe020933284e85c828798f1b60b2a5c +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..53bae6c7677c3bbbe415d698d7e0fce5de7021c2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9414a82560274f1bb4bd79cb1148b999af2dda7ea44231740b8c8ce81da9554e +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..548715ca543cdcd0be8f30f668e41412bcf85caa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59ab2e378aa76397688a582c2949a8b3d7915ffc348baadb41a244c8e05430a8 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a5e26bc2aadf05ceb24527f28644ea187ea5940d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4ab2601c91b812be1f53b49c3cd50cf4bfbdddebf0e83468e01af4488e2c30d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0384e7b8176c14f75bec801008e83de126429cfe --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8ccc97cd0a5cbc6928bd3a76dcef02608560beeadd8ad3488fa36c06e340206 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4c6e8d9070af53f0bf970dc869fe77b2cda79adf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9442460c6c2e98d8b21a1eb4fb81e2743b5cf4c4b3631332706c3b267b108af +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e67d8b16fe78a44f45a5a410601fb252f24c9599 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8774ff3cd5f1e0d035ab4707a89c75bf39486c86a4e48d45e1987f1fff713ab6 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4d9342f5717b2c2ba2048db51734ea884ca98005 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97ed08e9e39dac7b0fa7ecf663d36405d172ac085073af3f605d6d18112867c2 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..10d32969deb43e63b7aa70ef8d823178b08d4e5d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f77de2cdb0e34ddb82dc4c392236f55e1f169cfb1fa94a753149735b425177 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..677df0ee12d68cf6659faafaf8daab1d2b3c1ffe --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ab4242dd4aad129e5f3c50bd8ac4fe1896382824de66cad23ced17b9c9272e5 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e86edb36e3d9cd1580ae9550a1cd5dff0b60c22b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbf1f44bac110e13165820122ee3d7366a2a4c47eb7944a291fde41ec96b4490 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb775586468b42119415829549d39d8be0b3c889 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e030a13fb878a9c1b11ff77adad675dd0e48535cf874f9ee4f683923a35cfc63 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b677446742e54eda4054b16a3c765afa3e6e0742 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c78ce26577857d89f824977c70e0c8d7ef0fdc60bd16f0b88303f5a4bcde972f +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d67c8369d29541c9016d50741edec3e5e94b505 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a282b48aa1e2a7ee33e0d22e62331b29f6c91c246f2e2d5c9f5945356d83f493 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..20708ab6c520a72d8cbe5c9c93c5c0fc8bb9af00 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da105449f21c00a3987a4d58d702c833819cc0df073491ddb20fec8d375ce772 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f37789520d6c2dabed663bece8b875ab66bc95b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3161fda3b370046aaeb0123e44f62a3a26cb7be3da9e7eb22721bfb801ed7bc8 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..250bccb0e63e78da5827b16b803fa733f56b5a9a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36da9debb61b98ed82c6a56141e014a9a3109522171d64a01925945c8e923f05 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ee1877d16307e962fd56d48cb4200dc94743920 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:336617091bddb6a2bcd73fe3a3f5003f14bbfa7165c0b73d44c1d1e52751f4fb +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1fe73e7c9bf246bbd2f9d272999ac9071f878645 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c41713ed7a37ff45d9127c28eaa68ef22e6929062af04063c1cb23ea021dcfd2 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ccd70dd20e0dc86d587b2fbc31c76d2c3c179fa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdbbf1e48652f20a481262b5fb816c4d7a6ed64473d872462dac9b9f4e69982a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cac53b8b6a04431a878fcf0b07f7bc1a1df66ed1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3592f2c36e4757d9e13ca49830bd8cf166e3f5d22c353c52546bfd7cd498582f +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c68d90c6b1088da95c269df64f85772c88ff38d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8129a743447b03f14c87366a46eaca8d3faa5c6931c89e53799492b557da0899 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..46e960ba1a59c638d0a60bb0a48908e81b546abc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5eac55e54aa3bac9a505ce2e82afabfd73e4984e2faecaf36b2968eb59ddd9c6 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d35dcaeb954a1c2120c14f00d7bf135c376d8f71 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:452823c5740f88c73e9af6ffcaf5840c546ff076bb1c17d03e08512193943f3b +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a06c559de329572e4e19a77f7c2979e3f5f506f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ec1428e366cf339ac08b445d9f9993f5eb0c67dd0a1485716acc4cd6d00feee +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..27b5201dc57507ee8fe637f2f1b7b4ead2741a5f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29d7bc221215d8b0d4153123dfea9e34a285fa1f5691d5c83c1fd7346433f518 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f03d99f10559f58c27180d340fccdc9f635cfb5d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6e0889e6d25aeffd3eecfda8996023298f07fa49cf8996b438efe62c91845bc +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3d736a338d1331ae14b3ecf475d71feaa7e6e42 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:952f2a639e09e85c8fa64b34055c78503fc8b12866f3a687f949974cb2371de3 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..623965470316e69d02528b7c81509f4b45963fc6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4aa5971df5034db528bc5c504f2535cdd95d13f72206d328f90679e3e0da60c6 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1a0f0fc2d4d3725a895dd6c983d8327bb4f452fa --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6afe9477337f47302becea47ff74e1ac4db67c9a22086651000fe659cd004983 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3707cb43a8db2af06789f0b0b614809d8ef94bc3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37ca52897a0c201ef3e037954e699d7236dfae448be2de46180288fad1426709 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ba30b3e8e197f86a5c2d4dec3a5c1a321c44961b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f7792e37d3530a21600179b194a367f8a3fb4be50a34c12544cdcda6538d73e +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc3ec856369bfee1e8f1a345082f9f7bfee2efee --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89953bb1a81f6f95ac1370f73d3321cbc0ad3d751aa6948d501edaed6c6a02e0 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3255cb952b7320cc2e7741cfc7b98c95d51cb005 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb4095cf652b51098ebe24a15ef58dbecf730e251bc3e203be3053ddeaedb36d +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f6aa4f5ee8b13413731ed28ef2ce9ef9a7d5d37 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:730d8d94b96bc4208b6cc2588ba61167e5f046e6c1ef676baff1a20e738529e6 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e72e7bcd1684107e4eaeba5398700a1f4b205f86 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5ec93f8039e0894330f875ee9703c4bdda5b85d9be10b28b0cde255bd5dba10 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad86b79c081c781299febf3fec4cb3c5d5fd28c0 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddfefeccb886f261dee831eb58aa4515f03d7c2e9562ae00ec5be283cd1f21c8 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..05884483474067ec2659581460425ffcc3321f25 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:196edb58875cd870650046d67e20fd9758ad3ce1ce42acc487ddbeeac521f2c4 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d85808f025b632042cc68926606fb07aa177d4d3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4f1dab076ab3a5ea955a6acd762f3ce7dd25d1f5c28ec07744ac0d3601a6270 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4b8b181890100f6194a954fd46e92cbfbed25dc4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5ab0b57a15f4f3a2074ffd3914bd7aabb8d13380b25d4e0452163e97219b9df +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..06e85bb7c8be5b01d4119eace4294d9818edf203 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f745fdf16cf122d3f6da2f40b529c7d481cc70d70053cf6da552dbb28c6562b6 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b0fa8fa75f5ad5216e9c44085053197b794998ff --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf3a4bda2d13300da4f0fe2fa7f7b81dae789cb11745313e0227e9c79c3dfdd3 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..43819f794340483ddbcee9b90775f00cf48ccc10 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9faca6fa7d7f41303e84784edca38b44b56e38601974d78cb6d317c7b469663 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..85478aa15ef13ebc0ea0774d96e468143b0ee946 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:058d62a439cc855c4406d2fc7cfbd59b9c5d1d56aa32913998ecd25bf68c5be2 +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..19440bf21e77803def33626b4c18e57533d16b3a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2cdd3b6307d9ff787d1736d712ecbddc146c411e92ad45be87df1932add6b1c +size 167872727 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0b3ae0ebd7044356d7323405bfdd531fcd66bb15 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fcf6437e9d649c80957d7bb6be725e580c60d0c616500db2384676b9b0602be +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ef4cb192775ecaef73c5786c2b9b20231c3a063a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec5f1e77ead6022f5b1992f3de0d954de6e70fceaa9bbad6edd812f5389d98fa +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2881ab60d7a6e4292a06ff6f2aeacc0f81e24666 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d73991bae8975dbd982056e5a918c869d74c0c586573a90669e6267b240e04cf +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d82c4a56793ce901c45b318a6c319d25d8c572b9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f241ee7154d13fa91efb9c585f6fbf403494bf862f921a78cc77df678eb3255 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f1508e35e6a9f6a1247ade7d9861ff3b6b627da1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:861f7b4c73ff21860318c7465d6658280a6bd00f19c87fb38a1ec5a4cd85d54d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9ca0424833d167c84f334631bf85162d0e1ce7d2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20840cb548e29b9b0e4e8798ee228f91cc6416cd407af09c165016a40aae8883 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f9ca948994ec109dfffd0dcb477d7a4d81b902a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93ae74218347ad72fdd7152ec0b7e75ad69583db1185aac92a11ed613c13a8c0 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f4143e9c6fda0a24ccfbcdf2a3465f5beb01059d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf5377d51ce4f645451c9c5875a9b52c7a4d4bd835b74886842f40dd1f333bf7 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3574dd237c3f1a0351c389701a31789bd69ba4bf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d7bb57a7a9ae125f1147f0a6f9093ab92e090cca5f22f211b0f5a377d3a193c +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f62b5b94fbe87c761316fce3159a8ebf6d42422 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85450dcead1aacf5d1f1e8949d8c46aa9e0659eddf934094bd80334a1c27151a +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d593689d05e5932abab26a60178e945729c6b24c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bf7cbd3160cf1aa64e8a33e56d3aeefd74fa17826e8d95b0f8b9b26ac204f59 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e764dfb5451edc92a2ea1caae61343b20a30b645 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e293739554b1979540f507678f991ee33a2916e7eef112b491b2bfc61bc9ae3f +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..876073e6bc5a736a29ec22c6dad12e4a45217289 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:994cbded5dd682ae1c26d98fec397e2bc57a61cf398de6b493caa50a6ce0da67 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..872dc59dbf3256f38e2d1bf755d020a8a8012d3b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf3b7c915f8d61757413bee3ba88294166f497380ae506b587c20598852c7fb7 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0905b87180e021f7bb902d34cb0fe44afa6e70ed --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86492c3190e1deb16983e2071bb0b0561a06726be75e93c8e5d18518b1cb8b75 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..532302114faa1a4970f56971741d968b976fe3db --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41ea3c29ab6449f94afc6040b7cbf66e42d5c83d5c44a1a93f141bc743d85311 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0aa20a3f02255fcea9d630ef4401df1e5967afe4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2b9e73578cf4e724a46f07609c54b59b04a8f070c0cdf82e10e55a2eabe1733 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4b528e09af1e9348ab2ed2b43fba542ea3fab1bb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:082f58037fd129c6c2d9941eef90d999473880404416cee01266b02982dd61b6 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..74000826d6385023ac883684f9c24291a09b80ee --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95445aa357e83dfbff8d882f8a13fe75f2684421a2b55daf9303f76459d992cd +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8008b279b83f7e05cad841e153ca7105d2865a10 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fb4320bf8ff44dff9575fa66f31b27e5e5ea46c5c9eaaec74fc9b8eaf73cace +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..16264a2e5d448f40a97573d4c82513555bf72d4e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01d2e0ec26aa4c5c181f93c61e2da60e4136161f1ec8177cff958ca288e6c0e2 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2db7405a45c8ff92da9fe69841e98ab4fcfebe97 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27bff0d6803c7cc69bd9e147c9c7e9df1a590c722a9582bd1983c1287c1062b7 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5eda9fd190dc86ae13441a47727065e0a74fefe1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27da2a751f611c9f8aee736a3203ce52edb3039256d908c6762a13c5a3aa1dc1 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5962b2af87e85749cccb9cd78bc33f5fd7af8e9b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a309eae80c246cf6aa4cd158d8ff1564a56f7c807030682636d99a16ea56c7c9 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e53a3c1732bc1e31cd69bb3637cb60acaa9bcefc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2aa80ded8e0fef45663397f184505722e77aafd14082dea6740bb24733812173 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e9d4a219397debb6e7f9dd7d30d4fb4aaa963386 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45cfe82b1066d8578e4cf21589bfb2786c2d75319a049a03d2dbb025e3e21b93 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b559bf8ae8710bb36d96e8675d7637be43472ac2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f39d4229403c2bf6a40cca4f05d8687c3bc339e3bfb64d53785da398656a8045 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..96d72649139ca92633c1b278f2c45fe79e01f4c8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c59f673533286e0bb8288c848bb20cc8c8136a5cd443481f4a9cf64fdb63d8e0 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0e6d28cdd6281739bbf88bdf3e795bea67ab9fe5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c42fae5e52ca2deb53ab478e003a917124d8bcf2333361bba3bf2bf87caea8db +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb5b9f6c7c972d7685f69c3616341755ed170847 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b942e0ed21495eb05b7e0551041982557ac858848290b21d8d9b2a41b43bd08 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..edf8fbfac361e691a23c8c925c38ad35c0e25420 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83acd4692a3de58155c0f5cf91a564caa3251f1f63ae58395e270457382519ae +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..39878434e0ee1a3b9544b8bfb75f47dc65d6c8ec --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25de362cfc684fe4886d5f59dcc14171c2431a30fc813dce1af2d91cd2d3d564 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ead2a3e91906a873467d8deaecba0c73a8153bfb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ac72beb6641f549130b1b5cccf0f75f50d28675be24ab0586541fb463de69b2 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0714b18e1eb22f9b15c6eed9e353e2c2bc0138c8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57571a116212b58c3447976c1ef49415d5793f979291919965a1de2fbae9d514 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dd3ac838424a781b10bb6c04f00dc2f42cb4007a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:061d0811b1ef9468861d46cf8c392dca676ce119e85c2fffbbd52c2f68570238 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1ff2bade8425a1cd4b309fc9f4a7b7b68c6e59bc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59d643dd98e09fd4b469bc9ac7b10f2cc95bc542d1cbec1a54488eeff7e6cf81 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d1d2bb6acd779a8c3461d756ee2c1d502b2bb57 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bf5d590f24941c63bf399188ebb3a3419e36df0622d1025c5c36f95cd90f620 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3d53eeeeb107ab64466061ee0d7b74a06bcd3c2a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:309756faaba7dbfe8c133a945388cd73fb036d8713b0d48b0cae0fceace5b530 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb5b5eab53b7923933375f8a920d2a7e3cc6266c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0a0c37de8f908dc7e902753600d048ff5b41b9e74eaaec1615820152670f7d9 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..07109da6225b490bbb272679d446c22ab15bf860 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:459522946a0b786b8f9941881444e4ee84a75c1f138a89df3521b78599e3b338 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b52c76c3ccb16d116bf8532733096e786196fe23 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0480fae0f2563b74bf9b84f4832ef67105544671e0f632dd268f114c8e11ca9 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf31f2e0e0e921587ead95dda59669c73d5f0545 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb79a78e71ad1946832a9db396a583d9e343d3d70bed43023872a6cda466f976 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4680e36a7b898f3282f697d54f258b628e25a88b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3c912ecbb05e46fd6e38794fe74df666edb267773cd106f62b93eccc364ec42 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2b07b96c502d04ddf0a6e0b107f5e9ac93fd25fb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c479c2dfdd4ff0a6511dca3162a6ce57ce01c7dc03d88f655ee166530372004 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9dd3715cdb38b5540fb929a64dcae2395b3a3dd4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:106db0d8706ebd1fbf156562c83b09b78d74bf8e4684524c6cb2fe28f5b14b93 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b7b100e1f754e35e0321b05dff8ffc98d2c1f89c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00d71aefb8be0d100e6121f8ba1fd66bf6db4f6dab961290db97573f3ccf627d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..596ef3b29f1407406276cd5932037b418f488ae3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:693eeb368de69863f7ea7db77ccc78abbb78ccbfcbe349f9333714750d2cf54e +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c5afda11e0693258d8b4f9163476924c9275625 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed3e3076c6c1d3c0cff4b0b1a9b6058da9520942f648024a3a1a6e038c6ef662 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc0b344a3fc61870f11d77bc49b33413d9f5c2c2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:660ae8c362735540868858bfee0c9ce0b886e5f17d459b4584f3aebf4e492f5c +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..30003f25175883a49ab2349c79d67a87f8a766f1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d3e69aa2ad754e6f382ea0069ec9931d87a38c518ad2219c7af921f757f14c6 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..948c59407ec8245a2b988853c3da70e3f9142fcd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c9caca813f3db6732b1ce3861ff4b8f26d1b6eb01c82398e2e20b90cf6acd4b +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f3e169844608fc1ce40c0abcc0f79170ca6c2e2f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bccf5d9f351fc62469ba5ee187e90291ed30f5fcb654eeba6a1b843d9cdc4d7 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b14646b7253a93edca19e8e9ff60dd57f43b707 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2d60327f5e7c4c1dbef5f9d278e418ffe731daad5a70a4a2df44fb35f936ea1 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..481f925fd6806b8516cdf93ab5f9cccee9366a97 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75d6cd20e9fb15685bbb098fcaf7b179d1a63577556a8425ba6d4155a1109072 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8df1d1315453b8dc7ffbe22a09f1230b24b3a03d --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e589035d88650b43385839bccb430f6afccc4755c3f7c091c01d0712758447e +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..850287a03426b8236cc775de6c21ff62e6d30bac --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da7de2489d84a0643a77b84b6dfb0c8bfe670867bbbf9b5b2b31b72f741e7507 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a726e462cba75a1e90220114b18bc5e5d53130a2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86306fff425e1b6f89a6c1bf3a07071e35fb64e11336688fb438589b20306847 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2bf2009ba0d877019b76013e5774e133222cbe9c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bac0eeba4831518b306e0ae344b117098e56162f752fcc884c38ee3ae7419de5 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8e65a70aa5e1363ec5f2afa3381a19cdb5224f18 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fb870f3489bf2e7eaa74959d55082f5dfc8d2f34202ca4efc3013b118337ff1 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..58775ba86fefa615543ac498a73fad40b419ba4f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59ec5c32d75961ddb78b3981cf7d918edf4d28c3fa6170757f0114d6c16409b7 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d9ed6bac47f43865c9e37d25d89a82447e8ce2eb --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff3209f573705cf7ff23e84068615adb545ef119566b90b1935f08a11133109d +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..caa353fab08c61776124be900b18ab590d1db8cc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b2dbc2546be53abc737abb0b59d7803744af14a75320e449894419927706e7f +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cc1790e91eb03e29321b58b27ee0a39a40644c4f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce35a6b9bf6acc596453b5faa6fb1af47c2ec94bb3f04262352b51df20d6861e +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..10084e697967e20e5a5fc38e47921b8d95aadaa6 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e2dad5a11a532aa2bee71072df2caccf0fbcd402d80cf65e4efcf8e447fca92 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c11d2061d66df45f316f687e96b02b4a29fa44de --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76e51505f2ea9d3defa1eccc88eb5d7d59e76ade008a5a9db64495ca13f7a21e +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4dfdde5e53e7b87f3b4d806157187323f9f6b895 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1557592b96618f02fc7c852e10543c607b35440d9e3eede1d2ac9c4b010b4e65 +size 167872663 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b36cea15569599a0e6d215cddcb61f9cdb448ba8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d485e81dbeae971f71aece97d156907ab7959b6f8829084af2de0940737e91ab +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d4f8d3563bffcf6880aa8580d1674386d638f55e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f56ebae673a4ce0b38ce3b1d18e9889e5a5770a9cfa030ad06a11a9debb9b00 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..66e315931e8743370f09efd1205c012129971eca --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6de50f333bbb78f259cc48e9ce87c1b103aeb066c52a8c511ff66ca54cfebdc7 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..83260735dd3844de786db8960f42d8f7f106c138 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53205565721cb18e90fcc06d4e493b27de893b2543180a10033360e9c1fb5f26 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ba9474c8f82c3a8b72f774eff98e1c62bdb67f20 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5255f24a1262e55d914f38f44d9a61e342acd6a46dddf30ef616db8e358ebb6b +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..091c781c2db8734d34184f31ff53ddb2ee581e10 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7bed99b38dc0f78175b2e3a1d779a73a96cd11c577d98c0d1a52ba9a970cbff +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dd60a9477b9fa4d268e977ba626585e2f1032e85 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c59a924f01d4fb04b01a30754504c88cf4ac799f65469d621b9871f2f4e679bb +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..35bd756216e08d0090fb41d638249a39ce75bd3a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:561f355374e624b2af9c8f17d883a9345133291c74ba8df2c79b291586aa5013 +size 167872802 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..66d3c6bd6acb4f5656ef40b812dbe07495f42ecd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4210e460a2073d1b111ed79d73f9c66de30b50771d54929a8c4bf067e93a49df +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b9d3043d9a43271e7c5fa2d8e2a9ccd6912e1bd7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9edbc4d43f9c38dd3c5e4da7b3f5bb00177b0c3e86954112d589f4fee7fe4c0b +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..971d0b5cd7dc5da37f4a87118ad1d0bb01d7e06f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f2f1c99100cd2f315adfdbb9c6c95791d55c1fffd2515ee6d8382da7bf9b1a9 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..887851b1da5881ddf9d42ac3dfb52c15d9309b38 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07f43e3c58522c22d6a02e53feb1e4e4af9ba6f2dd4e18bba221953d4c6e0e97 +size 167872738 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd9b7c54f7b86136570e30e21aa1879fc88e1404 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd508c8915a9538f660142862b2051f86548420bb812046f6ef4058845aa21bf +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5925439f5c0aedf00ba1f3f2fc64bfd6c345b320 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fad3c4f237251cc3fdbfaf1daec664173467c094f0c8b65ec5166d76a526273 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c0a46a85ba9140ee44bb68c259f23e4cb2802bb4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45b17a3c75c7f641ad92f58f898ced93c6b2ef82a268b5ebf2b8cf27b8fa16f7 +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..195def931bfc5f90b44615a8aef637756c5c3da7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddc17be7eda5db2a6fb46fc33e8d632326e93ab67e11b9951b8af1c92dd25e7e +size 167872866 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..64ad1db3375fcc8a153cb85ed39f742dbef34979 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f68a815bbe507074dea663b02961f1a1d330cfb120e3bab0c7e3f640d051d301 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d77bd3cf92746d4929b1583cbef5e0ed18887a6b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7af04b280de93b8eeb2983e4c861de28017515d3810cd45fff6f3c239c3bd62 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7acae1d8f35a9584f7072515e6a8f9c512e3477b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f454a91115821c95b90aa2a920584ad9fb8b07de06da3cf6d86356ca5b566f4 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..096a14b5612fb6002920f85d3a4fec51a3cd368a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dec70b4487e9fce72f8e2d671361ec669b37dcc8d6caaf3914bc32ca60f0cf82 +size 167872674 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3d51bbaa6d41bd469563cf9edb836eeaff9f194 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04f6cacd7036e40f5788a02e7bf40dfcbe55b35332b42c8cd62bb8765ec04ce5 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9871dc44db6f1a4187af416f65d6569950f28386 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4016dd4376a60292b228e929735646000a460ec4be3e6687ee94fb5baccc2e9 +size 167872791 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_01-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c905ae60e49ed12f96546e15dcb6f2ec2759ee2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35a205e4bfe300d6b31a4b362c1db90923a0c3c4ed57c55e723a8b9eff44d4d4 +size 153552131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_01-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_01-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f2e8dbc88a417d42d8e662c77e626959f87c133 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_01-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:119814664081693c2e435f9cfe08b4e60bb91e5e327eb090c4fc4eb60b259e93 +size 153552131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_03-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1697acdc0d34deef0935fb00a0bc2187e2c71506 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da24e903c9f35848ec8b15ce98619657ec5cc5620efdb50d27a440fd7bccdb56 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_03-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_03-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1739f0c118d31b64678050ba171cfd42062189ef --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_03-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e9dead14d8f881290687f9e93cccb9ce4307c4f86de520d9d8e0a6c333eef93 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_04-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b6d1d9506cad12832671a06485b79d4aad972fc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a9e458bd59e9f8438bb1497544d0d6b44330a59e4b2f670362b0a9cfe4fe89e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_04-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_04-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f75f5fa0f3f5527d238346796618738dda09ec8 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_04-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f9a6f02f81a65c0373659d9a54ce143e55f726f18cf265c11904ee12745933d +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_05-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f285ea9d1f63fc6de7e2b69ce8de4163be9cedc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67a800b5fef73680eb7d4bb7cb5eb927823a8f9b592c1c29ee6239cc690c1347 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_05-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_05-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5bf62ecdd7be6fd9b297ec5fa1f891d1d884ccc --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_05-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc8f55b6373c56e9d54804d2738f616f9f267844214e2ddb345ae37d3fe65126 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_06-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..48bad73e9f2d228bd844e830e964bd99f429e61b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8aec4af2dd441e872a3fbc52f4b87491f156b2f6d6a4dc8973481d7df3449ab +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_06-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_06-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..80896288d059109d765b1c3bfab06e0f8411ecb3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_06-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a83b3991da989d2a738c4af365d983b8a679814f7196c3272261ff776c39e1ed +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_07-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c8ece921670345d871f6f201d93d5f5324cb1644 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d64426d488d81bf4bbb3dfef5d09cf758d7c8063dab4129ee1e7d2339bdb204 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_07-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_07-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0fb46db14dc013b18316447b34ca8ba8f6a5b79a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_07-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4eb2cc56b8d93532e106f8d6ecb1b59d1bf27fb69f7db3ae8ea9d9ae33f0a949 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_08-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..939f0b01db0d85148fdeeeaf8558af916e5d5c3f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01b5dd4af3b6b56796159d997818fa88d4f61ac4294e59ba7d4356cedc18bf86 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_08-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_08-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..39cf5fb2b7c33b262ec4490c4020a2b34b0b30ed --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_08-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b55d734f45d0bf61a09bfcd63bb2f0e6208d2936db9d4f65113bb67337f4fc6 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_09-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b100eb82c82e00a30890cc8860f303401bc28a5a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db51c8fa633cce079b534053cda558a0cdd2db6218d2a13499462243db52ca62 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_09-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_09-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3edd18eecc987012df90f1bf4516661a4a8aceed --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_09-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa7248d06a835c16eed0c4b2519786268dde96fd87d86fd787b5a8d9ee5751f4 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_10-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..77199f99f36c7372d866fb2bd0f92260a3bb9c2f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:845133cdd0ad72bf9ef9222b2a842dadbc5f08be7c110f9899708d9d57dc10cb +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_10-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_10-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e0e9fe93b0891482687b3ca61414d1f23aa90535 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_10-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2620e537e72c78fee34b9bc8df6ca4847b4cb2051091534ece39ecd556322611 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_11-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e144b7aef8696f4d04595c6cc59611910ae00bab --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9625f413a93f276cbf236c7c773d45d3f209c68e6493af7ab1eab11b3393560 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_11-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_11-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b03df000f21ccf6d3fb7b53eeef6ba21db634ee --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_11-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26cbf359c4720a622b721ecff2a15b8413990c46a5ee8494cd28076c00d47590 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_12-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0907cdbccc85e5185dc248d1495dbc4718d7d58b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc7de46019b2036b7fc24f7fb8dd73cd0116efec7a1a66d41b4ca09a0c493256 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_12-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_12-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..46e5219f624373f1347859a0f7ada062d374a8bd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_12-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d255e4a144bb5d26bf3361894bad5057c453127c15f3754e0c074f8bfb35977 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_13-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9ce4ba194eeb47acd17187d85ca4dc3ca0632200 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b221cce3c77ed5ec9074feb587d851adac11b9a933d16cec4c40cba7724f8a08 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_13-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_13-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..89879c9ae8bea096be0b239ba3ea088dd021da95 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_13-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e41e75619206737ca405a08a518423c22d19f41b83ab0a78d98f625c3ace17b8 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_14-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..110f2c882e171d2d7a1443c6eaaed72e936f2bdf --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47f9d99d50e203edff0d13d30648788ad969f977d246b2247ccd8218b9a6c47c +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_14-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_14-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..787056b7f146e4633b12a8a71480ca806d7a0f29 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_14-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c97a6f436b5e27836e4160f2dcb26f4a60b68f037ff995d6b4347ba837c327c5 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_15-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..690de23f21a49bbc78e5d56e9d85f39044626ef4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d455ac8d874f50230168d3933edeed774fdaa8737f77ba350dc7e0eaa72aec8 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_15-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_15-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3547a829ce88f10e1ae6cb4b0333e2bce0184c07 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_15-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0cf360726821f46878850999a728cdcb6c8c71ca4f522cc96ed3ac7b80118ef +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_16-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a29216cc24c154aaf7ae084da8c32267245bbda4 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5c92bc9d256064f4f6be6264cc0a2d47d30cede1a8e6b53f297d4414e7df7eb +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_16-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_16-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b31c54de90e631cdcf9ddc17d404cb346f847c2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_16-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e217a358e87ea2391146f2601739278d12ea4ef4d31e4b7cd7d5f9b8f18a18c +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_17-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e4e040a1eef394e1364aea1074714eae5bc57940 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ab2a2402d17bfd6b9322e61c232ef502623931f11aa02596e1c4d141f31d872 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_17-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_17-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d751fe87cf7c8e4340a95320f97404c3d0e6c7ba --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_17-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2aa1eaeddec61c1f3afd12d62f631d436210855c7286335b0d190bc5bbd6f1a5 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_18-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2ca9ac4aac34f397124052cddd62c95a7e9cc94 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64929e62f15d0e4434eb83fba280ab46c19e2a6c54474fd717e6c695374f184e +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_18-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_18-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0388b7632011374a344888dd3bcde0aa5259ac99 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_18-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ec7e0f176611edde1251e5371354b7ffa97b542e25ded0df3f63c158b29fc7f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_19-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a24fa1175073d66349af99bfad7196996626a5b9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aac5102ee8cf316bd7a5e743d98ca43a12dff076ec7c42864c8bcf613495583c +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_19-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_19-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4d002010fdc9570ea9783b62b88bf42c3a684883 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_19-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a13e30172a57e643797c291a78432b04d889ab0b3a2a297681447f27788acfbc +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_20-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..07072311fdcfe9c0c3b0e17307f8480fbd10b4e2 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5a4211c778b371c2765c2f178b9e7b968c2d6774364e1befe1faca0760a41b0 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_20-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_20-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b431c9665336b68ea29ef9b5a61d58159037fe5f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_20-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af672757351eabc8ff346be7cb74123425ac9283f59eb339ad02ed67f1b9dbce +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_21-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f5b2f97853e2ed16decd9afa9f89898be3d6be4c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:001602f8e47b7ab82bc0d940b8e4fb22323b2133004ba4f5a8fa91c731de11e4 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_21-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_21-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cdfecfdea23475dd7b4f80e66da918141f868984 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_21-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:644195559004179044d4426da50fa4ae604fc54ef7ccbf374ea0920872761fcb +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_22-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_22-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d96174579869443be36975776b6b6c9a6f2f9e0f --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_22-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ea4a8cb43f6c04f6b83a706ce6d0f8f499e75318570bbb1d694f18ac9245cde +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_22-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_22-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e61c668c1f3af59aeb9089c4683649bf6ae45890 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_22-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb5c16d924d6337c1475b7f89b613c680b28c8a55f0095695ed31a9e130d6201 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_23-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..513e65b3503a3e8a172a87586fee00dd9fa039c1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc041eba06c244183652398540bfea356625512119c1b929d6508ef35588ed66 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_23-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_23-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c4e635d43eb4ab357faf946bdd9fe325656364ea --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_23-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14f28389b6720fd5ca1a658ab1a345cadaa39ca4a34a81fb940720a4854ed1a8 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_24-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_24-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..db74616aca1843ce9aef9e4b77cbd5b92d0d031e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_24-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1222859b2c39e72e3ccb5d723ba41136e2ea36013c47e3c2b6adfea5cc6a0226 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_24-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_24-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ea8321d087a0f496beea2b71b89c3465e381687a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_24-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:331f5d7e1dfb301800f3bc13c79b6517c7ab5ce8b96acda12244e981ebfe451f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_25-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_25-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..14e33321ff71a7505258f9d7a2eec6b9b90110c1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_25-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79f94090825b7ca6d83dc7d46ce37824cf846568c7dfc3ba19cf024a579636aa +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_25-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_25-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bbc6bacfb398a383f251d14503e11ec445417d3c --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_25-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef9a8f6418f0aaf81b9bc705a49cad3e8065d73da2035ccb12ccba7101a9f683 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_26-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_26-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b1571aee6bfbbe1cba3867192666914cab9ad667 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_26-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cf1bfbf35847ebdec8bc40ce3f228d2dee4f8abc0174e0a775b3889c53d05fa +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_26-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_26-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..095b0954b162c049d19fb3e2c306e03b0e96d791 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_26-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98796b297dcafbbc72d2c8889605bfcf2884d9d6b213ab9cdc1b6566860c6bad +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_27-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_27-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a9c7740f826db45dc6625b4cd121d85e9ee5bea1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_27-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b370c4b14084c2b612a426c3ba8f8a9e841cf2efc8a6799a61597a7cbfc02ce3 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_27-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_27-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7764adfffeee9b5441c22db6db19973c100f6bab --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_27-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d62fee7e5314f8dd058962643fbec32fc9a123a0ca985578790b9d10243bae6 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_28-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_28-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2fc0bb3e084bbd58ef4900236e2b5ea446331124 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_28-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af2c2c26363f40085ff0f1ea89d5df6e70605f05b26f919ac813a81239e5ce14 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_28-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_28-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f9ec3db9f63a9c3844e40195ca011554f0fb811 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_28-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98236eab2f5296e3ade204a21f7bcaa22f0c9b07254e3cb26f5fa64a2f3a3d7f +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_29-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_29-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d502677a0a1b2bd782833e28aded1ce81ed135a9 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_29-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ff9cdb1ce834c631fa381ff5440a1ad5adeb52e21e7c880c70cfe228134bd3b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_29-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_29-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2823f0f1b62a8188e160ebe0348e6d31acec832 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_29-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98026707ff8b933305b3aaa6945bc75781a5ea9159daf91aaf56aa37e7db0d44 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_30-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_30-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e9e5c61df8d48cab03168da44b004b37dc9f1cff --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_30-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae0a67da0f088d74b17eb1d8fbf265f5aa0f95e0991d012780f12ad68d7445a1 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_30-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_30-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa1f7e1af1b0da4296c95faf1b4c6017fc8d8c49 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_30-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a649bd00af466f9dc9df9f6fe033c84a24632f810ac923c1d4bff76b93b2cdee +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_31-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_31-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2a4875a32111d6c1daeed78700603cb3800deca7 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_31-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e40bc85212c177cc5be52e1cd3033a0cc74aa635117940860673bdb73592e24b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_31-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_31-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..33ce239898a2d0691721989c2c5a630a368f8eb3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_31-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd67265ea060e2bf10aa59b2dce05fdc44baa91ae90d8824d28ea628e5b4d8c7 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_32-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_32-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..59102483849b37c724852897bc2402203e596480 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_32-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15f2152d14d5acaac392bb6a55e5bbedab24bff09701a6d19b5c73f832e6a439 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_32-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_32-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d426355b844f87ff1aa962489f1691e7d0011a3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_32-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52a19ec568bb8114c9d32eebda99916a7ddc63ca15e66ab006a4a02a9db35dc1 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_33-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_33-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8298c8ba7b12dc4c22a7425a3a8d5eceffd02e3e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_33-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a85d17c5c732b2f65b90549acac7975b5fac4479b1085361fd7c77ed67f3fc3 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_33-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_33-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7091aebc8be60ca8ad55a491b52f2907019bb70a --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_33-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:674cf41d34a894d27b0d8143e93a315d61a300e78f3d95c72088adc62584c2ec +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_34-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_34-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d3e383129de9c647e3bb51e18e2f549ebf46f2b --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_34-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4be1a460a78dbc5e80c7093c4f76b5d8c5c3c3ef7af76b07b2157c5dee2b39ae +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_34-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_34-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..923f769a614048473780892a426ac441a91d97de --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_34-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:595a49fce4bf40d6f42e56c4638f8546f29b1536af1538f0586bc71f58058f85 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_35-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_35-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f9dbff13ddab09488c084c2dbb4c94b4a2af2bd3 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_35-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acad4c6d17d09d1de83609eaf2cb5db83f5c8db0504fa5de1857d175fffa39e0 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_35-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_35-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..34245cca2be6cb6b8898894c8ff6afec40b583e1 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_35-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:587cf30bdcb4e24696db94a27ccefef3549217701073d2e238a0fa42442b115b +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_36-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_36-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..24d2bcd0235ade983e9387a78a9fe6e738689f1e --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_36-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:409a72d427666b3e6d57c15daa98d25c6af79d8de43c99284cc011b4ef0e9575 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_36-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_36-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..025b7f4ee57b9ec5876a2065293d082ece2267ed --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_36-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19fc773a9d54a1e171c3d37eef8894376f77d4ce64c0b6fb926a31d629dd7cdd +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_37-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_37-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..af414a2adcb33934212f994e9d023ef84ed70135 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_37-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52d7b705e92dc8be88fe72e1da510e586d21d02c8a932d6cc3be967eafece6bf +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_37-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_37-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..90bb347dbc35ef1f48aaf3d6d038875a85f2ce38 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_37-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:736bfc2aea399e3661befbd11eff417be556b20e10c7ca3b4460b48e87f03667 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_38-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_38-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..22c6981fe019b28d8cdcd02b69a246a030159b98 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_38-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d63898420f24e8fdc80bbaca2570745763efd46ed4ecf8ec340a08cd56c88738 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_38-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_38-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f06bc611d78733eccd026b9aadcd733c77412353 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_38-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd6e3150d3ba8dd15eaf8e1d3f6dea2ba6fa159a607d9f41eccb895a860af818 +size 95216131 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_40-model_00-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_40-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ae4eef57dc684faa7d099d337db0c550f376d94 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_40-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa031df6e8deb89a95fd4974812c4040e69d79251b9b82a969b779fba69b1fd0 +size 12483 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_40-model_01-model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_40-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d9f580111b645248f55779e2e8a9c57da48aa241 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/layer_40-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5f3a35e70b225d5b63b14eb544da35e0a9115295f8c919a771ce5fe8434f199 +size 12483 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/mp_rank_00_model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..93b053e6f2df6aaafdcf558777be10b9daedbbdd --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:932f966eb303230c668bb7b136ba70dfcd52243c75e24f1d844d0bb64380c9eb +size 51507 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/mp_rank_01_model_states.pt b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/mp_rank_01_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bbe00c18b91bc2b035984dea116bee78e9ae8967 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/global_step90000/mp_rank_01_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb157f878d079cfdb7170000eab96044f9466d7e2e026d4e400dd6051c9b1bbd +size 51507 diff --git a/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/zero_to_fp32.py b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/zero_to_fp32.py new file mode 100755 index 0000000000000000000000000000000000000000..e5249853c89179a3d5ad212ead8c464d6c979df5 --- /dev/null +++ b/4b284b12bc4opt2/checkpoints_4b284b12bc4opt2/zero_to_fp32.py @@ -0,0 +1,482 @@ +#!/usr/bin/env python + +# This script extracts fp32 consolidated weights from a zero 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: python zero_to_fp32.py . pytorch_model.bin + +import argparse +import torch +import glob +import math +import os +import re +from collections import OrderedDict + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, + OPTIMIZER_STATE_DICT, + SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, + ZERO_STAGE, + PARTITION_COUNT, + PARAM_SHAPES, + BUFFER_NAMES) + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage == 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_optim_files(checkpoint_dir): + # XXX: need to test that this simple glob rule works for multi-node setup too + optim_files = sorted(glob.glob(os.path.join(checkpoint_dir, + "*_optim_states.pt")), + key=natural_keys) + + if len(optim_files) == 0: + raise FileNotFoundError( + f"can't find '*_optim_states.pt' files in directory '{checkpoint_dir}'") + + return optim_files + + +def parse_model_state(file): + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = { + k: v.float() + for k, + v in state_dict["module"].items() if k in buffer_names + } + param_shapes = state_dict[PARAM_SHAPES] + + ds_version = state_dict.get(DS_VERSION, None) + + return buffers, param_shapes, ds_version + + +def parse_optim_states(files, ds_checkpoint_dir): + + total_files = len(files) + state_dicts = [] + for f in files: + state_dicts.append(torch.load(f, map_location=device)) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage == 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage == 2: + fp32_flat_groups = [ + state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] + for i in range(len(state_dicts)) + ] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], + 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print( + f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_file = get_model_state_file(ds_checkpoint_dir, zero_stage) + buffers, param_shapes, ds_version = parse_model_state(model_file) + print(f'Parsing checkpoint created by deepspeed=={ds_version}') + + if zero_stage == 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, + param_shapes, + fp32_flat_groups, + buffers) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, + param_shapes, + fp32_flat_groups, + buffers) + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, + param_shapes, + fp32_flat_groups, + buffers): + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print( + f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum([ + full_single_fp32_vector.numel() + for full_single_fp32_vector in merged_single_partition_of_fp32_groups + ]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum( + [sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + state_dict = OrderedDict() + + # buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print( + f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} " + ) + state_dict[name] = full_single_fp32_vector.narrow( + 0, + offset, + unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError( + f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print( + f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements" + ) + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, + param_shapes, + fp32_flat_groups, + buffers): + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + avail_numel = fp32_flat_groups[0].numel() * world_size + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + state_dict = OrderedDict() + + # buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in param_shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"{total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, + offset, + partitioned_numel) + for i in range(world_size)), + 0).narrow(0, + 0, + unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError( + f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print( + f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements" + ) + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + """ + + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + print(f"Saving fp32 state dict to {output_file}") + torch.save(state_dict, output_file) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument( + "checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument( + "output_file", + type=str, + help= + "path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)" + ) + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, args.output_file) diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..08e956628cfd3cea871e852e35041fb6f18a6f24 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c35e3163e0fc59810fa5c085cac8b19914eb7cd4490f220711aef2a9f93dc789 +size 167872663 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..83de7caaacf055e148f776a37d2290a569f17465 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a55897cfaaaf8dd78722f81915ccea892f43539207b0f2a619950494e0146fb +size 167872663 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..017fbce7ef2c93bbb934a76d9ce7a7579dca96d5 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5694bc29212263d509bc01553c5d88ebf4bb94f3fa6e0c10904984f912e0c807 +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f7c911941ad9d3232df8e4139340ee756703f3a --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cec7ec7b688f265c53ab7fc74c8bfb821cda0111fb32550279f977b79c8b8ae6 +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9be0a3379502c5d754fc2b5899e489c09a939323 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c979209d37c32612124266994f8a29967b9e1106a12072e0659eb1f5e293ce50 +size 167872813 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..60141ec6b24570a55f96988652ddf89f512c7c00 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4dc2cd875ddd4cf9b4ee37284fcb22c0d54906a35c127875c2d525fc0e87f7f +size 167872813 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b8dfe90e4b53d1a8600d28a9cde7426afea66b85 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1eed0d9b9bee79a95317f6e11bcf51d85e7ddec61a569738a2eac7b5aca773dc +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a1e5c7f0a8e631865c6f18e76ee010ba586ecfb --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f7b954bafa1b2b87f9739f95dfb73098fae1f5f9e762b7ef4a11106995f1e62 +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1b87aea73993e45606c5fa50d32a7c443bcaf165 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ff8bef63f660dc52b7db0e858711a2237b87f50078e43c03027fa57f371e53f +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e4e500c1fd9bd9443bd96dd5a11f88a84bb67852 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51599de736ed33ab7fae81b2ccd3025c423f88d2fd66a4b9d754f4118b442d0f +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e01fe30d2a523df4ad0993ed4e66abbc28adec0a --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65e03a034d385ed463c29d3ba9f25218d7875be31124ef8eb451d572498c8a32 +size 167872813 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2743d926f4dae3ef9c35eedc89713bf1fa518c8d --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe8323dc15819e3c6440aa2a18295b0da7f03e0382bb3cbe8262cde8d587a22a +size 167872813 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..70446986ebe7ac97ad5219f3d07c14950e3ae6f6 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:150c5fb40ba84b2520cb7bca6c52cf440f2ccb5e02bfa109337c7a4c613f7729 +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..722f961cf063472dac207051e1d70a88ad6d286a --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb4c839eed7eaa40a1ded3219a6e91dfd9766765f88864396f9229c1a151f66a +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..410823534f61c415a2b57a1444326ddcf644f206 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3200b3f388495a7455d4c087b8fcf7c34fd3cb5c03bcd78e0d8debcf4ac3c69 +size 167872685 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..70ae1c0fb056d905489fadb36fa344409aca3c54 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfec68b6b36c9378a987b0d8204eff42ca5587ce2cb95efe0793cbc333556b02 +size 167872685 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a3db06c8e2f5c59e66718eeadc7210a328c1f785 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5dfd827c18a440011ebc1b3651f7c8a211bbff65b8db79fe17b0c3fd9719f0da +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..52f65e54a5b1fd52f8a3531e414f7fe109a24888 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca2377090f9dba8b17f79abe6ac1488a02fb924d64548ea1ce1bf80200ea3656 +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..73dc646064b2a8871db632e1d3d9c25f63a95f63 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e7d9a9e9e016a44b5f8295c1c56f630ff5bade692c7ee78090a1a830a1134a9 +size 167872813 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..61d5208169a665ee5900f6fbe08dcd702797858c --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:020201d42285c985044edc80584babcf503fe489380766c1c5aabcfe2ae3fd4d +size 167872813 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1338e46a5cdbd23e1183e545172c2c53f22d0cae --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:723b3b1f3010c05954fae7e6de639b1607d450b5b378473f711ede43463dc5d0 +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7fd21c38409a0dc3fb5806797701d435cb9aa9bd --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:317381341a3729c4ac7c9a932a15c65808f04c9a8c285d0044b0449ca6f5aad6 +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..05553e175eb8fd9e523e6d4b74b75a9f23c1470b --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc2e0236168474d433feb18a59e666f95fcc2330d2af7ca0b184f08906f1189f +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aeae00eefe68d9189dd3a3e968ff64bd53c5ad42 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d09a040bd2dddf120b1a2488e806b56b1204498b284a18c7943afe12f749b72 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6c08119bdd0ce2697964b9ac5d18def767b91903 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dce25fdf9a363e9b34400d7e7b10594d2ecdcd29e0ca8419c176966a2fbc4df +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..57f1764cbc57fce0e8815ee4df8475eca93b701a --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0487d08b40893d417ecff6efb0bbdfe2ec849b3cfe66b9392557c92fbf1c951f +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..892d75a048cef94d69d2fd029036b69e065c9db6 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ad7184bd34f95a5c26a17cef244d4a68262bddf517e846068dc90ed0684afcd +size 167872813 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b8e83df9b68a2e3410755c22ee4e9ef48569883 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5d78f446177c64e1e4e711fa78a7304fad862cabe0a696762c4db9e978063c8 +size 167872813 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..792210070db2a5b3ec7bc49ecea9aaa60c0c3a1a --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:475bae0b32947338967589750d77844f95ed7995294867094c0eb0bbf43dee0c +size 167872621 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..08aa89a943d62b920a8d28667cd14188ff219195 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2a4226ad92b1e5f05b280d0f2efa3f4580e3b3f3ca209445863e34f27fdcc71 +size 167872621 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..642ff720af58d0a07ac9260176e74aecc0595eda --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e8c0535966ded2c787441a7ca4f3d338785e6b16650436fbd929f62df73fdc6 +size 167872685 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ccc788f0e5f9f97b1b1dc6be6e604bed1b26d11 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7fd5d5e5e6b5443374a65a0973e6137eba9011f51b995624a56e907e41101f1 +size 167872685 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..89434bfc3725e8d8733668a125b3c51103b29c99 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e76e5236937b76bc07b9e8e8639990a0bc97d489201c9ad986583e240081822e +size 167872813 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4df416880abb542511dc88d866b4d7875862c0bd --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdc16d724981a8afc32f1fbc9b8eef134c7803844b197254deb19ccfffeb1f08 +size 167872813 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5669aacc3977764c2eb7468f0d8719c61a0c2f41 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d88516b044964f2d713c000ff6723c280f699d9330920cba4cd354fedf34059 +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d1a0163e88c6356d93cfe38db5c73510014f725f --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97c05f161dd68f607797efe54a6bb04e7da5bcecd55bf6e785c747038313399e +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b818b444326bd2cec1ad9a92d8b23ec7b984dbcb --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d2e898916f55ea0ed3c4f084a4cd5a29c7dd1b15b6082c5096e34b75a7ed9c8 +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1bd6cb704bbf51af0ea007a1dbc37aa82ce99024 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:038bcc0c5d112a8df0c2037274d6e2b86e79d061979e9c159f5152335d6e7a45 +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..943f370b289c1fb0ccb6ec0b6b1b12fcd3baa89f --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c4a462cd424eaf22bf7b189d20bd4a6610bd7454acbc427f8e2db46cf3f921b +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e91268a81420f053135b53724b8a61da9fac10bf --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:116c744359b75873361e03838f67d0c56ed48d2e210ddffba36fbd1fd5501157 +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb107b59ac1c52a21eaa25623b3f0ca5557250d1 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3495c2fe81c2913efb6d1eceb059b5d16445c02aacf3f18a342e4212b38a41e +size 167872813 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5a55b09120b4f6dbdb1ca35d2798f6c4e3d8ba9e --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6634e8c67095c462e487689108e5a231a36b1651f5c093c634d7ab81fc52555b +size 167872813 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..09214e4e936cf5d62c493c2a816f3e7709d93d7f --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e5401ebee548576a5df2f191e86407f4407693e9d286bd5777d9bf4dba1702b +size 167872685 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c512b130cd1b5515dd1de832c78e5c1a28a51212 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e590565545a3b26d6948a7311a3e6939bc7ba58dea839017425bc375322a88a4 +size 167872685 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0bac06b6c8c3ae08321d3ed71354a5316a88d262 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aadb7121aa21b58755f46da7215c11aacc8699afb3136ea2c434794c84391910 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb7e630ac02dc5e1e26655b8011779409b4f1aac --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18745a5031bdc65344390810c22edf99f1a38e507b414315e4909961eced7611 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..52a29397123bfa7296d21fbb8fc076769198b550 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dc4c7a337cb5726334bea1cbbf2744f29433e8cf043af3589013a2e51249f29 +size 167872685 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..22b2d5a2f51063d3f5fbbcc3bea3599238f6e01f --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2d0c6679faa6f7ab012514c46b55313f64908dc80a0089aba72f97bf8b11868 +size 167872685 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d8a14c158929c5e89345c26fc6dd23e0ca6f19bc --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f798797d99202d27b73caa6891ed2c3943635e289b11eacee4b93aeadde43a64 +size 167872813 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..16e3773e235602568d99ea0eb5a9a19be86732f5 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb1cbdc52ee18fef7225edaa909f97c2f723cdd0b6b8cc84138c0271acce20e6 +size 167872813 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..083d5d90dab38568bb60392aef1f447e1042d956 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:516d83e5646b1bdb8b565f02e34fdc9b4d883fe431ff9d1d9ec337eecd7df2da +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3333fbbc5cc3c1b9d97e0af0a83196aa04b909c --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe5d17cd340e39cdb467bd4b748100691964900b7b15359317086dfb351a852e +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..510a02fc279f8619c688dc016781b3fd7ade9139 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6635f66f75a8e8321056ee100ef4e1a35779a100d16f9069956f0f58787cb330 +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0181373487f8ca274cfff2bb2b24e7b99dc6f7b3 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb82c4c5c862e8eef668044e4117a5fceb401789904980af6210a9cd6ca09945 +size 167872749 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..32364fc9fa8f708e8f38220fdef9184568ee1269 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ece2618c4e7bb45010e49c37e7df42896ceb98cf6f7b31347161b0d6e518e3b3 +size 167872685 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b58e3166d1de2b66745ac8ab5baf86f1958cab0 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8816669f3ab444c7cfbe18ea4ddd279aa894f20eda46f92febe993d691792658 +size 167872685 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0f6d6c9b8a1dd97769004630706def63ca746ecc --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dd6076ebee021a5a92bedb2d4c1978756cd0099074325913cacbbea79ac9a13 +size 167872941 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c6e66339deceeaa7037d42ccb63da349f50fd2f --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc20eb2f60ea2393946a38d041520670a8b26fc21b0c669658a2ca39cf7b6291 +size 167872941 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3033242d9c19a4d02d1cddcba283b28b947327c --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c52659c34df92c5cdbc7702c209fd8ee6271654242918484fc295fab9b8a8e3 +size 167872621 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e208abf149c3979d0a04c66ab4fa357d3435aa06 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfad30256676354276c0f8c3125a80a9418749d5865882e41a217b82e0055f1c +size 167872621 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..206da1573ba1240bc94d2d093a7da7cdb5017ad0 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f583ef1cb2bf5bd6968512ed8eb0d69f32ff8a41fbf06394b1f0ff159fac8bf0 +size 167872621 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b177f475820dc7ff1915ae51efb2f1c6e0a51dae --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb79d8ee79ac2789a3b398432bf3a8dc3fae0465760108128326c1c41ad89c3e +size 167872621 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4384136992de28539554ffa09e97c9d39eb96fb8 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c64833470e0b6cc980619f8b1bdc9ff5a2ea4a7ba353ddd5cc5c014b04acfe38 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..38b68d0f80c2f8310511d6ff3707fc0f6a6c7dc9 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2e6c1460e1c94d3ee30afb132c0f62be2d60796f9e02f321da59aab6ce32b6c +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2adb6877967e6fbbbdf19e4a6b93a7904917e3e8 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66883c0ee900cb78050175541f232d2af8013a33b68acbbe2e9fad629f5a2004 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6d125b2ec8869872eb700faf54c9ee23640b7639 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9674a49168eb7224024a1ffee177006051f030bc662620c7eb128bdfb37255b0 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1d0e219125bd3614e41ad1474272dbd2ead7607f --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afdaa822db8289feece04a7ea301c0e8ba8213bed97ed57241c7557f397a90d7 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..11300e2773135b836770df75b46d9a6961ccdc61 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8836551f0aa875a0b3b6fc885d97ec41f814552fd7931afb6b5c1a152a17124b +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bda36565c30da67a2f36d48bf262a245858acc82 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:323930a4a14f4f1dd01c8133a5f9fd3e18969b8d3ceb5f25398c9d7f3c77fda4 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b83c919ae53800b6c37cc891f89876ee1b1d0b6a --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:985fd4dbc52035cfc33fa750879b6658c0bceb4bd9f2130afdcf32bea91eae20 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..362378aa8e9c5d7bfebe050df3e2563578800bfa --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7a0c548cff2faf187a7337814732eaa7c96c801b3b018c828e0bd9d2e311c66 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1de4633407cb2590b5536a7a0d5d00847405a389 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:411e4fd7096ddb85addb48e4cda344c2542e195d587dc008beff2206b795783f +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1ffefea301881c69431db5fa44005f2e978c258 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:245176aae4b37c332eaf525eab5956e1c4ced46ebccb312befdae841dc73bf14 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..80dda6f16658586e67775e15bda905f4ec9a72f0 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3024d151b7ac5d8a924ae168d028764c110429124ed3fc482c9626b6baa4ad1 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e073c328eaaacf12db07497d05795ec6f226061f --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45e914d1892cde7eaef6d4c2da102e6ecac4e9d0b1ee692ff7d9495156d6aa49 +size 167872866 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..814f1a1db960e32971c677c8d1ff73b4ef6c6d13 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41c6fb7708f2eb7c7dd432c6de1fcd8007356414303b9e3a7d593ced96db469e +size 167872866 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a25af6fd99bc649e4a4a7d3c6cab810360ef1df --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4ca1849bf2011fcd3fa6cf2eac8a91b18a2d04bd424fae2f771124437435018 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c602757d8fcdf942ca16f3dfb6098f6b5c8a850b --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:14cdd06157117abc87d37120b1d72bf9c3a6897c091517825828402299794603 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7a38a52f24e098d0cb4642b0d921825f72f7bf3b --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f81dbc4412ca4dcfe118ba4663d388869314dfda24330019b4c6860809c118f +size 167872663 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8050e5c1ab98d6e2a81ba0313da502fb69b1644f --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17e0416b6ad57e4cbf37a5abaa61fbd2a1aa027a69362687d18122f1b6feb5ca +size 167872663 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f81b919cdc8c2d736bbbcd866a2f7f41b55d6e0 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6ea3a6df38cdda052797be28c999e3bb306baac498ea9cccb4aeaebb54953c2 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..52b2c3f57e829718860b59c0ee5f09a46f829940 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0df0b76f5b36aea5e2173536d6f40fbc7bf97d683712f11cbd633cd265952b44 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d984e6401b834631dd3fc248af5f14adc8f73a20 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9c7de9fd8814a2c6c95563857da7f26d3db43ca79db17bac110276a6a7a22ec +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..be3200659c19d822e2822f90bbb5f9a6c2265370 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:468faa9a173f91047d0c8e5ce4eab53d788316c95ce47f6ebdd594aa4c88e411 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d5b622783f3ab6442d9eba5139848db72b5cef8 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51123d67f69c5e24133d4569fe7e62b7802b8c06540f46122374b0b0f5ea185b +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..85149a7bbfba6831b3029d35b9b9ebb3a4f91e76 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00c33a88ca26c5159a1ad9c9f2e431661e99af64fe097104a0296e448c095a86 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f73e82cfade73ac886b1c034322c7c079d689f54 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d4e34a528128c9b052ac7ac91991f998917fb408bdd265766413662886bd36b +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..44f17fd711a6e9ce6940f408dc3690a261b7ffaa --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c47f90518f66dd84325cef4a18b9870a75ecdc4a3fd1813a3ce460fdfada6ea +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4e1aaa9f364cdf84eddda80c296ed8a86cb950ae --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:878032553ec3327db3785f32f42facef993de9bd31cf74774633bb5565e63c0c +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fed7c6651d25bfe447f1ab5650df87865f3683df --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6df71e8e9a46771f042429b485e9232bc31da650fa547fcbddc2ec9ef4308dc +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a2c8f4c3ccea55677c7063fc13c105448ac3058 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:affe02b15f7a08f5cffa145ac97de515e3fdc3d3c94f730b32adbf2b89b90c39 +size 167872866 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1f94f9de85887a66b37a66f2f46bde4ae609774e --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97acfe6013b0d33d17486fbfa1803fd4d41a3cba62f53d3b221a88cce32d386f +size 167872866 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f3750818a79f87cfdc66ad8f8d10554bc1d059a --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5856fca75bb5693d42b9025779217caac3c14a4416a0accf2fa6c7c84b6a397c +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b89d2c1f1df8343f54f6bc1b228ed97a325789f --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:088d1cdda845779cd736614c4e092c31fef7043d7d03e63373cd8307f02f3bd1 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..484c85e096bd4e2faf8acd9e97f330418c3d88b5 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5d89c50a7ebc0767c015562d8478437e81fd31d5260ca0aa68f7024139e381d +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a0cf93d6804460a6260108bcc5f6e7922e83cc21 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:26cecbc9101d1fedad70d3a71659944f0f7c0bcc301f1e7470765f8d27aeccdc +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..432a49afaaa7d1b77fc83775f7dae38da022c08f --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a0a4f1b027568d42cf54f330740a72649e63fffde0d98cede488c0c765e37ff +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8737a514a40f0e08a31d9ee638ea674e6591815b --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:770db7ebc33a3292bf699eb6a946376ba4d2af5125cc4c9c8619b02415fb425a +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cee6eb784d61b5c4051516062571ee5502b2598b --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:438c1dfbad9f411291b461dd630b5e627a4f6e605f4326c9c632d54456f05113 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fd27b7cd46eaa3b81bc6ea73b35ee87489ca5331 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:696d68ec3c5d13376bccb01145ffe049cfc60abab0da9542c9512da724a5ff2f +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a8ac5a48fcb5aa9c47972708bdeff0f4f73a4615 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66f7f1f840c55d257e9b7f8a8f3806bd7acc9805c050e33fa96e93c5f9fdb8a8 +size 167872727 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0898b94af435f2e193abd2ffaa45b88ed9b7685e --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d25175598e09b6acc0e34d348eb40d78a0f0b5ca4b50a03bf7808de873593d5 +size 167872727 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ea4af478a914d6bc9d0ec44d7ee712d45561398 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43cf4a1587de34d10116fca5e1e07bfdb688287500deaa8ada15de3c3c941dd4 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fc8d8688d2a9dd76495fbe6591fdde2712a420e0 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f9cc2014fb7822b8d7209846c772272f6aeafc97deee88639c0eaef4fe584b9 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cbd395c7e71d3e8c2690088be760b4777e3000a0 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8d170fba8f8f92fea1110f4916ee70edc9294aea547199f2b3832eaad991647 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bbe1179d98c3ad2149428a8f4aaed8e06fb64108 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7894701454e24b1fefd08244af0ab2a37cee6a62a4ef5a71a15276533f304280 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c0cdf147a2c44e4cf90618b610e3ea6b7b00a9ca --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:634cee112ccf0c041e1dfaadb66c5ccf6f85da6e3c141ba0156e2b6c9088a72a +size 167872866 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c628b2b84368ac00d47285dcabfb569c2d47d2a2 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7a23cd49001168f2fb7389052852a2c3a31d5af881fc84befc453c9f0cd88b7 +size 167872866 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..430a7d85f9aec9ee8890701d798bdab63282596f --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93bcbdb8105cc6ac5c5a8cf427af66ede5229b78dc57fd225a73a390ce7506ef +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8778647e344a9f8172e3bc0d7cf7a9964b3fc6f8 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00261bb44883ba05a03e2fc42ab0d051c8139333c0460198b826af4ede37c328 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b9657817300bdf0de1760a9c2f71089818aa8d8f --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9221ced584abcd868e3372a42a79534c1024c02315b7e5d18ed8916854fbe64b +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b7dae285c9a18f4cdda55058bec94bb2d377cbbc --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bc19d4fd243007c78dfcd11302ec12d467dc88d4e459029b7f2f90ebeea4b26 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a337151fcf836342b334cb11d2bc09d3ee991044 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d014af01ba0683d740533fd7c76120bb81f8ad5cbfb2f2ee9ef9c2997cd2856b +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6da1739d45b517ebccade7e45d1fcad7d124fbd --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e410592e73c43dd616d086356f8921439e1a40595ef10c32de77181b83aa2239 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ee056bb837ff14daf3e7eba782ac75971711e0b --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fe62767a2e9887f1bfa7f916061dafbc7c80a985c6dae4347057d758b2ed54e +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a30a0624e07d385c62302de54372c4dd5b211931 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d9269c4a4f71fbb1b020076fbc679261b94090f84ce284244aa91bde100a813f +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3dfcf504d2638808d667e79e3db11072da1ae7d5 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e90a7ae3253082230566e6102d187826344c5663810d488fa7a47220bd5a838 +size 167872866 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4fafc5a290a85a6e34e977bbbb2f5a6484fa0756 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ecbc4d3d4012bbb6132a51d84efbdb7dc2f185f853938c615fdc2d99778b76b +size 167872866 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bd3bac1811a20a729e2ff5eae70531034357e501 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69e5f35387bcd443c820b7831a605de4b561878f72d2496adc45eb930cbc2f51 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a11e521893a158c456f89383e0c4945d9efa05a6 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebd88e946d3f34933a604dc264c42a0d16c289329debc140cc5c64c3d44c16ad +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..feccca2ca94cb09c5e02c1074f7f38e12cbf24b1 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d88aaf2a052d71bea3539692ee62f2daa6e9e49df07bb4785e31036c65124ec5 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cbe42747b9535dc50b3a63b2fdc2b6753dbf6802 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee54ac3d0350fda77b1ff1413b3364c9451fe2d0eb3b8a67dcddbb535d8f4652 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d6100c75fc165c5afdf75ffdb21c0952370ef830 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1480c70b80dcec6f7fcb3f8c9d2b3217623412dd6592ffbb9d9a526f46c73a9 +size 167872663 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f693253634f34b837c05469d1c59fbd1d0f0c9c --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9dc5b03b01e4b18981fe22202a0e264d8ddd8c74fb79311f72221a462e2ec39 +size 167872663 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9727ac873097c274f33111e02cc642b4f2e88c13 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:772375c44abc15ce8a7430f36f286c270b6e26de1762c74db7a85495aec4cd9a +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b59dd7cf83fe04538feb9b68bede799773544b2f --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:765a4d5937bd8d3290a563f1f756bb864c566dfb48eb685c3a161ecfa2783567 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5abf7eed9f9e0da2741390fd9cb7a984e9718d5 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47fcb6c9d186ce027e6564cf6143d05472dfc28824beac415729374cd079bba9 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3630ab1c01b0dec19cae590c6febf0db37bbcb1a --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:790057f9060ba31ce07f54e5b5e3a63c83a9b626edd5cf2334d8369c04b5453e +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..25415e248dc5885ed85d69924cc015946469e125 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8e688c380ea2479fb46083dbeb8ed1425556f120ac3063c93028ddff64b48c7 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1ddbc6f6aa6993dd3f8f824d8aad5cdb6726792f --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:388d83c7566595bb918e1e51d62ceafa85d19c26a71a017bb97d0e9dba6ab8eb +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e22418c57708734d05c6afa45e63c01e8f959e2 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba985f329f0a29b4bcde201239f7905ac14d31e6c0a3199688125821e151dc9f +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d7f4b5b72e5a9c9b8121cfde312c61337420e151 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:474577bb8ac0ac76fd5dde230f0dac8580896d0033bcab514a93746b9672e765 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c4d314c20c2b7d66618cdb73595cd48d1b89f02 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb4d52d54e276a4a567ce29bd83eafa0d5248edba756c6b2261c4188e878ce7a +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d8bdd1c66a7557908549628f10b67100eeb5199 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:258aa612fae4946f2889a43e08acf01c2b65b1d7c87de33efe113ab7efe601c9 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ffae9eaba4efa2f5bf30a8c7108b5d745b57c0c --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a744dbdc122263bf73a14bc6caa41fe086667a9f3617d3b50c3e51645a3a4474 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..998c5abd1a0e9f9042771fe578c83656a6c0b5c5 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96869241a48617052c8855b3c8ee8fea722dd43774ab1ad6555ca81a51299734 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ae2b4a1f6493140428e6d2b8f5c328470d1852a0 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69b56341dd60f931837037efcd1621966578a1ba04e0e8b1898dcccb50aa11b6 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7d3269444e4ac019621f78372492e4b8f9c58857 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7cfe8c5943a3ee8cb35ab94c1c492053a39f5df2875f0bc25574c3b3725933eb +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..abd11f490b7e8137be7e4bfc899d8b8ded505175 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd4246db938dffcafd62a382d24dc30a7b02c93d7359a4ca49d7ef2eb89742ac +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0be6b47369336677e80692cfc71beac96e05de51 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6cd09865d747a86b0c3c7b895ad3d7d02c241eeaeff02c09ffa5a6ac0919881 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d58897659aae6bb5cea6392f110cb033cef0f262 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9673171b41a747782fce26020c8a10e7b1877c655d6c771c25e3008d85159740 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff343da639e3480f849b0635d333b360ca8f1218 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:949c9f5eb8b49d77842bc2e5bb0fdc57e8d50345a12d9d651a81b34d7acc5360 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f6d255e5839992c5967144e7d4bca13a5644a81 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9eada9e5c070dc8aab6e7766d91e46bc44ac506b3e18529452b32a6f33a94ec +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d2cb3eb56087ee2b862a0c73cbd579a454383e30 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9da2bfcb0838073d0ae40dab54605068cbd75f4f4c258ed1812e0bc97db72d17 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..343a6152369b9af55bfeeb85c51cafb60226d09d --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95d1ac5ca74bcd93f482599b76aeda2c48c53e15b26569631001f0b527937abb +size 167872791 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2137c3aa463787543563bdb0b2f22f1eeea9d8d7 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e90965a2d85879afe8f79bd4308884ab02364cd8eed37d4a2c6f4d000fa7133 +size 167872791 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dad45aa8765a8a696cb0f2e2052c64eaef4a9024 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdb050eeeeccd2e61a4e415e010a14eee926b6bdb33242d992019144bd8302b1 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..29eb1c622e38b92d6cd8f4a65b02c78d6ad90f5f --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bf94038a8b714f6c9955d144af78c7d77b0a69770b35532e2c99eb4a2cf24fa +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..38d05a3ea4edd0bbcae89f95a154000988cef539 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cf300dd5a1f566ac3c818166cec6d76f215f772953fc043f350b27bd6371c5e +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c800d9c027966b9f9d707eb044955231f5948a27 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34ffb61cf8112b082370d66a7b6268138138f45994126510881e89acf92b59f5 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bb7ef27df79e0fd55634050767f0897e82cc2539 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe40df3bfec1b095f1d2b69e25b1207177931ae6a263f5239421fcaca53f37ce +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..05a850491b12a29ac232ba96443f1476f09dcae6 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e078d3035917dccdc6a22c80327931d17bc86e1ff88a9891057d3f89eaf78b8 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb85be720f0a9befa6978475012604f71b321ae9 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bad7b295e16e58d80cef71595cb24444812e877c7267393d2ae0f87b72c9b4a0 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a762f5284e9a14581058fe50679bd623f646e186 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fabb313e4adc45fe459105f66d8bfd64a5b0ce8b6f8c22d0431a89a82d89a308 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d54eda32ce06fad6f63644e9fdd9cbc87a31baa4 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1a5fa48fa0e5fe518a81aafa31ccd201cee4c6ef3de91a629c648a2ceb8c680 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a1df9c62bd196daff5f89313eef8ccf65b96c0dc --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:080f5715920e18428fc6d1cbc3453d3426a8a91b8074d5bd5aee25ab60db9cc2 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..11dd38721979846eba1dd96cf639c0b9882e74ad --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6229f9594499eebb20955f6eff26fa5b151c3507b5fbd5a19d9a2444194c8cc +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..44c0167ccc268d8655f559b9c68f8096e4d3ecb7 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea8db7dcdf812dd54b0a74eca7900fb49137a9e1e5b8be2eb5e660591d62408a +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..056aa2224e26cad9356649d80778dad0645ff1f7 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13f1d7b3a22398776a15a196c4b32dff3340b96405a88e7d3f2848afa782cc44 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bfc5642e8f3e13eb7e8febee40217a1660d502a6 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7021cbd9c5b944ad114f328c248ee43698e1d8a475b2a6852bab1229642de28 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0c29bb406b0ecaaf26a10e8356166b67130ae409 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7503aeea987b6dac72d79310bd9009e48a7ebc1a6eab26ef0d167cfd0206f46 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..930ff3bc47b952c5f87ad902b45266073057b58b --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5394dbdd73ef1837ab55e4a3acfa2a4d95581076e1a48e797bb7ce3c801d6e84 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..94415e9bf629962e73b1532080137f8b43844d7b --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5243fcb262a9028a0ca7386c7a0e35b8f1d5774456a6d61d6c70310b3a3c72a2 +size 167872866 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ed3cc415fc5824ce714abb92353fae8c95d00039 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f322a7bc2ebbea36c350af7fe3bf4bf545ad4c9dac86b9abf9539868c1e6ef64 +size 167872866 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d077df882fab5e8507855881cb3d27dee3cfc9b8 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed78c4a7fd92d034935e38c92d2a600092d839bc5dccc1ef9a09b46b2d84cc68 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c80a9d4459ababcc744d6c8514a710250acc5668 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cf3a52413d240a0749aeffa5b1ae7675ec1631400255fbf138fae8462c3f748 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..94657ad8b2485999c5e3b4d973c1ad04b666e12f --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:343db23947d1569506e761afeb484ad7ab0bd596ce22211d6788c20ad45eef2a +size 167872727 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d10509515af85e0f8fdef76462579d28269ddb50 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5703817c1b2ada68a6d86770b016c6307d6aa14d20420fe51d4f9f0dfb4b3279 +size 167872727 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..09bf0bfaf7a571581a5a337a47f535d9770e751d --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a3cab92d30c29ce6f0d179e8adfcbc5def4a02547dc9c6a7c5c4ea2ef6faff8 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b84cbc34985e2445f3269c5888f14edb1794ef80 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc269e46c4a1fe15062483b292f7ec32f8d7342745c16be18c7407afa5223186 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c4631bded099ebfacdcb6b095da7f5ea4758265 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b9a93509d25a980021731fbf4b3b78de9cf4fbdc3af4297e464a84309654608 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..503a1dabc80bb4dc0e979fd9394ccb813aa457e1 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0bbe3e3c6ba9411b639262943546cb2aabcece6691c5d3c922b1b59f0a19e88 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..78e2ca3f8237a563d9f4e2329e746b6fc82ea165 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0976334a42057bbacb7667e5f643d66daf088294b4c856a69bb6f1c6cc2d6e7f +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..15cf8d56092c331c9ef0e9cd2c07e3741c3b4b20 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e24c993fca168c261324fc065f282a7a3d1e0a1c6483e9807849c2c3f7412cf +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..00c400ff272441a1e93dd931f6950f9c5b8096bb --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d251d4f8ca8c358f4bb353a0d9455f0719411a3601edf4611430a7a3c8674f0 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..57252a2f36c46a3deb39901cb604aa45153381b3 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b21056d4f933ebee877d4a5e63e35c2dda256aa6dc110ba4d904453c79399f2 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e37ca0e49dbbd82171e8eb7fd88eaf0d576182ac --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:352d58be7af3ce919b35bffb45d509f4879e7a4039ed4abad107733620bda113 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b9318003c289e48390f83bc0cf92d74f0669567b --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6690da8412ac99d1332139a8a3a7f94c22d2926c8532ce4dbcecf3fbd6c1daad +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..be88222c8565d8047d749550bdd8772fa4b82ea8 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23cd982aa973b627bacca36cd6259e9d56ff7fd6307c670d1c24e83cb8c2cfe1 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7479f4765f853fce261e9a26bd28767e5b221506 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7905d338ff5b800eac9a82b97525a5237573673845aee420a8aa6dd815bbb4da +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0f2fffbeab51d4049a82e52017fd3942407b7538 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9e53a7001f448b5e19f7a032574cdafc6606c87de3cc28131873e3edeaa6572 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b13ed268573bc3d144a6a621a8d3659e638e80eb --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7156291c405114ed662f1cb6ddcd79f9482f1be4f64a0316587998b3efff485 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bc8262494989494aca1346964e9899e17af9f414 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f295c2dc84973af43afdb495cea645fae6051b74cee5654e88441be3b02973e +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4b1e1b49b31df9c7a5ca55e71ef1e4b45a2baf84 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d26be727492b44759b235058a2a61cf630fb6e7a2065dceb1ebaf345c4b29bdf +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..61d53359cd13ba8bc1a7eac8f4027d2e71138e46 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1b9323b2b3298f9ec9d1e81855cf2e5a77389e042f49132f3142bbe04357c8b +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b6b98c6a6c0eef8a66ac934743d160765e30727 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9975e12998a22e35c9465acd7959dfe4bf133d42a90b9032f764b8c203ad1869 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6080f8bdc9c152cf844c977a4965d0abdcd94efc --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d6b69634ebd50fdc6b2a40841a443631c2f8520786c8e146a4101cb91dadfdf +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d8ac0daf1e59f5d3704be76d05ffc4b0d113d953 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd39e0a458909805797d27250ebdb918c13895d5cdba11b9d158a58f52c1fde5 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b53ff4eadc5517c62b38773e948f4ec59a61ad23 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a65cd10a275517a6e615a130cd50b70e0833d6e9413af453826f8fb30a313362 +size 167872663 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..719aa219ad4abfb102f9e86cc989b274bfc4da68 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:034576f9c3cbb276b361ced04bbb8864b43f097e402392ea24407b45b4def04c +size 167872663 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1e7baa50604ddfe32eccf525ee9688a43f2e8774 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79bde54e3ba4e8ff6ead3178501cad1a4227596eb53796ff371de970c157e084 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..db8d43d677d7a1479a6e40d9bde1e622a0b7dafe --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6c1a52fbeb55336d75ccb6908a050249d1f7d37f300ae5caa6bf0c6802e807a +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d3b0de4962e92f40342e24297173a5789110e37 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b27d37844e6a31c774bf51ba6754b6282c1a3ffa33ac3858d903ff37bd616eb5 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8689a9cd05b0f64731664e5c38ee6175aa5bede2 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ee7bfc0e9ce110cdaf041fb20c51dbe74ccb41dc90efa62dd6fd330b61d6307 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7bb3133823ca287f3006c0149d4433787c42296a --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74813c592447f4ff78651b24eaf57c635497906121f96353927b4c1e600da3a9 +size 167872866 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..613be01e733d025cca6a4f1bf355d0fa655bacd0 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e1bf3cbb1c214f2ef25962eab44d4a424143448219bd7e6269720c9cdf8e33b +size 167872866 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8bf3cb81e478190b5e90c3bb7fb198a2d51ffef0 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75f7801915214854dba40827daa41908861618f2d655de56cdf85d96caefdea5 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec009a8bcbe25bd292a52d4ab8ac3ba0e593cc1e --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc1dce7168318d0a350e520e4e8632db40344eb86486d6be091b9ae6cf194ed8 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1028df0fc3f5d3541d4bb0ad48b86459a3a36859 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:044f66989ba6327b4d18e4bc9eb20793f1e045b55cd05140c8d28a3e5148704f +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..89c570a09cd059427797fdf8440d58f4bf2a4679 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:def6858d7ac5064a73c5e2fe70c074ec262c03cd14d4483bbb17159d52f9d80c +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..42aa0b34730d4b03602a8f120f057de395c607f0 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dabc37f9daf7373557d1718ddd66063ec0db614e1656372f70b84e0eda49ed5 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bb40138d2134831d1471b1c198ff57bae3fb51f1 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cebc90547dccfa1c864bbb3cfc2d72e7589c871cd9d65e651417bc02d1196e18 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a719402b7d17e3ce23dce116afd4a06625d985a --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfedb2d518ba446932d500a5128637cbe56cce8f844221b54e03b22784d2f73a +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d036b78ffe465596155d2490d7e06e58ed7438cd --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25ab1e22b91bbfc7d691bdc65acfde37af5e4f9a9b9640d792cf0a8eefa1a540 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..33c434f2df9343b9419321c93599c603558256b2 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8a96291f934bf2287cad33482ee70266df3eb4209db986a726875cb34eef851 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..440a6349a10f9dd357237c8e0fd7b6cb2f36efef --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7cda1791c038bb17a8359ed748548dfa38e38f1fef53597d19bbd77234fdbf1 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..92df7abd138b043deff2d99af829a4f95a3a30eb --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b107fdcde6875b433ff0a8128708230c7c56e084e9a2033fe86dff9173c8188c +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..576766ad70fbd11929530e54135bdb82f03ddb4d --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:266bf75a44c8c484fa395a8da6f51afc9215326f15b3e201553f18629f3eb940 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ae3873d284efab55923acaeb5c4d90762a55f857 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95707a200d0567144ff113179e601cf4ca8f2849710f63d145124c51a2ea64b1 +size 167872866 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c34c5ac2c250143c7af323a8f1e7b15a27cbcc24 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa50d7f0567ada34cf88e3598a70451457497f59ad83887fcd4922f3431b5105 +size 167872866 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8d8bacb5656e6a94067e1effdd3eb540023e75b5 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:222d5c41bb3d950e61d8d79d2bcf19446dc7248687216a78e14a659b0da65bf5 +size 167872663 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..32d78c7f2d35000b2ae533df7c35115a90a3df61 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e9463c744416d0d96feec7a8127bbfa3a6566bbeb8ae98ade8f098a68a930f7 +size 167872663 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2e013bffcb58a80bbe9c80eb2689659ead532743 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:843aa321923bbef055292af78bf35ab3bb7151cc702e1ea383c19acc6f4a385d +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ae3c668ed5221e779ea84901ccb919429af4c175 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cfe144079da41eba4c6d60d3b245c7600b9ff2c6d7f5ffaf3092ae7170f5f2f +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f1adecfd6bad3d3c268a04fcf2283a63a4e953a7 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19fa247899b430823e45c9e27d4da588a739dc659f04df4a94f913f33a9e075b +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e84656a54ecfa1fe6e128edfb91370ce2216a304 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc81235f295ea1a22b1619a28fbdb8db85405b908c3e6211035609ed53f00313 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..283661cb7b0ade763af7cd1b422d94cf7fcb705b --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:490ed6642f100aacf50e671ae83721e938cb9cadcde97f355adefbee7b8f635a +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..61563501ff3ce7172a420cf87ac954ac0b8ce447 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83ad7229e7d47b7f577e9490d5cd3dde0d11b6be98b05d525d4a077160839541 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9559dc5fb542a6cb2bf554b4118769070d8a3aa3 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ea0f0dee202f99b5db5b9781298854f54ae86acbeca0852cf64dc122d76fce8 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1fca0d67f113a9e0cc3e1a57aa1cd75fbc5d2953 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aee602cb36ebdd4b1dd653b010acb8a9147bc0e6556e6602f4fe6ff36b76d40d +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b24a8f3eb387174072ff0fe8f1e0815c64da16ce --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0465f5d9a4621a4bef3fe85b170ad5570bae94933389172b3ecf6851e783a193 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..20250530fd93ae297a1b126d4ecb6cfc7895ddf9 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb1cbd6b1fcb76025d3adf3707ab3df4e3c83a75ce5aff471c5ee72f4c5e42a7 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b1f1b412b9a7e4307c17d99c9a2ba48e94ca409b --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c789eed63407271258840e716a2b15b1220ab17a03bc7544ffda83b7fea71fce +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9529d5650b4a436c773052b0e477bfd2e16f9582 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffcc089a8b58ad8e52ad426c3af5bc3003aa1d3257fc4125a9c4f1afe4b63371 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..92d7e7c8ba52fcbefde8757a15187eab704a08fa --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a06299938aa1dda5046617268f48c94d6494c4b96163adf86bf82fa38e45d31c +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..99bc71950e578378eacc5086e4a34307c634c2dd --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee6a68f3bae352853305e5d8cf51449a2bbbdce7d6764df8421a55f9c41a1375 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..efe9f9dbacd60abd3858bd4392785cfaf89f6770 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7323782652f7481c809c6d71b139a6472ae4dc2160ec0da84bebcb0e1b6b00aa +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5918b9f73e4343ecdd620e5aa08260a48772d553 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b54422e458641ac33f09ac3b1558f4421c52b89057e063bf9e30c875eef5630 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0a86e4e4fff5a8b0e4cccaff062a4beaa8d09e2c --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a410a62f9d2bbedd8936a579b9aa452c190c286a1a3642e5d698b54b438a1273 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f1c79c498430dd31fddab48d9c125ca10625ffe3 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23192d980443149eac1b675acb97fbfe72747a222fd16124a31c1f14c5fadfdf +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1d075975e14d8b0484662e215cfa9d34b2067f16 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b907340e579a9d0d9a6221f77ced494839fe1e6b60848247fb4af46d33a6873c +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..848135b1208e9609ea2927bef078856a4158e080 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fe0e6a86a97f2aed24bf3214e12c0b9ddbdf27f9bbea4f5544b56f06a940093 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7d5d6b19c12d6d960b1c387a5029b86f1311d8ad --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe65c6b97ce4dcbe82b6dc360e31273455d6cebd7575e3b3df531b9e45869f4d +size 167872663 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..84b5b442d5bb2b6b2e937741e641139eaa4c3fd7 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bcb6a9bb8af02b1afb8523b43bb6b4e1b8c78b13e4238559f2771254209f1520 +size 167872663 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1fca7b772f194cc05b2925f4cfaac15b76c31817 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74ea4eed1e238056322a72402a848a789bcfd90aad0df6c0b6b9bed711715ecc +size 167872866 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6a6b56b43dc0f05053b584acf0649b5f6a70dba --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0c5cf6f3eaa8eee18e5bc18d1bc1c9528f454a7ec2be549332b0bcdad8c4896 +size 167872866 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe9fda8fc15b63c43dab394735de519fcd8e010d --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9be59d5fb0e65f945a4e5f369f0ec982108502562a374dacc466d9f5847981e +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff9e866e2967caf4e70bf2fa2b9945b3af64014e --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:521ec60dcd89d0735684262d6b2e646dd4b9bf55c7a6567f13122623cf60f310 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..80238d2ae1355c94ad16fa575e1728d5df32e2c9 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7453205bc455617006e0422881289fd9caceab6046edae21c3fd94b9dbe9d394 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a3412962415ea7f75b7eae9848f5d48272836a69 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:247ce9b8c49e4110678bc4fe21c28f2a60e7eea6f5552b7677f60434eae05117 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d8962cb12ca97ddb1b9eb28e6c91224b69266d52 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fad6f6089ff760acf0162a10553c4c7a82dc0be6d36efa8cee9da9ffa9e48e5 +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..12d1cbaca83a03d00713291fb534f7cc65ba8f09 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:498e5c7bd5b2500b9713b8c30530cb03ab7239c917dbd051c2fb33f6496b878c +size 167872802 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8206a978d4ab58f6679e27c797dadeb32163481e --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9de6803ada2fdabdba9bfe8f638d24a92d669b4965c8748c7144e87374e28db +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..967b5bfd8c106e441ce05ac1ea29cf213692bfc9 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b11426531aa0a92a9276cf3a5f76f3b5f5c6fa8bc08c1b21f8159d61b46ae83 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb31a97c8f2a2a2843c20e3e17d2c167cf211505 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90818e70de70477084a822f80f32c95190097ad96c13594140f255b8df6bd3a1 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7975460b54daa79104e5e9d5b02e9eba79b556e1 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:500d20bf7e2b8fa2de865f2da8e7ee420a1712bc28b1bcf6f36c5d146da0b0d3 +size 167872738 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f3ce91b9777b932da729187fdf06514593e218e --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5009f9896c45d092f9b890578bda916defaecef928a351318c8d1de3c5742e1b +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..76bd6a005d9d200df89ceebcf2224d0f4072ca0b --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae43a0393a9d070ee7b135e447f419c02d899d4d577fd7a9f87d0de68aa15241 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e085a036218ef27fe7e26a7297b4ef99192c9cce --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e040a1d9529be3030431d5358d0fdd9e2f4d7a81dd22d47592708c2a76dbd61 +size 167872866 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3aef98ac7eaa361d3f8c3e2145bf6906a92f6e83 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa692502313281163627250e3743ed39f60e4b18d7e33196ddcd3df5fb8caeb5 +size 167872866 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a7ae526a3e2e841df0a2e2da0bf4ebd0ad5cffc9 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2bbf4812e39f9c7d872669c8469d2bca69d62c95f901b9ad308f40bdad1e4879 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..499ea093157adc69b195f44d3de482f273db27a3 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b3669af1b44799212e2cb97ec174abdf82b1ceb71fb1edc05979eac3db19688 +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e0b72ed0c99a63a2761994273d7618abeb21d3f2 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e256248e6112898a2a01796760262afefd76fd8176b0db33b27cb5b9968e705e +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0378f9d45b6a1ceac5087044fdad290ff6095ea9 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6ddd5f919521e53cf8e9df5bce233ea8139f059ac704a984534b3b370044a4e +size 167872674 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..949dde19b36710d612e7d17d0ce432f4e51067d6 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a35567f661557c89f9aed594036cd1b40490c83b7e1af5bcfffab5a20e08115e +size 167872791 diff --git a/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9234fd5e5343313d3ada9f5b9ec4b486d0900ad9 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ad6b43066f01eaadc9ea90877783576f5188857e7c59ed5900f7779599635fd +size 167872791 diff --git a/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e11512424386a5645ecf55c0452a4afeab503393 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93c594857bf11044852d5eabbca521d303ab4f5d3a6e69d0af1ae088a0c183db +size 153552131 diff --git a/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a2b8fb784b58848dd2590f9e2829fd5aec7453ef --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_01-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb98237b864f777b869d22ea5c5846ba9fa91bb7b164521f05f2467787be17cd +size 153552131 diff --git a/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5fc991d74021668e4661c329408a2545c9651ed0 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edcc463799fac1426b238571010abf9cf217d3e24f1ab52398a204cbdae4e617 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..12cf5d2855d7554d061e3cde3a836641c0cda821 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_03-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7738ffbe26535f235258b6b4a7e13ab059555a7eb470ec66efc327ccc486a7fa +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a021e582ef5466f348c8fd5513cc8d462f999f23 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64d866d0734c9d2da7fa2e9154568b5d73b91625cd00b73f6e99950a53f7f3da +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1fd7c38e588ad20b2c66ec480d8e4594ec95da15 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_04-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:860100a3a261ba61004bd104da8b6e61d2f6002646f57a3f3d1426957eb9f78b +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dabff4a325c5b83c5bfb0981106f9ad74faa993a --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05f961f5bc0c8e112dcbdcedcd5a49ddac7bf27c1974f669df40fdbdf51dc9ed +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f577c26107961938a4685cff7f186c1fec4e0cad --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_05-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3adf9d24a241fa88518413e7df7cc840db7e776a50b547e1edc998d1306f90c7 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a8fb1524b6ce965e34142ba432adcf27ad67fed --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:966a712033973e013875c7fa1efc25b1ee84e0f78f1ce86c7f25257ba6e1ca3d +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b7bbae05b349d716d90d48e45adba3eef3d9edb4 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_06-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1dc5445b69d7a7c28a9f0e9b2b398533a372f5a7e41276343cbe23217f5f0903 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b07e3ec5c815876528a54c9d03fb91943d9b713 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c87989ce0a79afb630d16cad1926a5b3153ee4fbceb67c96d0c9ff118e45101b +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a53bd041ab52fdf4b7e8c38b4c33b7e9cda7a233 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_07-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acb732c228a4e21f4243b145a67e91e438cafa1531d8614be50380173c7fcd99 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e595fd3741e269913232e9bdee563ee9d7c66d9 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1fedc0ddeb6b98e6079597a5eddbfe13cdde8d45ac56710c865c73850b5716d +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ea0905cab37bf488e4ddaf14fe3c12c5736a9472 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_08-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5978e0cd03a27cf9cc9760eb685b8c2c4686c6bbb6812d257a8300e02d57390e +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..28c97a1a3b43e922918139d69187b092aeee1ac1 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e5710404e325e941d54f29d840a93d1c5875ae80e4ee434d9c8a02885114d44 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5fefea4c5c77e4eaf3c2982e73efdc0c130d5de --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_09-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53b8927831344616353b44615eee21102f1c21e6703161263679fd380cea9863 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c9d122325de8708ead6fd7e2210dd289ba25742e --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6017dd770c8534b742a811d8b9d6c14564bc0f8b13ced52bee2f02ecd2acb0ae +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b62d0922c08fec1d3f0c9c9f334ab0abacd60e4f --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_10-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f2ab3c062bf41b12db1449cb7bdeb0d2ff3dde3e718106fde74b6f2be4a2f4b +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ce285aad015a5a2f28544e7a6dcdc16624fdc68c --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:229d942eea8e93d4c496caf8bcd46a86be3d38360d2f3e5e2fa9b151f9d2a70c +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf14a9b7d9db68a0acc2b0d8e65ab907f8f1a5fc --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_11-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2215641fe6836976a6858bca56c48461abecd07318201fd01ffbeb57c0768fd1 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1277ff4644d69b53831582e8f02db41cb6d36c1a --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d434b7997ca6ec04713e705481a15aacf5d59cd9e6fe75ee9980b005de5648ae +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..50d7d56f65b08ffdb2ecee697166cff68282e174 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_12-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4f88f532b6d44038f2dbd5abb63aeceee069f975d35d829bdba6fc8d21a73cb +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..815a8f68c3a0f876fb058a51940a41b125349c37 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfea7f77bf4ec35fa2a88c43bcd943111098e59ca05159a4b6ae1fcbf985a549 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e9594a87428ec4b55beac8e60e568e490be86dc8 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_13-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59e0ad73950fcf9057effce5e853887a531d99858060d3dde2aa9bdf8f2bea1e +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..af4889cdd972b5e62abdd33498f01d1c7ad58f8b --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:add78d80b05549089e2496136586f96c1334bb929a9ce142fa5ec8dc7e45a9a5 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..112fc0c7bccf54664a3863c5b701cb47e2e036ad --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_14-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2980849c15d3adc803dc58d08620301c0cbe387a1cca2c7e33692da8a6a28dec +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dda2ef4b09b4ae1bfb8725a49fc148360fb55e29 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f40fd515b3af4939751371d3e56d18f7b7244f90cb91f460f7bc196e9df7cb18 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..29e4cd75413d289d9efa609a82748ee881a90843 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_15-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8169ca5261a0c9910dfdaf2e09e6cc9a9050012c990b8e44467cdfc5c524283 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1f7938ab8752a426236d40e2709ad2b8c5b4ac84 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dcbf118711f9fa587869df904c2a2f3c16af0a90f7f39a571e4e86c7d4285e82 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c85d3a6a761f2356f5256186b794a7b8080c258 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_16-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23d06b326b07869fb6b1c81ab20f19fe628df94009d60d8eb5b95f4ade64f0e1 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2b0331640bf90bf124de34173dab969f2b27480a --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:20a4815cf949fa904d474f9898cee007cb7d40f9c4f58cbbc95ce51906c2054e +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa42126e039b37781c1ccc034e8da2d496bda12a --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_17-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47792782a5c5367edb4faa3a5af0160f38cdcc2dc04ccc5037f4c63324b4fe20 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..646af30c3971296dd346c8e342c35e30055ee77a --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35a7d265a9b6c633ccf94e69192103cccf92f56a28c6e0e2df8ad308f3ac1822 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f1cde855c72578e40c2ca6b4fe3a9e45c2e37882 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_18-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98d702666eceb71f046f93240a16ca3b0ef1d2501cdb8814384a49c23ccc8b11 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f805ff66573e7101ab6ca2ec02462668cd5bc721 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:976bd3788f1661ad6592e4a4c86654e4f7490ef22cccbfe4c6ed50a7d7045af7 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..92177ec2d1682ba5db8a8fba542ab8bca1e8386e --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_19-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb16a79012a78aa18a9f9d659607ab122786e4b4e41ad13c920ae4269b6cfd91 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f435bfee95a0d6fee158e11ab41dca46a9eb7e3c --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2324ca408ea130496fb7cdf37d01794474bd28075762023fff2386e4e415ba0 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..29d7e79d8ad8f99f0b7659ffa6a2850b5e6be999 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_20-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78a66267f3d4929ade17da86a58e137a1ccd0ea4258399210f11c1b62b93bd98 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d24b635db606780f58a538d5a6b2704c81e9fb01 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9da3af42b07bbb442de44c0ccd8e5b5376c7873789112da2c140d2ae008bda32 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ab60ca72121a825ea6443e97346147ac2c4b282 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_21-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6e66ba75d055f184ac967e05c825971db97e833d4a3e1ce2cef44780c8d60f3 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b7e198a1f6d808b15010d077d1e0e22152f01b1 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_22-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcbc253a2833a282d0b9389e5517462cbe349e1192d57f59be98b5efd490d00b +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a4db19458cbb6619cbee621d24cc627ada549e63 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_22-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a336fab6dafc751f8f571ef5598efa5bc39d3b043c53009bd1577af8d8b45e2 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..311bb3c73b624811378a1f391b4c21c08656aad1 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d7eb095f3e951396bafdda68c7c1e4e33481f79004213aad5b1ac83287af582 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..08eac25e5ee9b2bc59eb007efded3f91a0522610 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_23-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:682daaf4467e58bdbb3f5053b841d181dc678808b1ef9bc245c0a5301f068c18 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a09f6943ab98378c95c2627929f81f5167c6f407 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_24-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1b78fc0d572c7c4d0677455331ea5f06fa83299ebd346ae1d084034af98f064 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c9ecdf9790a1395c2068d743e8c21d3ada86d853 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_24-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d94e5fadfb2f21d229a76f67bbda9421caaf716d889205a01374733492611a5 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..85c69d6b6a4eb5e10b3d07561b00fb44f09c025f --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_25-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c97921142ce1faac88af168715ca9cf968e5a86b59eb8a2cf308787d9bad371d +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2b76f6863c2f8f223789ecc449489f7699bd59a6 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_25-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:349a54494396fccc14fb8624bc058b5aa18bdfa9842a83dca45e20c53041e774 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f34badfc0852e6f53b2f952a0574fc9b01d9ac30 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_26-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c57bc81aa2ff58b62d1fd6d8b22121e2ba3d7a6e541af2c618d8843bb05791f +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..584c40d312e664d229c8e4b2dcf7c0818c225371 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_26-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ec80f322c4749598bfb1cf3daf94b21524a8c95b3eeea8a39ef1c51bdf0ddbb +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5bb5f1edc503e1904307d7dfedb278dcbc622fc5 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_27-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80d7e67af4450a908756be0ec8eb1fd1a5a61ab89a8474e53aaf56e19f9283dd +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c70bb977a37fab4d8934c097c85c75903921e29d --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_27-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77c08d4804c7261a40349fdbe908db5d523eed2faff85dfa7355c24706633425 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..22597a659606b1c8c7a0ca9cd5ef2df0e8d1c68c --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_28-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:701be06b2d9b1e4f281e4e8386b4ba7d02a8f36468d92fb290ea7b58d46dae96 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b397f0bc06ce2b50c6934876262cd2a6c5e69720 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_28-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:196b78f7f39cd65c8b379caffbb539e7b97b94fa95b281af9e847320d6b39ba7 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7cdde93af0f0326dabfbfa04cd33a2f9e21ac963 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_29-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0dc4554d466b5a0aa63460bd55ec645e7059a7d0dbff188fb9af891fadcee76a +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..324770dd0f9720cf54799c940ac4eedf79a08abb --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_29-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:814200da331a41a1cc6f9b6fa3a97dcd5c31798ada53c37101448101ad67529f +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d18c998b0ef63806e57b92ad57d9a0d0b6391b8d --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_30-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:853ff465aabd8fb31972e14fe9e8a9ebe1c5feae69839d77fd5573e1b7510c13 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..24edaba3b6468d1e87bbebf14c1800bc3e3dbf8e --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_30-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b4bbce9853c7a09fa779a0dc9fa104f2b69ac2dc43fe50a9d453b659350b7d1 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e17ba46561ee3de91413f0e2f88c36ff3ef47ddf --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_31-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d55d53e4ee9c2b3579c66bd4ee4cfd698fcf66c8f55c1f996a4edf52379b7f9 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3276a27c289b0a55312a2bbfa71f6fe1e8ef698e --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_31-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8230d18b02064f1def728a1e35a05aead0be1589e355ba7fe7cf31ee38f28c1 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..294a15d2fb9eb2c0739616816569d97dea321f1c --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_32-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ea27ce677762c51a5121cd87ecb657d8e1cd15e10c4d95be3fe10c32385822e +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7de19e7e8f9c1c3773e2188acd999ac07dffcc19 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_32-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d64f471e1d69fc4d6bb3ee18193714fc5bdb8795a7a8d9d3d90c781ceca0a4cc +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..89281e89c3011b2a0eb46d7e9ced8abc1246b447 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_33-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:936d9991e0b30b02b803de38d946a32244ab03d3ef5f33fe08207e0a66cbb4ca +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3e682d55377ea35a3a71cccc420c43dba3145911 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_33-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:248615d4180e86f8b7a88ae9d99487ffcd8f1a92dbf97a1c464dc0a82dcddf60 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d3041defcbde64655d5a510f302eb1d99279534 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_34-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d49bf3c3943795995e9fbdb4c00c8262cf85d0a528c4357a587af590cd3407d5 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..448f232a52319cc698e5a6f3bc74ddadcb0edb6e --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_34-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:457b94f7193ef0918b19ab28c16de1fb81b5a1487129ba6f84420db638437e17 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..72a97acef11b2305dd63e7d2fe2312ec270ac752 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_35-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79face36405b905ddcfdfebe566063a046296546d4d84397a49351b4164c1dfc +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..531b8d6c4072334f3ca8cf9c1620afd94a1517f7 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_35-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba27a80046df5c3f554fe268de85d356e747c3e01c2133d90b1269a43e9959fc +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..384793f3c0a5faef39082f1ef88a421e5d9235e5 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_36-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02d5b2b3621f66dace5f994c4e50f4ca01e71e57dcf648730a766e113c6e8290 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cb866fdd318fcbbb0b8252accd9446c4f3a09832 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_36-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9321367e372a6853a0ab67b958696e57a2913c3a519936a99e55f587fb7be76 +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f13649ab106a2898213c6439212f33b2a3a09d7 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_37-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e4a831b17d7c03702610fdd046078634e7937a21a62f6208bc67f3aea7639bb +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a7bb03f3ef0b2d5146ced2a825b72229305307e --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_37-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15a58129f874c275b7be9686fadf2d0d61cd1252c07e5997aedd98189e4f152f +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d8d3186dac647a6795af9c07596d38b2df233a7f --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_38-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5988f9af35c68852b9eb2ada3184a204ddb5fa1fea510b26119f59c56be8520b +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c8fb4bac9bab0d2f02fffb4aa3e58cbd308f735e --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_38-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a54d3783eb46eec44216d4d8293c5b38733326c03eb8e0cfa618b15c31253ef +size 95216131 diff --git a/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..85d2e9105a187216e4d9e484c184d2ab04af9390 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_40-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fd3b24f2c93494d514ad94ee5d992da529ef90b43acf1a83a99cc9a6502756e +size 12483 diff --git a/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt b/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6493be4ae440407bcf7dec78ca1933de74f20451 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/layer_40-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3381292a3c26023eae10fe1c79b472daedeaa64055b1821a1efa57e09334bc02 +size 12483 diff --git a/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt b/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..105887ab54d1278cab8c3fa9d085c0c422db358b --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7556c019340ad4ef9e7edd92b9012fb0fb2fa83eb991c9209e8577a8b2e6fc4 +size 51507 diff --git a/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt b/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb3a20bcf65f21bdef3e5011d04df6451c45ee31 --- /dev/null +++ b/4b284b12bc4opt2/global_step95367/mp_rank_01_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d509a40add0a27d2b44bdaba177a8051efd65be3da25ee10f1b13947aa8401ca +size 51507 diff --git a/4b284b12bc4opt2/latest b/4b284b12bc4opt2/latest new file mode 100644 index 0000000000000000000000000000000000000000..5541d02f038d8d352f8300f277c4a5ab5c7d330b --- /dev/null +++ b/4b284b12bc4opt2/latest @@ -0,0 +1 @@ +global_step95367 \ No newline at end of file diff --git a/4b284b12bc4opt2/sbatch_4b284b12bc4opt2.sh b/4b284b12bc4opt2/sbatch_4b284b12bc4opt2.sh new file mode 100644 index 0000000000000000000000000000000000000000..667dfebeb59226ea7723faf8d049206e3895afdb --- /dev/null +++ b/4b284b12bc4opt2/sbatch_4b284b12bc4opt2.sh @@ -0,0 +1,167 @@ +#!/bin/bash +#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901 +#SBATCH --nodes=32 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=40 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 48:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + +VARIANT=4b284b12bc4opt2 + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" + +TRAIN_DATA_PATH=train12b.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_12B_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + +PP_SIZE=1 +TP_SIZE=2 + +MICRO_BATCH_SIZE=2 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_3802M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=10000 + +# Tokens: 84_000_000_000 +# -> Samples: 41_015_625.0 +# TRAIN_SAMPLES=41_015_625 +# TOKENS: 115_208_849_730.55898 +# Samples: 56_254_321.1574995 +#TRAIN_SAMPLES=56_254_321 +TRAIN_SAMPLES=48_828_125 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.95 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 488_281 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1000 \ + --eval-iters 100 \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683366008.nid005063.107858.0 b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683366008.nid005063.107858.0 new file mode 100644 index 0000000000000000000000000000000000000000..30d94f78a02492bd18baadaff7be5530c8f930a3 --- /dev/null +++ b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683366008.nid005063.107858.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fc66eccccf54076e7b1d9f614e27d4976ae1e5cdc2f6549bb57ba79e74b25a5 +size 40 diff --git a/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683366264.nid006401.14361.0 b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683366264.nid006401.14361.0 new file mode 100644 index 0000000000000000000000000000000000000000..5451d1a8870962ed63862f361a003aab401e550d --- /dev/null +++ b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683366264.nid006401.14361.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:953dbf70e8b71320d5f0caafbcaeed23a44ed5140c55348794242679da1cc456 +size 40 diff --git a/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683366511.nid005063.112529.0 b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683366511.nid005063.112529.0 new file mode 100644 index 0000000000000000000000000000000000000000..960f451adbe2d29f03eb8a1b76fb4384950986c0 --- /dev/null +++ b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683366511.nid005063.112529.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e65875dadbd7748d280bd80326f3e7a24e0d36085554aa2f442d5054982cb552 +size 40 diff --git a/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683366850.nid006518.100515.0 b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683366850.nid006518.100515.0 new file mode 100644 index 0000000000000000000000000000000000000000..10427b035d3381841dea4369790d8e52a9f53835 --- /dev/null +++ b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683366850.nid006518.100515.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19cb4d5ae6d6477445724e56f1549486303adf6e92b693d01cd620a3350bcd39 +size 40 diff --git a/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683367085.nid005063.120643.0 b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683367085.nid005063.120643.0 new file mode 100644 index 0000000000000000000000000000000000000000..05f05ba74f066ea0e8fab0aa75e7bae33d6d5689 --- /dev/null +++ b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683367085.nid005063.120643.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e31a2c844428798e84c2ebd7e58e1bb95f5eba50e54bbde1a5a32b5dfab6072 +size 40 diff --git a/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683372981.nid006518.15047.0 b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683372981.nid006518.15047.0 new file mode 100644 index 0000000000000000000000000000000000000000..23a0ca6af8cc22929b9003a56ab3f82f93227198 --- /dev/null +++ b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683372981.nid006518.15047.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a61a46db96d2cd14166a22b1f2a724b5d21644321821df83bd697bad39ea4615 +size 40 diff --git a/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683373903.nid006518.23681.0 b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683373903.nid006518.23681.0 new file mode 100644 index 0000000000000000000000000000000000000000..7d8f18972a224e190b1624b9876cd31192a2c2b4 --- /dev/null +++ b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683373903.nid006518.23681.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:273ef3187b33d1c6d825be5b951b3f1db15977bc2028acf4792e7c8130f4eaa6 +size 19602 diff --git a/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683374397.nid005061.25042.0 b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683374397.nid005061.25042.0 new file mode 100644 index 0000000000000000000000000000000000000000..e42295abf04bb153b719247e959432868352de01 --- /dev/null +++ b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683374397.nid005061.25042.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd2d74e392a33110704763fd08bd2a23ca6c9670a0e71d34ab2c92b419c25268 +size 40 diff --git a/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683374744.nid006518.33803.0 b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683374744.nid006518.33803.0 new file mode 100644 index 0000000000000000000000000000000000000000..4a51389e06f66fc4f54440f12c412743d145af8e --- /dev/null +++ b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683374744.nid006518.33803.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3fea9e1d324fd201388c6dbe25d1e93ccc85769269b61e0378085912909814a +size 89553134 diff --git a/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683533472.nid005647.37173.0 b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683533472.nid005647.37173.0 new file mode 100644 index 0000000000000000000000000000000000000000..406a6388c57d5e2948bc11b88770cc050b22a952 --- /dev/null +++ b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683533472.nid005647.37173.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe51ff354e0e67281b1c3281f4315ca55ab6f96c3466c63a725131847822ce95 +size 33483902 diff --git a/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683592046.nid006360.71770.0 b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683592046.nid006360.71770.0 new file mode 100644 index 0000000000000000000000000000000000000000..da4da690de3ae7cb1afb07363aaf085148e00ab2 --- /dev/null +++ b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683592046.nid006360.71770.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6515ad09c836a797a19bf4acd095186010e494ce221be03510d6949da7604d6 +size 40 diff --git a/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683613091.nid006518.57787.0 b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683613091.nid006518.57787.0 new file mode 100644 index 0000000000000000000000000000000000000000..0a135a726fffa45d1ca12600d273c64327bc62bb --- /dev/null +++ b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2/events.out.tfevents.1683613091.nid006518.57787.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce61d1f5435b0b96aca6c44a7e8c3e7e7ceb41b84e785654fca668e201002992 +size 81548994 diff --git a/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2val/events.out.tfevents.1683753341.nid006518.129937.0 b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2val/events.out.tfevents.1683753341.nid006518.129937.0 new file mode 100644 index 0000000000000000000000000000000000000000..434600225a08abc2350d3cdfa935fbe016b0f89a --- /dev/null +++ b/4b284b12bc4opt2/tensorboard_4b284b12bc4opt2val/events.out.tfevents.1683753341.nid006518.129937.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ab258f5469acebb5dfbbeb0117cd8017961032559909e5b3a7af18a5c88647d +size 980 diff --git a/4b284b12bc4subopt/3476831.err b/4b284b12bc4subopt/3476831.err new file mode 100644 index 0000000000000000000000000000000000000000..828bae5ac629a62776df4c9506ff7bde2bb99121 --- /dev/null +++ b/4b284b12bc4subopt/3476831.err @@ -0,0 +1,3641 @@ +27: 2023-05-08 11:19:54.015456: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-08 11:19:54.015472: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-08 11:19:54.015502: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-08 11:19:54.015531: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-08 11:19:54.015532: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-08 11:19:54.015534: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-08 11:19:54.015503: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-08 11:19:54.015547: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-08 11:19:54.015827: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-08 11:19:54.015919: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-08 11:19:54.016163: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-08 11:19:54.016170: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-08 11:19:54.016156: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: 2023-05-08 11:19:54.015934: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-08 11:19:54.015956: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-08 11:19:54.015971: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: 2023-05-08 11:19:54.016155: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-08 11:19:54.016162: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-08 11:19:54.016259: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: 2023-05-08 11:19:54.016069: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-08 11:19:54.016102: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-08 11:19:54.016127: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-08 11:19:54.016284: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-08 11:19:54.015977: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-08 11:19:54.016004: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-08 11:19:54.015920: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-08 11:19:54.015957: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-08 11:19:54.016142: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-08 11:19:54.016172: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-08 11:19:54.016304: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-08 11:19:54.016325: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-08 11:19:54.016029: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-08 11:19:54.016044: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-08 11:19:54.016296: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-08 11:19:54.016474: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-08 11:19:54.016492: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-08 11:19:54.016515: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: 2023-05-08 11:19:54.016364: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-08 11:19:54.016378: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-08 11:19:54.016414: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: 2023-05-08 11:19:54.015980: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-08 11:19:54.015952: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-08 11:19:54.016194: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-08 11:19:54.016195: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-08 11:19:54.016210: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: 2023-05-08 11:19:54.016413: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-08 11:19:54.016427: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-08 11:19:54.016460: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: 2023-05-08 11:19:54.016406: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-08 11:19:54.016403: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-08 11:19:54.016070: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-08 11:19:54.016291: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-08 11:19:54.016323: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-08 11:19:54.016333: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-08 11:19:54.016436: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-08 11:19:54.016450: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-08 11:19:54.016030: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-08 11:19:54.016030: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-08 11:19:54.016505: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-08 11:19:54.016515: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-08 11:19:54.016552: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: 2023-05-08 11:19:54.016466: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-08 11:19:54.016493: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-08 11:19:54.016541: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: 2023-05-08 11:19:54.016554: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-08 11:19:54.016563: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-08 11:19:54.016622: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-08 11:19:54.016524: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-08 11:19:54.016531: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-08 11:19:54.016453: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-08 11:19:54.016422: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-08 11:19:54.016740: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-08 11:19:54.016751: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-08 11:19:54.016792: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: 2023-05-08 11:19:54.016726: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-08 11:19:54.016745: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-08 11:19:54.016741: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-08 11:19:54.016522: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-08 11:19:54.016511: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-08 11:19:54.016597: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-08 11:19:54.016607: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-08 11:19:54.016621: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-08 11:19:54.016538: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-08 11:19:54.016551: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-08 11:19:54.016671: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-08 11:19:54.016684: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-08 11:19:54.016689: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: 2023-05-08 11:19:54.016665: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-08 11:19:54.016698: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-08 11:19:54.016750: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: 2023-05-08 11:19:54.016286: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-08 11:19:54.016534: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-08 11:19:54.016543: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-08 11:19:54.016557: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: 2023-05-08 11:19:54.016547: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-08 11:19:54.016813: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-08 11:19:54.016832: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-08 11:19:54.016536: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-08 11:19:54.016545: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-08 11:19:54.016576: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-08 11:19:54.016540: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-08 11:19:54.016564: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-08 11:19:54.016553: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-08 11:19:54.016571: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-08 11:19:54.016590: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: 2023-05-08 11:19:54.016789: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-08 11:19:54.016865: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-08 11:19:54.016900: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-08 11:19:54.016901: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-08 11:19:54.016937: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-08 11:19:54.016847: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-08 11:19:54.016833: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-08 11:19:54.016827: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-08 11:19:54.016681: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-08 11:19:54.016690: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-08 11:19:54.016696: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-08 11:19:54.016576: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-08 11:19:54.016540: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-08 11:19:54.016578: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-08 11:19:54.016738: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-08 11:19:54.016763: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-08 11:19:54.016650: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-08 11:19:54.016667: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-08 11:19:54.016728: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-08 11:19:54.016728: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-08 11:19:54.016728: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-08 11:19:54.016800: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-08 11:19:54.016820: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-08 11:19:54.016839: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-08 11:19:54.016752: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-08 11:19:54.016740: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-08 11:19:54.016767: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-08 11:19:54.016777: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-08 11:19:54.016792: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: 2023-05-08 11:19:54.016623: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-08 11:19:54.016771: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-08 11:19:54.016784: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-08 11:19:54.017003: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-08 11:19:54.017018: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-08 11:19:54.017092: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-08 11:19:54.017102: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-08 11:19:54.017112: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: 2023-05-08 11:19:54.016745: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-08 11:19:54.016760: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-08 11:19:54.016761: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: 2023-05-08 11:19:54.016754: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-08 11:19:54.016755: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-08 11:19:54.016759: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: 2023-05-08 11:19:54.016770: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-08 11:19:54.016664: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-08 11:19:54.016804: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-08 11:19:54.016807: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-08 11:19:54.016817: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-08 11:19:54.017088: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-08 11:19:54.017100: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-08 11:19:54.017119: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: 2023-05-08 11:19:54.016801: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-08 11:19:54.016813: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-08 11:19:54.016740: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-08 11:19:54.017082: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-08 11:19:54.017100: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-08 11:19:54.017079: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: 2023-05-08 11:19:54.017177: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-08 11:19:54.017202: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-08 11:19:54.016834: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-08 11:19:54.017148: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-08 11:19:54.017160: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-08 11:19:54.017273: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-08 11:19:54.017259: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-08 11:19:54.017179: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-08 11:19:54.017123: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-08 11:19:54.017148: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-08 11:19:54.017156: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-08 11:19:54.017164: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-08 11:19:54.017282: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-08 11:19:54.017290: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-08 11:19:54.017293: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-08 11:19:54.016871: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-08 11:19:54.016885: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-08 11:19:54.016961: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-08 11:19:54.016936: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-08 11:19:54.017192: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-08 11:19:54.017196: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-08 11:19:54.017171: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-08 11:19:54.017152: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-08 11:19:54.017344: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-08 11:19:54.017364: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-08 11:19:54.017366: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: 2023-05-08 11:19:54.017152: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-08 11:19:54.017164: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-08 11:19:54.017167: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: 2023-05-08 11:19:54.016897: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-08 11:19:54.016872: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-08 11:19:54.016911: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: 2023-05-08 11:19:54.016936: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-08 11:19:54.016960: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-08 11:19:54.016976: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-08 11:19:54.016972: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-08 11:19:54.017281: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-08 11:19:54.017183: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-08 11:19:54.017195: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-08 11:19:54.017213: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: 2023-05-08 11:19:54.017209: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-08 11:19:54.017407: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-08 11:19:54.017421: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-08 11:19:54.017216: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-08 11:19:54.017212: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-08 11:19:54.016936: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-08 11:19:54.016987: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-08 11:19:54.017216: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-08 11:19:54.017356: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-08 11:19:54.017429: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-08 11:19:54.017437: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-08 11:19:54.017402: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: 2023-05-08 11:19:54.017225: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-08 11:19:54.017235: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-08 11:19:54.017237: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: 2023-05-08 11:19:54.017023: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-08 11:19:54.017627: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-08 11:19:54.017045: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-08 11:19:54.017639: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-08 11:19:54.017062: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-08 11:19:54.017707: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-08 11:19:54.017750: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-08 11:19:54.017797: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-08 11:19:54.017808: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-08 11:19:54.017802: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-08 11:19:54.017832: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-08 11:19:54.017843: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-08 11:19:54.017845: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-08 11:19:54.017841: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-08 11:19:54.017845: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: 2023-05-08 11:19:54.017879: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-08 11:19:54.017891: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-08 11:19:54.017953: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-08 11:19:54.017944: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-08 11:19:54.017957: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-08 11:19:54.017963: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-08 11:19:54.017967: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-08 11:19:54.017962: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-08 11:19:54.018219: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-08 11:19:54.018232: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-08 11:19:54.018224: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-08 11:19:54.018254: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-08 11:19:54.018258: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-08 11:19:54.018265: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-08 11:19:54.018273: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-08 11:19:54.018283: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-08 11:19:54.018441: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-08 11:19:54.018458: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-08 11:19:54.018458: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-08 11:19:54.018467: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-08 11:19:54.018468: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-08 11:19:54.018472: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-08 11:19:54.018466: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-08 11:19:54.018483: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-08 11:20:07.871213: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:07.871253: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:07.871264: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:07.871295: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:07.871285: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:07.871309: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:07.871316: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:07.871314: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:07.871867: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:07.871898: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:07.871914: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:07.871936: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:07.871937: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:07.871954: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:20:07.871977: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:20:07.872011: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:20:07.872272: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:20:07.872027: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:20:07.872050: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:20:07.872057: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:20:07.872075: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:20:07.872047: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:20:07.872308: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:20:07.872081: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:20:07.872319: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:20:07.872367: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:20:07.872352: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:20:07.872375: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:20:07.872377: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:20:07.872350: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:07.872578: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:07.872596: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:07.872632: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:07.872637: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:07.872646: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:07.872663: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:07.872674: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:07.872667: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:07.872849: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:07.872868: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:07.873014: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:07.872895: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:07.873033: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:07.872965: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:07.872896: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:07.873041: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:07.873002: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:07.872903: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:07.873064: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:07.873021: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:07.872904: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:07.873058: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:20:07.873009: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:07.873027: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:07.872911: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:07.873055: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:07.873005: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:07.873061: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:07.873024: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:07.872921: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:07.873020: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:07.873079: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:20:07.873029: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:07.873046: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:20:07.873116: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:07.873094: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:07.873034: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:07.873067: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:20:07.873053: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:07.873026: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:07.873105: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:07.873041: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:20:07.873057: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:07.873050: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:07.873028: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:20:07.873126: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:07.873127: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:07.873046: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:20:07.873069: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:20:07.873147: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:07.873140: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:07.873054: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:20:07.873043: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:07.873044: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:20:07.873145: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:07.873134: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:07.873064: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:20:07.873075: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:07.873055: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:07.873155: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:20:07.873161: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:07.873161: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:07.873063: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:20:07.873066: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:07.873069: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:20:07.873155: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:20:07.873246: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:07.873148: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:07.873075: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:07.873177: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:20:07.873165: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:07.873076: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:07.873181: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:20:07.873171: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:07.873101: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:07.873192: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:20:07.873265: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:07.873093: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:07.873204: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:20:07.873275: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:07.873312: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:07.873204: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:20:07.873288: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:07.873208: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:20:07.873297: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:07.873040: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:07.873216: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:20:07.873305: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:07.873332: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:20:07.873307: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:07.873341: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:20:07.873315: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:07.873357: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:07.873067: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:07.873366: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:07.873072: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:07.873380: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:07.873108: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:07.873394: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:07.873108: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:07.873385: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:07.873374: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:07.873130: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:20:07.873085: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:07.873111: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:07.873490: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:07.873111: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:20:07.873115: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:07.873396: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:20:07.873125: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:07.873404: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:20:07.873142: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:07.873547: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:07.873513: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:07.873418: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:20:07.873137: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:07.873542: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:07.873438: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:20:07.873146: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:07.873550: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:07.873423: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:20:07.873146: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:07.873567: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:07.873559: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:07.873437: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:20:07.873260: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:07.873574: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:07.873569: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:07.873437: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:07.873591: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:07.873574: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:07.873589: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:07.873567: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:07.873605: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:07.873602: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:07.873609: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:07.873634: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:07.873626: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:07.873648: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:07.873648: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:07.873665: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:07.873662: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:07.873687: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:07.873674: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:07.873704: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:07.873675: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:07.873709: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:07.873688: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:07.873707: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:07.873679: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:07.873695: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:07.873651: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:07.873669: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:20:07.873579: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:07.873691: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:07.873671: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:20:07.873604: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:20:07.873599: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:07.873682: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:07.873686: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:20:07.873616: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:20:07.873603: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:07.873809: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:07.873701: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:07.873658: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:07.873708: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:20:07.873623: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:20:07.873629: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:07.873717: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:07.873704: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:20:07.873637: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:20:07.873634: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:20:07.873707: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:07.873828: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:07.873773: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:07.873707: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:07.873672: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:07.873721: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:20:07.873646: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:20:07.873644: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:20:07.873623: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:07.873857: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:07.873732: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:07.873686: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:07.873719: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:20:07.873655: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:20:07.873646: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:20:07.873633: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:20:07.873724: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:07.873871: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:07.873731: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:07.873699: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:07.873728: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:20:07.873661: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:20:07.873641: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:20:07.873646: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:20:07.873737: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:07.873875: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:07.873787: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:07.873704: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:07.873725: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:20:07.873658: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:20:07.873651: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:20:07.873753: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:07.873838: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:07.873805: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:07.873693: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:20:07.873673: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:20:07.873656: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:20:07.873744: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:07.873882: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:07.873817: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:07.873710: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:20:07.873644: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:20:07.873753: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:07.873892: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:07.873816: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:07.873714: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:20:07.873660: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:20:07.873769: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:07.873839: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:20:07.873761: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:07.873831: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:07.873850: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:07.888464: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-08 11:20:07.888493: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-08 11:20:07.888511: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-08 11:20:07.888521: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-08 11:20:07.888535: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-08 11:20:07.888539: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-08 11:20:07.888556: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-08 11:20:07.888767: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-08 11:20:07.889369: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-08 11:20:07.889398: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-08 11:20:07.889433: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-08 11:20:07.889464: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-08 11:20:07.889307: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-08 11:20:07.889339: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-08 11:20:07.889437: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-08 11:20:07.889455: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-08 11:20:07.889537: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-08 11:20:07.889575: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-08 11:20:07.889459: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-08 11:20:07.889495: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-08 11:20:07.889360: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-08 11:20:07.889384: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-08 11:20:07.889488: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-08 11:20:07.889501: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-08 11:20:07.889539: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-08 11:20:07.889391: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-08 11:20:07.889400: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-08 11:20:07.889411: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-08 11:20:07.889503: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-08 11:20:07.889553: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-08 11:20:07.889584: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-08 11:20:07.889503: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-08 11:20:07.889510: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-08 11:20:07.889418: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-08 11:20:07.889530: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-08 11:20:07.889536: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-08 11:20:07.889425: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-08 11:20:07.889435: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-08 11:20:07.889498: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-08 11:20:07.889657: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-08 11:20:07.889689: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-08 11:20:07.889449: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-08 11:20:07.889453: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-08 11:20:07.889457: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-08 11:20:07.889528: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-08 11:20:07.889547: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-08 11:20:07.889550: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-08 11:20:07.889557: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-08 11:20:07.889320: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-08 11:20:07.889352: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-08 11:20:07.889651: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-08 11:20:07.889682: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-08 11:20:07.889600: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-08 11:20:07.889382: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-08 11:20:07.889595: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-08 11:20:07.889701: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-08 11:20:07.889707: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-08 11:20:07.889650: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-08 11:20:07.889682: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-08 11:20:07.889600: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-08 11:20:07.889742: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-08 11:20:07.889645: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-08 11:20:07.889739: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-08 11:20:07.889773: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-08 11:20:07.889729: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-08 11:20:07.889701: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-08 11:20:07.889711: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-08 11:20:07.889718: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-08 11:20:07.889408: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-08 11:20:07.889404: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-08 11:20:07.889617: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-08 11:20:07.889630: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-08 11:20:07.889627: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-08 11:20:07.889635: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-08 11:20:07.889541: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-08 11:20:07.889568: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-08 11:20:07.889640: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-08 11:20:07.889651: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-08 11:20:07.889536: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-08 11:20:07.889764: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-08 11:20:07.889797: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-08 11:20:07.889736: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-08 11:20:07.889730: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-08 11:20:07.889740: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-08 11:20:07.889412: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-08 11:20:07.889441: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-08 11:20:07.889508: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-08 11:20:07.889534: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-08 11:20:07.889782: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-08 11:20:07.889813: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-08 11:20:07.889424: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-08 11:20:07.889807: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-08 11:20:07.889838: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-08 11:20:07.889790: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-08 11:20:07.889813: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-08 11:20:07.889545: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-08 11:20:07.889547: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-08 11:20:07.889773: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-08 11:20:07.889798: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-08 11:20:07.889644: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-08 11:20:07.889801: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-08 11:20:07.889827: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-08 11:20:07.889700: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-08 11:20:07.889669: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-08 11:20:07.889676: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-08 11:20:07.889587: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-08 11:20:07.889677: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-08 11:20:07.889551: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-08 11:20:07.889704: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-08 11:20:07.889725: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-08 11:20:07.889748: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-08 11:20:07.889453: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-08 11:20:07.889554: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-08 11:20:07.889826: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-08 11:20:07.889852: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-08 11:20:07.889830: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-08 11:20:07.889839: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-08 11:20:07.889573: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-08 11:20:07.889580: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-08 11:20:07.889822: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-08 11:20:07.889803: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-08 11:20:07.889834: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-08 11:20:07.889722: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-08 11:20:07.889695: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-08 11:20:07.889707: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-08 11:20:07.889711: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-08 11:20:07.889600: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-08 11:20:07.889620: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-08 11:20:07.889846: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-08 11:20:07.889567: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-08 11:20:07.889576: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-08 11:20:07.889785: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-08 11:20:07.889473: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-08 11:20:07.889482: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-08 11:20:07.889570: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-08 11:20:07.889585: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-08 11:20:07.889586: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-08 11:20:07.889884: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-08 11:20:07.889910: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-08 11:20:07.889866: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-08 11:20:07.889869: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-08 11:20:07.889869: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-08 11:20:07.889872: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-08 11:20:07.889845: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-08 11:20:07.889855: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-08 11:20:07.889865: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-08 11:20:07.889870: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-08 11:20:07.889601: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-08 11:20:07.889608: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-08 11:20:07.889746: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-08 11:20:07.889748: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-08 11:20:07.889756: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-08 11:20:07.889719: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-08 11:20:07.889639: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-08 11:20:07.889585: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-08 11:20:07.889800: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-08 11:20:07.889810: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-08 11:20:07.889828: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-08 11:20:07.889556: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-08 11:20:07.889495: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-08 11:20:07.889493: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-08 11:20:07.889907: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-08 11:20:07.889939: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-08 11:20:07.889917: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-08 11:20:07.889609: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-08 11:20:07.889871: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-08 11:20:07.889882: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-08 11:20:07.889613: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-08 11:20:07.889617: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-08 11:20:07.889859: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-08 11:20:07.890046: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-08 11:20:07.889835: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-08 11:20:07.889821: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-08 11:20:07.889853: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-08 11:20:07.889932: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-08 11:20:07.889896: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-08 11:20:07.889810: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-08 11:20:07.889831: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-08 11:20:07.889788: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-08 11:20:07.889827: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-08 11:20:07.889828: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-08 11:20:07.889843: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-08 11:20:07.889969: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-08 11:20:07.889972: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-08 11:20:07.889955: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-08 11:20:07.889576: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-08 11:20:07.889680: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-08 11:20:07.889812: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-08 11:20:07.889894: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-08 11:20:07.889897: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-08 11:20:07.889886: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-08 11:20:07.889898: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-08 11:20:07.889904: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-08 11:20:07.889831: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-08 11:20:07.889847: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-08 11:20:07.889848: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-08 11:20:07.889864: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-08 11:20:07.889874: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-08 11:20:07.889865: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-08 11:20:07.890027: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-08 11:20:07.889856: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-08 11:20:07.889863: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-08 11:20:07.889757: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-08 11:20:07.889598: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-08 11:20:07.889952: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-08 11:20:07.889920: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-08 11:20:07.889930: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-08 11:20:07.889941: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-08 11:20:07.890080: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-08 11:20:07.889925: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-08 11:20:07.889856: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-08 11:20:07.889891: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-08 11:20:07.889885: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-08 11:20:07.890042: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-08 11:20:07.890050: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-08 11:20:07.889793: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-08 11:20:07.889798: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-08 11:20:07.889612: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-08 11:20:07.889982: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-08 11:20:07.889986: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-08 11:20:07.889991: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-08 11:20:07.890124: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-08 11:20:07.889951: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-08 11:20:07.889967: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-08 11:20:07.890120: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-08 11:20:07.890030: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-08 11:20:07.889902: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-08 11:20:07.889905: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-08 11:20:07.889810: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-08 11:20:07.889814: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-08 11:20:07.889625: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-08 11:20:07.890005: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-08 11:20:07.889986: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-08 11:20:07.890096: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-08 11:20:07.890098: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-08 11:20:07.890124: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-08 11:20:07.889999: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-08 11:20:07.889845: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-08 11:20:07.890203: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-08 11:20:07.889960: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-08 11:20:07.889990: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-08 11:20:07.890172: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-08 11:20:07.890031: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-08 11:20:07.890047: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-08 11:20:07.890065: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-08 11:20:07.890093: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-08 11:20:07.890104: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-08 11:20:07.871955: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:07.871969: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:07.889289: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-08 11:20:07.889315: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-08 11:20:07.889342: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-08 11:20:07.889357: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-08 11:20:07.889352: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-08 11:20:07.889371: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-08 11:20:07.889556: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-08 11:20:07.889574: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-08 11:20:07.890082: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-08 11:20:40.655936: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:20:40.655963: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:40.656420: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:20:40.655957: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:40.656440: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:40.656294: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:20:40.655972: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:40.656447: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:40.656314: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:20:40.655984: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:40.656458: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:40.656334: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:20:40.655986: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:40.656465: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:40.656342: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:20:40.655997: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:40.656478: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:40.656351: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:20:40.656038: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:40.656478: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:40.656357: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:40.656494: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:40.656368: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:40.656371: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:20:40.669043: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:40.669549: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:40.669234: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:20:40.669043: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:40.669546: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:40.669238: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:20:40.669053: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:40.669559: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:40.669242: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:20:40.669056: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:20:40.669064: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-08 11:20:40.669552: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:40.669242: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:20:40.669064: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-08 11:20:40.669068: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-08 11:20:40.669072: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-08 11:20:40.669552: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:40.669569: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-08 11:20:40.669242: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:20:40.669064: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:40.669555: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:40.669244: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:40.669256: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-08 11:20:40.669064: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:40.669574: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-08 11:20:40.669578: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-08 11:20:40.669580: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-08 11:20:40.669244: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:40.669256: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-08 11:20:40.669070: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:40.669583: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-08 11:20:40.669580: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-08 11:20:40.669253: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:40.669263: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-08 11:20:40.669069: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:20:40.669094: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-08 11:20:40.669719: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:20:40.669263: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-08 11:20:40.669268: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-08 11:20:40.669273: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-08 11:20:40.669096: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-08 11:20:40.669097: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-08 11:20:40.669097: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-08 11:20:40.669275: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-08 11:20:40.669279: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-08 11:20:40.669736: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-08 11:20:40.669736: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:20:40.669752: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-08 11:20:40.681212: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:40.681243: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:40.681274: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:40.681286: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:40.681302: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:40.681309: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:40.681323: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:40.681336: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:40.683268: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:40.683295: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:40.683299: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:40.683308: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:40.683311: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:40.683322: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:40.683327: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:40.683330: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:40.683612: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:40.683631: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:40.683639: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:40.683653: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:40.683656: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:40.683663: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:40.683668: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:40.683677: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:40.683927: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:40.683943: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:40.683954: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:40.683964: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:40.683969: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:40.684148: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:40.683986: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:40.684053: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:40.683993: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:40.684165: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:40.683996: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:40.684177: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:40.684073: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:40.684200: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:40.684078: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:40.684202: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:40.684090: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:40.684175: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:40.684204: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:40.684100: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:40.684210: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:40.684101: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:40.684236: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:40.684106: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:40.684196: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:40.684335: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:40.684228: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:40.684201: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:40.684214: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:40.684224: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:40.684354: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:40.684226: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:40.684361: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:40.684237: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:40.684372: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:40.684245: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:40.684378: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:40.684424: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:40.684393: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:40.684393: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:40.684401: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:40.684444: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:40.684466: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:40.684483: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:40.684493: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:40.684496: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:40.684498: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:40.684502: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:40.684423: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:40.684439: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:40.684457: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:40.684454: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:40.684468: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:40.684642: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:40.684483: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:40.684491: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:40.684497: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:40.684649: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:40.684698: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:40.684670: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:40.684673: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:40.684679: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:40.684707: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:40.684762: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:40.684691: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:40.684727: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:40.684692: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:40.684727: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:40.684734: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:40.684736: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:40.684783: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:40.684743: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:40.684792: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:40.684746: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:40.684813: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:40.684849: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:40.684752: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:40.684820: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:40.684832: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:40.684859: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:40.684839: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:40.684878: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:40.684912: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:40.684880: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:40.684897: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:40.684901: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:40.684912: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:40.684920: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:40.684932: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:40.684948: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:40.684958: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:40.684971: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:40.684949: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:40.684977: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:40.684968: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:40.684986: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:40.684979: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:40.684991: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:40.684998: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:40.684998: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:40.684999: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:40.684992: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:40.685009: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:40.685112: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:40.685012: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:40.685109: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:40.685138: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:40.685142: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:40.685145: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:40.685146: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:40.685163: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:40.685173: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:20:40.685047: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:20:40.685075: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:20:40.685068: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:20:40.685084: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:20:40.685097: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:20:40.685099: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:20:40.685108: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:20:40.685112: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:40.686741: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:40.686767: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:40.686792: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:40.686810: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:40.686812: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:40.686818: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:40.687005: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:40.687007: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:20:40.688597: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:20:40.688618: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:20:40.688640: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:20:40.688647: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:20:40.688667: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:20:40.688663: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:20:40.688661: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:20:40.688677: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:40.690028: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:40.690048: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:40.690054: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:40.690071: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:40.690070: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:40.690076: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:40.690087: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:40.690098: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:20:40.695911: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:20:40.695929: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:20:40.695934: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:20:40.695951: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:20:40.695947: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:20:40.695962: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:20:40.696070: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:20:40.696074: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:40.703137: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:40.703146: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:40.703147: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:40.703145: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:40.703151: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:40.703145: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:40.703154: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:40.703158: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:20:40.703173: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-08 11:20:40.703176: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-08 11:20:40.703179: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-08 11:20:40.703185: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-08 11:20:40.703186: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-08 11:20:40.703187: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-08 11:20:40.703190: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-08 11:20:40.703190: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-08 11:20:40.703747: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:40.703752: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:40.703756: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:40.703758: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:40.703773: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-08 11:20:40.703772: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-08 11:20:40.703765: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:40.703775: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:40.703788: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-08 11:20:40.703789: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-08 11:20:40.703794: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-08 11:20:40.703814: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-08 11:20:40.703854: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:40.703854: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:20:40.703893: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-08 11:20:40.703895: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-08 11:20:40.705014: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:20:40.705033: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:20:40.705044: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:20:40.705050: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:20:40.705060: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:20:40.705063: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:20:40.705073: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:20:40.705071: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:40.711947: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:40.711967: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-08 11:20:40.711977: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:40.711982: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:40.711994: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:40.712002: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:40.712016: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-08 11:20:40.712017: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-08 11:20:40.712009: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:40.712011: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:40.712022: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-08 11:20:40.712019: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:40.712010: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:20:40.712026: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-08 11:20:40.712031: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-08 11:20:40.712033: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-08 11:20:40.712036: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-08 11:20:40.712055: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:20:40.712067: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:20:40.712093: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:20:40.712103: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:20:40.712116: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:20:40.712120: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:20:40.712127: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:40.713136: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:40.713136: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:40.713147: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:20:40.713231: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:40.713146: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:40.713169: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-08 11:20:40.713146: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:20:40.713246: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:40.713154: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:20:40.713265: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:40.713169: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-08 11:20:40.713160: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:20:40.713265: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:40.713195: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-08 11:20:40.713196: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-08 11:20:40.713198: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-08 11:20:40.713277: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:40.713200: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-08 11:20:40.713202: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-08 11:20:40.713276: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:40.713237: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:20:40.713280: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-08 11:20:40.713288: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:20:40.713406: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:40.714027: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:20:40.713923: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:40.714053: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:40.714047: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:40.714069: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:40.714076: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:20:40.713952: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:40.714079: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:20:40.713977: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:40.714092: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:20:40.714001: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:40.714100: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:20:40.714011: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:20:40.714012: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:20:40.714035: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:20:40.714038: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:40.714245: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:40.714247: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:40.714250: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:40.714256: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:40.714258: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:40.714266: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:40.714273: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:40.714274: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:20:40.714292: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-08 11:20:40.714293: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-08 11:20:40.714300: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-08 11:20:40.714292: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-08 11:20:40.714295: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-08 11:20:40.714297: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-08 11:20:40.714308: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-08 11:20:40.714307: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-08 11:20:40.714524: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:40.714523: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:40.714527: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:40.714529: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:40.714528: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:40.714529: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:40.714547: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-08 11:20:40.714547: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-08 11:20:40.714547: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-08 11:20:40.714547: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-08 11:20:40.714548: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-08 11:20:40.714553: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-08 11:20:40.714544: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:40.714574: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-08 11:20:40.714628: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:20:40.714649: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-08 11:20:40.714895: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:40.715108: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:40.715118: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:20:40.714910: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:40.715118: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:40.715129: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-08 11:20:40.714923: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:40.715125: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:20:40.714936: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:40.715134: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:40.715141: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-08 11:20:40.714944: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:40.715141: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-08 11:20:40.715136: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:20:40.714943: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:40.715145: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-08 11:20:40.715137: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:20:40.714941: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:40.715138: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:40.715157: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-08 11:20:40.715064: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:20:40.715160: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-08 11:20:40.715161: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-08 11:20:40.715165: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:20:40.715342: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:20:40.715378: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:20:40.715391: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:20:40.715418: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:20:40.715422: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:20:40.715418: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:20:40.715526: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:20:40.715529: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:40.715429: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:40.715433: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:40.715434: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:40.715434: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:40.715438: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:40.715440: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:40.715438: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:40.715437: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:20:40.715462: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-08 11:20:40.715464: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-08 11:20:40.715473: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-08 11:20:40.715472: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-08 11:20:40.715476: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-08 11:20:40.715477: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-08 11:20:40.715479: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-08 11:20:40.715480: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-08 11:20:40.715861: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:40.715880: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-08 11:20:40.715883: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:40.715890: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:40.715890: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:40.715895: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:40.716054: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:40.715895: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:40.716061: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:40.715896: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:40.715912: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-08 11:20:40.716064: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:40.716073: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-08 11:20:40.715905: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:20:40.715915: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-08 11:20:40.715918: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-08 11:20:40.715922: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-08 11:20:40.715924: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-08 11:20:40.715927: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-08 11:20:40.715929: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-08 11:20:40.716161: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:20:40.716163: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:20:40.716166: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:40.716108: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:20:40.716167: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:40.716246: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:20:40.716166: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:40.716248: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:40.716107: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:40.716263: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:20:40.716169: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:20:40.716177: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-08 11:20:40.716249: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:40.716112: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:20:40.716166: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:40.716249: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:40.716113: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:20:40.716360: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:20:40.716169: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:20:40.716179: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-08 11:20:40.716252: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:40.716117: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:40.716264: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:20:40.716182: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-08 11:20:40.716185: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-08 11:20:40.716187: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-08 11:20:40.716409: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:40.716381: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:40.716255: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:40.716120: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:40.716267: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:20:40.716186: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-08 11:20:40.716189: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-08 11:20:40.716189: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-08 11:20:40.716256: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:40.716258: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:40.716265: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-08 11:20:40.716124: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:40.716266: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:40.716386: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:40.716382: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:40.716265: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-08 11:20:40.716262: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:40.716127: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:20:40.716139: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-08 11:20:40.716417: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:40.716267: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:20:40.716336: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:40.716410: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:20:40.716261: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:40.716383: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:40.716272: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-08 11:20:40.716273: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-08 11:20:40.716271: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-08 11:20:40.716140: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-08 11:20:40.716140: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-08 11:20:40.716145: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-08 11:20:40.716418: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:40.716267: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:40.716382: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:40.716412: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:20:40.716270: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:20:40.716278: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-08 11:20:40.716389: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:20:40.716275: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-08 11:20:40.716277: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-08 11:20:40.716280: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-08 11:20:40.716149: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-08 11:20:40.716150: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-08 11:20:40.716153: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-08 11:20:40.716350: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:20:40.716420: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:40.716270: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:40.716368: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:20:40.716340: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:40.716387: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:40.716412: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:20:40.716277: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-08 11:20:40.716274: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:40.716395: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:40.716401: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-08 11:20:40.716153: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-08 11:20:40.716421: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:40.716269: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:40.716283: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-08 11:20:40.716346: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:40.716385: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:40.716412: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:20:40.716272: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:40.716401: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-08 11:20:40.716401: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-08 11:20:40.716397: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:40.716352: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:20:40.716425: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:40.716284: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-08 11:20:40.716286: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-08 11:20:40.716286: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-08 11:20:40.716347: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:40.716385: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:40.716415: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:40.716424: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-08 11:20:40.716278: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:40.716397: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:40.716356: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:20:40.716422: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:40.716284: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-08 11:20:40.716283: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-08 11:20:40.716288: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-08 11:20:40.716345: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:40.716389: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:40.716424: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-08 11:20:40.716417: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:20:40.716273: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:40.716397: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:40.716411: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-08 11:20:40.716406: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:40.716360: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:20:40.716423: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:20:40.716424: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:20:40.716288: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-08 11:20:40.716344: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:40.716390: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:40.716420: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:20:40.716436: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-08 11:20:40.716279: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:20:40.716295: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-08 11:20:40.716402: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:20:40.716418: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-08 11:20:40.716408: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:20:40.716421: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-08 11:20:40.716321: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:40.716359: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:20:40.716422: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:20:40.716433: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-08 11:20:40.716350: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:40.716394: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:20:40.716417: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-08 11:20:40.716435: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-08 11:20:40.716438: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-08 11:20:40.716436: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-08 11:20:40.716298: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-08 11:20:40.716306: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-08 11:20:40.716305: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-08 11:20:40.716426: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-08 11:20:40.716428: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-08 11:20:40.716431: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-08 11:20:40.716413: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:40.716554: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:40.716365: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:20:40.716434: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-08 11:20:40.716435: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-08 11:20:40.716438: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-08 11:20:40.716434: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:20:40.716449: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-08 11:20:40.716350: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:20:40.716366: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-08 11:20:40.716417: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-08 11:20:40.716418: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-08 11:20:40.716421: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-08 11:20:40.716440: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-08 11:20:40.716440: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-08 11:20:40.716307: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-08 11:20:40.716308: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-08 11:20:40.716411: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:20:40.716325: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:40.716582: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:40.716364: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:20:40.716440: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-08 11:20:40.716442: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-08 11:20:40.716443: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-08 11:20:40.716455: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-08 11:20:40.716447: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:20:40.716366: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-08 11:20:40.716371: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-08 11:20:40.716375: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-08 11:20:40.716419: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-08 11:20:40.716423: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-08 11:20:40.716425: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-08 11:20:40.716414: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:20:40.716339: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:40.716584: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:40.716561: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:40.716364: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:40.716382: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-08 11:20:40.716444: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-08 11:20:40.716442: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:20:40.716377: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-08 11:20:40.716376: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-08 11:20:40.716379: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-08 11:20:40.716424: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-08 11:20:40.716418: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:20:40.716431: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-08 11:20:40.716326: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:40.716589: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:40.716562: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:40.716571: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-08 11:20:40.716383: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-08 11:20:40.716382: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-08 11:20:40.716387: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-08 11:20:40.716442: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:20:40.716380: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-08 11:20:40.716431: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-08 11:20:40.716454: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-08 11:20:40.716457: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-08 11:20:40.716330: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:40.716594: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:40.716603: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-08 11:20:40.716566: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:40.716579: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-08 11:20:40.716391: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-08 11:20:40.716394: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-08 11:20:40.716397: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-08 11:20:40.716448: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:20:40.716455: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-08 11:20:40.716453: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-08 11:20:40.716338: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:40.716603: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-08 11:20:40.716604: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-08 11:20:40.716567: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:20:40.716621: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:20:40.716397: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-08 11:20:40.716450: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:20:40.716519: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:20:40.716545: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-08 11:20:40.716338: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:20:40.716354: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-08 11:20:40.716602: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:40.716616: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-08 11:20:40.716570: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:20:40.716457: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:20:40.716482: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-08 11:20:40.716354: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-08 11:20:40.716349: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:40.716600: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:40.716570: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:20:40.716623: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:20:40.716484: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-08 11:20:40.716485: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-08 11:20:40.716487: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-08 11:20:40.716363: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-08 11:20:40.716371: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-08 11:20:40.716374: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-08 11:20:40.716603: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:40.716573: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:20:40.716590: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-08 11:20:40.716630: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:20:40.716488: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-08 11:20:40.716491: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-08 11:20:40.716371: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-08 11:20:40.716377: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-08 11:20:40.716380: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-08 11:20:40.716607: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:40.716631: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-08 11:20:40.716592: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-08 11:20:40.716595: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-08 11:20:40.716596: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-08 11:20:40.716631: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:20:40.716635: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-08 11:20:40.716635: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-08 11:20:40.716637: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-08 11:20:40.716596: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-08 11:20:40.716598: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-08 11:20:40.716633: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:20:40.716659: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-08 11:20:40.716659: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-08 11:20:40.716653: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:20:40.716664: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-08 11:20:40.716661: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-08 11:20:40.716665: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-08 11:20:40.716658: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:20:40.716668: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:20:40.716693: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-08 11:20:40.716700: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-08 11:20:40.716705: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-08 11:20:40.716069: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:20:40.716363: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:40.716378: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:40.716070: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:40.716083: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:20:40.716364: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:40.716383: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:40.716088: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-08 11:20:40.716077: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:20:40.716366: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:40.716381: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:40.716078: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:20:40.716366: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:40.716382: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:40.716399: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-08 11:20:40.716081: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:40.716101: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:20:40.716368: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:20:40.716374: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-08 11:20:40.716388: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:20:40.716099: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-08 11:20:40.716106: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-08 11:20:40.716108: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:20:40.716371: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:40.716392: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:40.716401: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-08 11:20:40.716109: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:20:40.716372: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:20:40.716379: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-08 11:20:40.716393: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:20:40.716405: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:20:40.716380: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:20:40.716382: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:20:40.716384: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-08 11:20:40.716416: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-08 11:20:40.716417: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-08 11:20:40.716420: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:20:40.716385: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:20:40.716387: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:20:40.716388: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-08 11:20:40.716422: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-08 11:20:40.716425: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: Successfully preprocessed all matching files. + 0: Detected CUDA files, patching ldflags + 0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... + 0: Building extension module scaled_upper_triang_masked_softmax_cuda... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module scaled_upper_triang_masked_softmax_cuda... + 0: Successfully preprocessed all matching files. + 0: Detected CUDA files, patching ldflags + 0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... + 0: Building extension module scaled_masked_softmax_cuda... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module scaled_masked_softmax_cuda... + 0: Successfully preprocessed all matching files. + 0: Detected CUDA files, patching ldflags + 0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... + 0: Building extension module fused_mix_prec_layer_norm_cuda... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module fused_mix_prec_layer_norm_cuda... + 4: Successfully preprocessed all matching files. + 0: Successfully preprocessed all matching files. + 0: Successfully preprocessed all matching files. + 5: Successfully preprocessed all matching files. +28: Successfully preprocessed all matching files. +28: Successfully preprocessed all matching files. +28: Successfully preprocessed all matching files. +31: Successfully preprocessed all matching files. +31: Successfully preprocessed all matching files. +29: Successfully preprocessed all matching files. +29: Successfully preprocessed all matching files. + 1: Successfully preprocessed all matching files. + 1: Successfully preprocessed all matching files. + 1: Successfully preprocessed all matching files. +15: Successfully preprocessed all matching files. +15: Successfully preprocessed all matching files. +11: Successfully preprocessed all matching files. +11: Successfully preprocessed all matching files. +17: Successfully preprocessed all matching files. + 2: Successfully preprocessed all matching files. +31: Successfully preprocessed all matching files. +13: Successfully preprocessed all matching files. +13: Successfully preprocessed all matching files. +13: Successfully preprocessed all matching files. + 7: Successfully preprocessed all matching files. + 7: Successfully preprocessed all matching files. +16: Successfully preprocessed all matching files. +16: Successfully preprocessed all matching files. + 7: Successfully preprocessed all matching files. +31: Successfully preprocessed all matching files. +16: Successfully preprocessed all matching files. +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: + 2: + 2: + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: +13: +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: + 6: + 6: +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: + 9: + 9: + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: + 8: + 8: +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: +10: +10: +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: +14: +14: +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: +18: +18: +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: +20: +20: +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: +19: +19: +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: +12: +12: +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: +22: +22: +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: +21: +21: +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: +23: +23: +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: +24: +24: +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: +25: +25: +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: +26: +26: +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: +30: +30: + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: + 3: + 3: +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... + 1: Building extension module utils... + 1: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Loading extension module utils... + 1: Loading extension module utils... + 1: Loading extension module utils... + 1: Loading extension module utils... + 4: Loading extension module utils... + 4: Loading extension module utils... + 4: Loading extension module utils... + 0: Loading extension module utils... + 4: Loading extension module utils... + 0: Loading extension module utils... + 0: Loading extension module utils... + 2: Loading extension module utils... + 2: Loading extension module utils... + 2: Loading extension module utils... + 2: Loading extension module utils... + 5: Loading extension module utils... + 5: Loading extension module utils... + 5: Loading extension module utils... +16: Loading extension module utils... + 5: Loading extension module utils... +16: Loading extension module utils... +13: Loading extension module utils... +15: Loading extension module utils... +14: Loading extension module utils... +13: Loading extension module utils... +14: Loading extension module utils... +16: Loading extension module utils... +13: Loading extension module utils... +15: Loading extension module utils... +16: Loading extension module utils... +14: Loading extension module utils... +15: Loading extension module utils... +17: Loading extension module utils... +14: Loading extension module utils... +13: Loading extension module utils... +15: Loading extension module utils... +17: Loading extension module utils... + 9: Loading extension module utils... +10: Loading extension module utils... +17: Loading extension module utils... +17: Loading extension module utils... + 9: Loading extension module utils... + 9: Loading extension module utils... +10: Loading extension module utils... +10: Loading extension module utils... +11: Loading extension module utils... + 8: Loading extension module utils... +10: Loading extension module utils... + 9: Loading extension module utils... + 6: Loading extension module utils... +11: Loading extension module utils... + 8: Loading extension module utils... + 6: Loading extension module utils... + 8: Loading extension module utils... +11: Loading extension module utils... + 6: Loading extension module utils... + 7: Loading extension module utils... +11: Loading extension module utils... + 8: Loading extension module utils... + 6: Loading extension module utils... + 7: Loading extension module utils... + 7: Loading extension module utils... + 7: Loading extension module utils... +18: Loading extension module utils... +18: Loading extension module utils... +18: Loading extension module utils... +18: Loading extension module utils... +28: Loading extension module utils... +28: Loading extension module utils... +20: Loading extension module utils... +29: Loading extension module utils... +12: Loading extension module utils... +22: Loading extension module utils... +28: Loading extension module utils... +20: Loading extension module utils... +19: Loading extension module utils... +22: Loading extension module utils... +28: Loading extension module utils... +29: Loading extension module utils... +20: Loading extension module utils... +31: Loading extension module utils... +21: Loading extension module utils... +12: Loading extension module utils... +29: Loading extension module utils... +23: Loading extension module utils... +22: Loading extension module utils... +31: Loading extension module utils... +21: Loading extension module utils... +20: Loading extension module utils... +23: Loading extension module utils... +29: Loading extension module utils... +19: Loading extension module utils... +22: Loading extension module utils... +12: Loading extension module utils... +24: Loading extension module utils... +25: Loading extension module utils... +31: Loading extension module utils... +12: Loading extension module utils... +21: Loading extension module utils... +23: Loading extension module utils... +19: Loading extension module utils... +24: Loading extension module utils... +31: Loading extension module utils... +25: Loading extension module utils... +26: Loading extension module utils... +23: Loading extension module utils... +19: Loading extension module utils... +24: Loading extension module utils... +21: Loading extension module utils... +26: Loading extension module utils... +25: Loading extension module utils... +25: Loading extension module utils... +26: Loading extension module utils... +24: Loading extension module utils... +26: Loading extension module utils... +30: Loading extension module utils... +30: Loading extension module utils... +30: Loading extension module utils... +30: Loading extension module utils... + 0: Loading extension module utils... + 3: Loading extension module utils... + 3: Loading extension module utils... + 3: Loading extension module utils... + 3: Loading extension module utils... +27: Loading extension module utils... +27: Loading extension module utils... +27: Loading extension module utils... +27: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: +26: +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... + 0: Building extension module utils... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module utils... + 0: Loading extension module utils... + 0: Loading extension module utils... + 0: Loading extension module utils... + 1: Loading extension module utils... + 2: Loading extension module utils... + 1: Loading extension module utils... + 2: Loading extension module utils... + 1: Loading extension module utils... + 2: Loading extension module utils... + 4: Loading extension module utils... + 4: Loading extension module utils... + 4: Loading extension module utils... + 2: Loading extension module utils... + 4: Loading extension module utils... + 1: Loading extension module utils... + 7: Loading extension module utils... + 7: Loading extension module utils... + 7: Loading extension module utils... + 7: Loading extension module utils... + 5: Loading extension module utils... + 5: Loading extension module utils... + 5: Loading extension module utils... + 6: Loading extension module utils... + 5: Loading extension module utils... + 6: Loading extension module utils... + 6: Loading extension module utils... + 6: Loading extension module utils... + 9: Loading extension module utils... + 8: Loading extension module utils... + 8: Loading extension module utils... + 8: Loading extension module utils... + 9: Loading extension module utils... + 9: Loading extension module utils... + 9: Loading extension module utils... + 8: Loading extension module utils... +11: Loading extension module utils... +11: Loading extension module utils... +13: Loading extension module utils... +11: Loading extension module utils... +10: Loading extension module utils... +11: Loading extension module utils... +13: Loading extension module utils... +10: Loading extension module utils... +10: Loading extension module utils... +13: Loading extension module utils... +13: Loading extension module utils... +10: Loading extension module utils... +12: Loading extension module utils... +16: Loading extension module utils... +15: Loading extension module utils... +12: Loading extension module utils... +15: Loading extension module utils... +16: Loading extension module utils... +12: Loading extension module utils... +15: Loading extension module utils... +16: Loading extension module utils... +15: Loading extension module utils... +16: Loading extension module utils... +14: Loading extension module utils... +17: Loading extension module utils... +12: Loading extension module utils... +14: Loading extension module utils... +17: Loading extension module utils... +14: Loading extension module utils... +17: Loading extension module utils... +14: Loading extension module utils... +17: Loading extension module utils... +18: Loading extension module utils... +18: Loading extension module utils... +18: Loading extension module utils... +20: Loading extension module utils... +18: Loading extension module utils... +19: Loading extension module utils... +20: Loading extension module utils... +19: Loading extension module utils... +19: Loading extension module utils... +20: Loading extension module utils... +19: Loading extension module utils... +20: Loading extension module utils... +22: Loading extension module utils... +22: Loading extension module utils... +22: Loading extension module utils... +22: Loading extension module utils... +21: Loading extension module utils... +21: Loading extension module utils... +21: Loading extension module utils... +21: Loading extension module utils... +28: Loading extension module utils... +28: Loading extension module utils... +28: Loading extension module utils... +23: Loading extension module utils... +23: Loading extension module utils... +28: Loading extension module utils... +23: Loading extension module utils... +23: Loading extension module utils... +29: Loading extension module utils... +24: Loading extension module utils... +29: Loading extension module utils... + 3: Loading extension module utils... +24: Loading extension module utils... + 3: Loading extension module utils... +29: Loading extension module utils... +24: Loading extension module utils... +24: Loading extension module utils... +29: Loading extension module utils... +31: Loading extension module utils... +31: Loading extension module utils... +26: Loading extension module utils... +31: Loading extension module utils... +25: Loading extension module utils... +31: Loading extension module utils... +25: Loading extension module utils... +26: Loading extension module utils... +25: Loading extension module utils... +26: Loading extension module utils... +25: Loading extension module utils... +26: Loading extension module utils... + 3: Loading extension module utils... +30: Loading extension module utils... +30: Loading extension module utils... +30: Loading extension module utils... +30: Loading extension module utils... + 3: Loading extension module utils... +27: Loading extension module utils... +27: Loading extension module utils... +27: Loading extension module utils... +27: Loading extension module utils... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... + 7: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 7: + 7: Loading extension module utils... + 7: Loading extension module utils... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... + 7: + 7: Loading extension module utils... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: + 0: + 0: + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: + 1: + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: +30: +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: +29: +29: + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: + 5: +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 1: + 1: + 1: Loading extension module utils...Loading extension module utils...Loading extension module utils... + 1: + 1: + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +13: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +13: +13: Loading extension module utils...Loading extension module utils... +13: +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 2: + 2: Loading extension module utils... + 2: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +13: No modifications detected for re-loaded extension module utils, skipping build step... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +28: +28: Loading extension module utils...Loading extension module utils... +28: +15: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +13: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 1: + 1: Loading extension module utils... + 1: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +13: No modifications detected for re-loaded extension module utils, skipping build step... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 9: + 9: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +11: +11: Loading extension module utils...Loading extension module utils... +11: +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 2: + 2: + 2: Loading extension module utils...Loading extension module utils...Loading extension module utils... + 2: + 2: +13: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +16: +16: Loading extension module utils...Loading extension module utils... +16: + 1: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 1: + 1: Loading extension module utils... + 1: Loading extension module utils... + 9: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... + 9: + 9: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +15: +15: Loading extension module utils...Loading extension module utils... +15: +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 8: + 8: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... + 8: + 8: + 8: + 8: Loading extension module utils...Loading extension module utils... + 8: + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 6: + 6: Loading extension module utils...Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: + 6: Loading extension module utils... +30: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +30: +30: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...Loading extension module utils... +30: +30: +30: Loading extension module utils... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... + 8: + 8: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... + 8: + 8: Loading extension module utils... + 9: + 9: Loading extension module utils...Loading extension module utils... + 9: +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +10: +10: +10: Loading extension module utils...Loading extension module utils... +10: Loading extension module utils... +10: +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... + 3: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 3: + 3: + 3: Loading extension module utils...Loading extension module utils...Loading extension module utils... + 3: + 3: +13: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +13: +13: Loading extension module utils... +13: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +25: +25: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +25: +25: +25: + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 0: + 0: + 0: + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 6: + 6: Loading extension module utils...Loading extension module utils... + 6: +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... +30: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +25: +25: Loading extension module utils... + 0: Loading extension module utils... + 0: Loading extension module utils...Loading extension module utils... + 0: + 8: No modifications detected for re-loaded extension module utils, skipping build step... +31: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 8: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +29: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +29: +29: Loading extension module utils...Loading extension module utils... +29: +31: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... + 3: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 3: + 3: Loading extension module utils...Loading extension module utils... + 3: +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +10: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +10: +10: Loading extension module utils...Loading extension module utils... +10: +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... + 3: + 3: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... + 3: + 3: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... +31: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +31: +31: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +31: +31: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +31: Loading extension module utils... +31: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step... +31: + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... +29: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... +19: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... +19: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +18: +18: Loading extension module utils... +18: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... + 5: Loading extension module utils...Loading extension module utils... + 5: +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +14: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +14: +14: Loading extension module utils...Loading extension module utils... +14: +14: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +14: +14: Loading extension module utils...Loading extension module utils... +14: +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: No modifications detected for re-loaded extension module utils, skipping build step... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: +12: +24: Loading extension module utils... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: No modifications detected for re-loaded extension module utils, skipping build step... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +24: Loading extension module utils... +12: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +12: +12: Loading extension module utils...Loading extension module utils... +12: +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +24: +24: Loading extension module utils... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +21: +21: Loading extension module utils...Loading extension module utils... +21: +21: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +21: +21: +21: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils...Loading extension module utils... +21: Loading extension module utils... +21: +21: +21: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +21: +21: Loading extension module utils... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Loading extension module utils... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step... +20: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +20: +20: Loading extension module utils...Loading extension module utils... +20: +20: No modifications detected for re-loaded extension module utils, skipping build step... +20: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step... +20: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... +20: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step... +20: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step... +20: Loading extension module utils... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... + 4: +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: +22: +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +17: +17: Loading extension module utils...Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: +17: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +22: +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +17: +17: Loading extension module utils... +22: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +22: +22: +22: Loading extension module utils...Loading extension module utils...Loading extension module utils... +22: +22: +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: +26: +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +27: +27: Loading extension module utils...Loading extension module utils... +27: +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +27: +27: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +26: +26: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/utils.py:349: UserWarning: Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings + 0: warnings.warn("Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings") diff --git a/4b284b12bc4subopt/3476831.out b/4b284b12bc4subopt/3476831.out new file mode 100644 index 0000000000000000000000000000000000000000..e4f5c1b8df9883a233174cb38403b404c19e11cd --- /dev/null +++ b/4b284b12bc4subopt/3476831.out @@ -0,0 +1,40510 @@ +Model parameters: d_model 2560 ffw_size 10240 kv_size 128 n_heads 20 n_layers 34 +Megatron-DeepSpeed/pretrain_gpt.py --tensor-model-parallel-size 2 --pipeline-model-parallel-size 1 --num-layers 34 --hidden-size 2560 --num-attention-heads 20 --kv-channels 128 --ffn-hidden-size 10240 --seq-length 2048 --max-position-embeddings 2048 --micro-batch-size 2 --global-batch-size 1024 --train-samples 1 --vocab-file gpt2/vocab.json --merge-file gpt2/merges.txt --clip-grad 1.0 --kill-switch-path kill-switch-4b284b12bc4suboptval --bf16 --optimizer adam --adam-beta1 0.9 --adam-beta2 0.999 --adam-eps 1e-8 --lr 2e-4 --min-lr 2e-5 --lr-decay-style cosine --lr-decay-samples 1 --lr-warmup-samples 0 --clip-grad 1.0 --weight-decay 1e-1 --override-lr-scheduler --reset-progress --no-load-optim --log-interval 10 --save-interval 1000 --eval-interval 1 --eval-iters 100 --eval-only true --tensorboard-dir tensorboard_4b284b12bc4suboptval --tensorboard-queue-size 5 --log-timers-to-tensorboard --log-batch-size-to-tensorboard --log-validation-ppl-to-tensorboard --save lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt --load lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt --train-weighted-split-paths-path train1b5.txt --valid-weighted-split-paths-path val.txt --data-impl mmap --deepspeed --deepspeed_config ds_configs/3476831.json --zero-stage 0 +START 3476831: Mon 08 May 2023 11:18:47 AM EEST + 0: + 0: + 0: ======================= ROCm System Management Interface ======================= + 0: ================================= Concise Info ================================= + 0: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 0: 0 41.0c 100.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 2 41.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 4 46.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 5 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 6 43.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: ================================================================================ + 0: ============================= End of ROCm SMI Log ============================== + 7: + 7: + 7: ======================= ROCm System Management Interface ======================= + 7: ================================= Concise Info ================================= + 7: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 7: 0 47.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 2 44.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 3 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 4 49.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 6 45.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: ================================================================================ + 7: ============================= End of ROCm SMI Log ============================== + 9: + 9: + 9: ======================= ROCm System Management Interface ======================= + 9: ================================= Concise Info ================================= + 9: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 9: 0 45.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 1 52.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 2 50.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 4 41.0c 100.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 6 40.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: ================================================================================ + 9: ============================= End of ROCm SMI Log ============================== +24: +24: +24: ======================= ROCm System Management Interface ======================= +24: ================================= Concise Info ================================= +24: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +24: 0 43.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 2 44.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 4 47.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 6 39.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: ================================================================================ +24: ============================= End of ROCm SMI Log ============================== +28: +28: +28: ======================= ROCm System Management Interface ======================= +28: ================================= Concise Info ================================= +28: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +28: 0 42.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 2 44.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 4 43.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 5 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 6 40.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 7 39.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: ================================================================================ +28: ============================= End of ROCm SMI Log ============================== +22: +22: +22: ======================= ROCm System Management Interface ======================= +22: ================================= Concise Info ================================= +22: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +22: 0 50.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 2 37.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 3 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 4 48.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 5 54.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 6 44.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 7 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: ================================================================================ +22: ============================= End of ROCm SMI Log ============================== +13: +13: +13: ======================= ROCm System Management Interface ======================= +13: ================================= Concise Info ================================= +13: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +13: 0 43.0c 97.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 2 46.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 4 44.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 6 41.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 7 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: ================================================================================ +13: ============================= End of ROCm SMI Log ============================== +29: +29: +29: ======================= ROCm System Management Interface ======================= +29: ================================= Concise Info ================================= +29: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +29: 0 41.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 2 38.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 4 45.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 6 43.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: ================================================================================ +29: ============================= End of ROCm SMI Log ============================== +10: +10: +10: ======================= ROCm System Management Interface ======================= +10: ================================= Concise Info ================================= +10: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +10: 0 44.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 2 41.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 4 46.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 6 41.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 7 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: ================================================================================ +10: ============================= End of ROCm SMI Log ============================== +19: +19: +19: ======================= ROCm System Management Interface ======================= +19: ================================= Concise Info ================================= +19: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +19: 0 44.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 2 44.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 3 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 4 42.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 5 54.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 6 39.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: ================================================================================ +19: ============================= End of ROCm SMI Log ============================== + 5: + 5: + 5: ======================= ROCm System Management Interface ======================= + 5: ================================= Concise Info ================================= + 5: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 5: 0 43.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 1 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 2 45.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 4 46.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 6 42.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: ================================================================================ + 5: ============================= End of ROCm SMI Log ============================== + 4: + 4: + 4: ======================= ROCm System Management Interface ======================= + 4: ================================= Concise Info ================================= + 4: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 4: 0 44.0c 99.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 2 41.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 4 37.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 6 35.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: ================================================================================ + 4: ============================= End of ROCm SMI Log ============================== + 3: + 3: + 3: ======================= ROCm System Management Interface ======================= + 3: ================================= Concise Info ================================= + 3: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 3: 0 46.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 2 42.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 3 39.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 4 44.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 6 44.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: ================================================================================ + 3: ============================= End of ROCm SMI Log ============================== + 1: + 1: + 1: ======================= ROCm System Management Interface ======================= + 1: ================================= Concise Info ================================= + 1: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 1: 0 43.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 2 43.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 4 43.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 6 42.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: ================================================================================ + 1: ============================= End of ROCm SMI Log ============================== +20: +20: +20: ======================= ROCm System Management Interface ======================= +20: ================================= Concise Info ================================= +20: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +20: 0 47.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 2 45.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 4 46.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 6 42.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: ================================================================================ +20: ============================= End of ROCm SMI Log ============================== +11: +11: +11: ======================= ROCm System Management Interface ======================= +11: ================================= Concise Info ================================= +11: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +11: 0 38.0c 97.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 2 41.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 4 42.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 5 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 6 47.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: ================================================================================ +11: ============================= End of ROCm SMI Log ============================== +12: +12: +12: ======================= ROCm System Management Interface ======================= +12: ================================= Concise Info ================================= +12: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +12: 0 44.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 2 43.0c 98.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 4 47.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 6 40.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: ================================================================================ +12: ============================= End of ROCm SMI Log ============================== +26: +26: +26: ======================= ROCm System Management Interface ======================= +26: ================================= Concise Info ================================= +26: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +26: 0 43.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 2 44.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 4 45.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 5 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 6 45.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: ================================================================================ +26: ============================= End of ROCm SMI Log ============================== +14: +14: +14: ======================= ROCm System Management Interface ======================= +14: ================================= Concise Info ================================= +14: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +14: 0 44.0c 99.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 2 45.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 4 43.0c 98.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 6 45.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: ================================================================================ +14: ============================= End of ROCm SMI Log ============================== + 2: + 2: + 2: ======================= ROCm System Management Interface ======================= + 2: ================================= Concise Info ================================= + 2: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 2: 0 44.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 2 47.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 3 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 4 45.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 6 44.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 7 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: ================================================================================ + 2: ============================= End of ROCm SMI Log ============================== + 8: + 8: + 8: ======================= ROCm System Management Interface ======================= + 8: ================================= Concise Info ================================= + 8: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 8: 0 50.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 2 38.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 4 42.0c 98.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 5 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 6 43.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: ================================================================================ + 8: ============================= End of ROCm SMI Log ============================== +18: +18: +18: ======================= ROCm System Management Interface ======================= +18: ================================= Concise Info ================================= +18: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +18: 0 46.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 2 47.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 4 49.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 6 43.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: ================================================================================ +18: ============================= End of ROCm SMI Log ============================== +27: +27: +27: ======================= ROCm System Management Interface ======================= +27: ================================= Concise Info ================================= +27: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +27: 0 50.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 2 44.0c 97.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 4 45.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 5 52.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 6 44.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 7 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: ================================================================================ +27: ============================= End of ROCm SMI Log ============================== +17: +17: +17: ======================= ROCm System Management Interface ======================= +17: ================================= Concise Info ================================= +17: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +17: 0 45.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 1 52.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 2 45.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 4 43.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 6 42.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: ================================================================================ +17: ============================= End of ROCm SMI Log ============================== +31: +31: +31: ======================= ROCm System Management Interface ======================= +31: ================================= Concise Info ================================= +31: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +31: 0 45.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 2 42.0c 97.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 4 41.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 5 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 6 45.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: ================================================================================ +31: ============================= End of ROCm SMI Log ============================== +15: +15: +15: ======================= ROCm System Management Interface ======================= +15: ================================= Concise Info ================================= +15: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +15: 0 46.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 2 38.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 4 43.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 5 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 6 45.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: ================================================================================ +15: ============================= End of ROCm SMI Log ============================== +16: +16: +16: ======================= ROCm System Management Interface ======================= +16: ================================= Concise Info ================================= +16: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +16: 0 46.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 2 42.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 4 44.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 5 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 6 44.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: ================================================================================ +16: ============================= End of ROCm SMI Log ============================== +21: +21: +21: ======================= ROCm System Management Interface ======================= +21: ================================= Concise Info ================================= +21: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +21: 0 43.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 2 42.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 4 44.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 6 42.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: ================================================================================ +21: ============================= End of ROCm SMI Log ============================== +30: +30: +30: ======================= ROCm System Management Interface ======================= +30: ================================= Concise Info ================================= +30: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +30: 0 44.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 2 44.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 3 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 4 43.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 6 41.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: ================================================================================ +30: ============================= End of ROCm SMI Log ============================== +25: +25: +25: ======================= ROCm System Management Interface ======================= +25: ================================= Concise Info ================================= +25: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +25: 0 46.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 2 44.0c 99.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 4 40.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 6 46.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: ================================================================================ +25: ============================= End of ROCm SMI Log ============================== + 6: + 6: + 6: ======================= ROCm System Management Interface ======================= + 6: ================================= Concise Info ================================= + 6: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 6: 0 46.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 2 47.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 4 47.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 6 43.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 7 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: ================================================================================ + 6: ============================= End of ROCm SMI Log ============================== +23: +23: +23: ======================= ROCm System Management Interface ======================= +23: ================================= Concise Info ================================= +23: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +23: 0 43.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 2 47.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 4 45.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 6 43.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 7 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: ================================================================================ +23: ============================= End of ROCm SMI Log ============================== + 7: Launching on nid006013 (7/32), master nid006006 port 9999, GPUs 8, CUDA: True +27: Launching on nid006033 (27/32), master nid006006 port 9999, GPUs 8, CUDA: True +14: Launching on nid006020 (14/32), master nid006006 port 9999, GPUs 8, CUDA: True + 8: Launching on nid006014 (8/32), master nid006006 port 9999, GPUs 8, CUDA: True +20: Launching on nid006026 (20/32), master nid006006 port 9999, GPUs 8, CUDA: True + 4: Launching on nid006010 (4/32), master nid006006 port 9999, GPUs 8, CUDA: True + 9: Launching on nid006015 (9/32), master nid006006 port 9999, GPUs 8, CUDA: True +12: Launching on nid006018 (12/32), master nid006006 port 9999, GPUs 8, CUDA: True +24: Launching on nid006030 (24/32), master nid006006 port 9999, GPUs 8, CUDA: True +18: Launching on nid006024 (18/32), master nid006006 port 9999, GPUs 8, CUDA: True + 5: Launching on nid006011 (5/32), master nid006006 port 9999, GPUs 8, CUDA: True +28: Launching on nid006034 (28/32), master nid006006 port 9999, GPUs 8, CUDA: True +26: Launching on nid006032 (26/32), master nid006006 port 9999, GPUs 8, CUDA: True +11: Launching on nid006017 (11/32), master nid006006 port 9999, GPUs 8, CUDA: True +29: Launching on nid006035 (29/32), master nid006006 port 9999, GPUs 8, CUDA: True + 1: Launching on nid006007 (1/32), master nid006006 port 9999, GPUs 8, CUDA: True + 3: Launching on nid006009 (3/32), master nid006006 port 9999, GPUs 8, CUDA: True +10: Launching on nid006016 (10/32), master nid006006 port 9999, GPUs 8, CUDA: True +22: Launching on nid006028 (22/32), master nid006006 port 9999, GPUs 8, CUDA: True +19: Launching on nid006025 (19/32), master nid006006 port 9999, GPUs 8, CUDA: True +17: Launching on nid006023 (17/32), master nid006006 port 9999, GPUs 8, CUDA: True + 2: Launching on nid006008 (2/32), master nid006006 port 9999, GPUs 8, CUDA: True + 0: Launching on nid006006 (0/32), master nid006006 port 9999, GPUs 8, CUDA: True +13: Launching on nid006019 (13/32), master nid006006 port 9999, GPUs 8, CUDA: True +31: Launching on nid006037 (31/32), master nid006006 port 9999, GPUs 8, CUDA: True +15: Launching on nid006021 (15/32), master nid006006 port 9999, GPUs 8, CUDA: True +25: Launching on nid006031 (25/32), master nid006006 port 9999, GPUs 8, CUDA: True + 6: Launching on nid006012 (6/32), master nid006006 port 9999, GPUs 8, CUDA: True +21: Launching on nid006027 (21/32), master nid006006 port 9999, GPUs 8, CUDA: True +30: Launching on nid006036 (30/32), master nid006006 port 9999, GPUs 8, CUDA: True +16: Launching on nid006022 (16/32), master nid006006 port 9999, GPUs 8, CUDA: True +23: Launching on nid006029 (23/32), master nid006006 port 9999, GPUs 8, CUDA: True + 0: using world size: 256, data-parallel-size: 128, tensor-model-parallel size: 2, pipeline-model-parallel size: 1 + 0: accumulate and all-reduce gradients in fp32 for bfloat16 data type. + 0: using torch.bfloat16 for parameters ... + 0: ------------------------ arguments ------------------------ + 0: abort_on_unmet_fused_kernel_constraints ......... False + 0: accumulate_allreduce_grads_in_fp32 .............. True + 0: adam_beta1 ...................................... 0.9 + 0: adam_beta2 ...................................... 0.999 + 0: adam_eps ........................................ 1e-08 + 0: adlr_autoresume ................................. False + 0: adlr_autoresume_interval ........................ 1000 + 0: apply_query_key_layer_scaling ................... True + 0: apply_residual_connection_post_layernorm ........ False + 0: attention_dropout ............................... 0.1 + 0: attention_softmax_in_fp32 ....................... False + 0: bert_binary_head ................................ True + 0: bert_load ....................................... None + 0: bf16 ............................................ True + 0: bias_dropout_fusion ............................. True + 0: bias_gelu_fusion ................................ True + 0: biencoder_projection_dim ........................ 0 + 0: biencoder_shared_query_context_model ............ False + 0: block_data_path ................................. None + 0: checkpoint_activations .......................... False + 0: checkpoint_in_cpu ............................... False + 0: checkpoint_num_layers ........................... 1 + 0: clip_grad ....................................... 1.0 + 0: codecarbon_dir .................................. None + 0: consumed_train_samples .......................... 0 + 0: consumed_train_tokens ........................... 0 + 0: consumed_valid_samples .......................... 0 + 0: contigious_checkpointing ........................ False + 0: cpu_optimizer ................................... False + 0: cpu_torch_adam .................................. False + 0: curriculum_learning ............................. False + 0: data_impl ....................................... mmap + 0: data_parallel_size .............................. 128 + 0: data_path ....................................... None + 0: dataloader_type ................................. single + 0: DDP_impl ........................................ local + 0: decoder_seq_length .............................. None + 0: deepscale ....................................... False + 0: deepscale_config ................................ None + 0: deepspeed ....................................... True + 0: deepspeed_activation_checkpointing .............. False + 0: deepspeed_config ................................ ds_configs/3476831.json + 0: deepspeed_mpi ................................... False + 0: distribute_checkpointed_activations ............. False + 0: distributed_backend ............................. nccl + 0: embed_layernorm ................................. False + 0: embedding_path .................................. None + 0: encoder_seq_length .............................. 2048 + 0: eod_mask_loss ................................... False + 0: eval_interval ................................... 1 + 0: eval_iters ...................................... 100 + 0: eval_only ....................................... True + 0: evidence_data_path .............................. None + 0: exit_duration_in_mins ........................... None + 0: exit_interval ................................... None + 0: ffn_hidden_size ................................. 10240 + 0: finetune ........................................ False + 0: fp16 ............................................ False + 0: fp16_lm_cross_entropy ........................... False + 0: fp32_residual_connection ........................ False + 0: gigaflos_no_embeds .............................. 0 + 0: global_batch_size ............................... 1024 + 0: glu_activation .................................. None + 0: hidden_dropout .................................. 0.1 + 0: hidden_size ..................................... 2560 + 0: hysteresis ...................................... 2 + 0: ict_head_size ................................... None + 0: ict_load ........................................ None + 0: img_dim ......................................... 224 + 0: indexer_batch_size .............................. 128 + 0: indexer_log_interval ............................ 1000 + 0: inference ....................................... False + 0: init_method_std ................................. 0.02 + 0: init_method_xavier_uniform ...................... False + 0: initial_loss_scale .............................. 4294967296 + 0: kill_switch_path ................................ kill-switch-4b284b12bc4suboptval + 0: kv_channels ..................................... 128 + 0: layer_norm_fusion ............................... True + 0: layernorm_epsilon ............................... 1e-05 + 0: lazy_mpu_init ................................... None + 0: load ............................................ lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt + 0: local_rank ...................................... None + 0: log_batch_size_to_tensorboard ................... True + 0: log_interval .................................... 10 + 0: log_learning_rate_to_tensorboard ................ True + 0: log_level ....................................... None + 0: log_level_replica ............................... None + 0: log_loss_scale_to_tensorboard ................... True + 0: log_num_zeros_in_grad ........................... False + 0: log_params_norm ................................. False + 0: log_path ........................................ None + 0: log_timers_to_tensorboard ....................... True + 0: log_validation_ppl_to_tensorboard ............... True + 0: loss_on_targets_only ............................ False + 0: loss_scale ...................................... None + 0: loss_scale_window ............................... 1000 + 0: lr .............................................. 0.0002 + 0: lr_decay_iters .................................. None + 0: lr_decay_samples ................................ 1 + 0: lr_decay_style .................................. cosine + 0: lr_decay_tokens ................................. None + 0: lr_warmup_fraction .............................. None + 0: lr_warmup_iters ................................. 0 + 0: lr_warmup_samples ............................... 0 + 0: make_vocab_size_divisible_by .................... 128 + 0: mask_prob ....................................... 0.15 + 0: masked_softmax_fusion ........................... True + 0: max_position_embeddings ......................... 2048 + 0: mean_noise_span_length .......................... None + 0: memory_centric_tiled_linear ..................... False + 0: merge_file ...................................... gpt2/merges.txt + 0: micro_batch_size ................................ 2 + 0: min_loss_scale .................................. 1.0 + 0: min_lr .......................................... 2e-05 + 0: mmap_warmup ..................................... False + 0: no_load_optim ................................... True + 0: no_load_rng ..................................... None + 0: no_save_optim ................................... None + 0: no_save_rng ..................................... None + 0: noise_density ................................... None + 0: num_attention_heads ............................. 20 + 0: num_channels .................................... 3 + 0: num_classes ..................................... 1000 + 0: num_layers ...................................... 34 + 0: num_layers_per_virtual_pipeline_stage ........... None + 0: num_workers ..................................... 2 + 0: onnx_safe ....................................... None + 0: openai_gelu ..................................... False + 0: optimizer ....................................... adam + 0: optimizer_fusion ................................ True + 0: override_lr_scheduler ........................... True + 0: pad_vocab_size_to ............................... None + 0: params_dtype .................................... torch.bfloat16 + 0: partition_activations ........................... False + 0: patch_dim ....................................... 16 + 0: pipeline_model_parallel_size .................... 1 + 0: position_embedding_type ......................... PositionEmbeddingType.absolute + 0: pp_partition_method ............................. None + 0: profile_backward ................................ False + 0: query_in_block_prob ............................. 0.1 + 0: rampup_batch_size ............................... None + 0: rank ............................................ 0 + 0: remote_device ................................... none + 0: reset_attention_mask ............................ False + 0: reset_position_ids .............................. False + 0: reset_progress .................................. True + 0: retriever_report_topk_accuracies ................ [] + 0: retriever_score_scaling ......................... False + 0: retriever_seq_length ............................ 256 + 0: reweight_loss_based_on_position_frequency ....... False + 0: sample_rate ..................................... 1.0 + 0: save ............................................ lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt + 0: save_interval ................................... 1000 + 0: scatter_gather_tensors_in_pipeline .............. True + 0: scattered_embeddings ............................ False + 0: seed ............................................ 1234 + 0: seq_length ...................................... 2048 + 0: sgd_momentum .................................... 0.9 + 0: short_seq_prob .................................. 0.1 + 0: skip_train_iteration_range ...................... None + 0: split ........................................... None + 0: split_transformers .............................. False + 0: sync_tp_duplicated_parameters ................... False + 0: synchronize_each_layer .......................... False + 0: tensor_model_parallel_size ...................... 2 + 0: tensorboard_dir ................................. tensorboard_4b284b12bc4suboptval + 0: tensorboard_log_interval ........................ 1 + 0: tensorboard_queue_size .......................... 5 + 0: test_weighted_split_paths ....................... None + 0: test_weighted_split_paths_path .................. None + 0: tile_factor ..................................... 1 + 0: titles_data_path ................................ None + 0: tokenizer_name_or_path .......................... None + 0: tokenizer_type .................................. GPT2BPETokenizer + 0: train_iters ..................................... None + 0: train_samples ................................... 1 + 0: train_tokens .................................... None + 0: train_weighted_split_names ...................... ['train'] + 0: train_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document']] + 0: train_weighted_split_paths_path ................. None + 0: train_weighted_split_splits ..................... [['0:1']] + 0: train_weighted_split_weights .................... [['1.0']] + 0: universal_checkpoint ............................ False + 0: use_bnb_optimizer ............................... False + 0: use_checkpoint_lr_scheduler ..................... False + 0: use_contiguous_buffers_in_ddp ................... True + 0: use_cpu_initialization .......................... None + 0: use_one_sent_docs ............................... False + 0: use_pin_memory .................................. False + 0: valid_num_workers ............................... 2 + 0: valid_weighted_split_names ...................... ['validation'] + 0: valid_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document']] + 0: valid_weighted_split_paths_path ................. None + 0: valid_weighted_split_splits ..................... [['0:1']] + 0: valid_weighted_split_weights .................... [['1.0']] + 0: virtual_pipeline_model_parallel_size ............ None + 0: vocab_extra_ids ................................. 0 + 0: vocab_file ...................................... gpt2/vocab.json + 0: weight_decay .................................... 0.1 + 0: world_size ...................................... 256 + 0: zero_allgather_bucket_size ...................... 0.0 + 0: zero_contigious_gradients ....................... False + 0: zero_reduce_bucket_size ......................... 0.0 + 0: zero_reduce_scatter ............................. False + 0: zero_stage ...................................... 0 + 0: -------------------- end of arguments --------------------- + 0: setting number of micro-batches to constant 4 + 0: > building GPT2BPETokenizer tokenizer ... + 0: > padded vocab (size: 50257) with 175 dummy tokens (new size: 50432) + 0: DeepSpeed general environment info: + 0: torch install path ............... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch'] + 0: torch version .................... 1.13.0+rocm5.2 + 0: torch cuda version ............... None + 0: torch hip version ................ 5.2.21151-afdc89f8 + 0: nvcc version ..................... None + 0: deepspeed install path ........... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/deepspeed'] + 0: deepspeed info ................... 0.7.5, unknown, unknown + 0: deepspeed wheel compiled w. ...... torch 1.13, hip 5.1 + 0: **** Git info for Megatron: git_hash=unknown git_branch=unknown **** + 0: > initializing torch distributed ... + 0: [2023-05-08 11:22:05,729] [INFO] [comm.py:633:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +31: > setting tensorboard ... + 0: > initializing tensor model parallel with size 2 + 0: > initializing pipeline model parallel with size 1 + 0: > setting random seeds to 1234 ... + 0: > initializing model parallel cuda seeds on global rank 0, model parallel rank 0, and data parallel rank 0 with model parallel seed: 3952 and data parallel seed: 1234 + 0: > compiling dataset index builder ... + 0: make: Entering directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' + 0: make: Nothing to be done for 'default'. + 0: make: Leaving directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' + 0: >>> done with dataset index builder. Compilation time: 0.109 seconds + 0: > compiling and loading fused kernels ... + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.cpp [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.hip [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] + 0: Total number of unsupported CUDA function calls: 0 + 0: + 0: + 0: Total number of replaced kernel launches: 87 + 0: [1/1] c++ scaled_upper_triang_masked_softmax_hip.cuda.o scaled_upper_triang_masked_softmax_hip.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/opt/rocm/lib -lamdhip64 -o scaled_upper_triang_masked_softmax_cuda.so + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.cpp [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.hip [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] + 0: Total number of unsupported CUDA function calls: 0 + 0: + 0: + 0: Total number of replaced kernel launches: 63 + 0: ninja: no work to do. + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda_kernel.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_hip_kernel.hip [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] + 0: Total number of unsupported CUDA function calls: 0 + 0: + 0: + 0: Total number of replaced kernel launches: 67 + 0: [1/1] c++ layer_norm_cuda.o layer_norm_hip_kernel.cuda.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/opt/rocm/lib -lamdhip64 -o fused_mix_prec_layer_norm_cuda.so + 0: >>> done with compiling and loading fused kernels. Compilation time: 24.680 seconds + 0: time to initialize megatron (seconds): 78.970 + 0: [after megatron is initialized] datetime: 2023-05-08 11:22:38 + 0: building GPT model ... + 0: [2023-05-08 11:22:39,126] [INFO] [utils.py:827:see_memory_usage] Before Building Model + 0: [2023-05-08 11:22:39,127] [INFO] [utils.py:828:see_memory_usage] MA 0.0 GB Max_MA 0.0 GB CA 0.0 GB Max_CA 0 GB + 0: [2023-05-08 11:22:39,127] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.32 GB, percent = 7.8% + 0: SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None + 0: Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=0, model=1): 1, ProcessCoord(pipe=0, data=1, model=0): 2, ProcessCoord(pipe=0, data=1, model=1): 3, ProcessCoord(pipe=0, data=2, model=0): 4, ProcessCoord(pipe=0, data=2, model=1): 5, ProcessCoord(pipe=0, data=3, model=0): 6, ProcessCoord(pipe=0, data=3, model=1): 7, ProcessCoord(pipe=0, data=4, model=0): 8, ProcessCoord(pipe=0, data=4, model=1): 9, ProcessCoord(pipe=0, data=5, model=0): 10, ProcessCoord(pipe=0, data=5, model=1): 11, ProcessCoord(pipe=0, data=6, model=0): 12, ProcessCoord(pipe=0, data=6, model=1): 13, ProcessCoord(pipe=0, data=7, model=0): 14, ProcessCoord(pipe=0, data=7, model=1): 15, ProcessCoord(pipe=0, data=8, model=0): 16, ProcessCoord(pipe=0, data=8, model=1): 17, ProcessCoord(pipe=0, data=9, model=0): 18, ProcessCoord(pipe=0, data=9, model=1): 19, ProcessCoord(pipe=0, data=10, model=0): 20, ProcessCoord(pipe=0, data=10, model=1): 21, ProcessCoord(pipe=0, data=11, model=0): 22, ProcessCoord(pipe=0, data + 0: =11, model=1): 23, ProcessCoord(pipe=0, data=12, model=0): 24, ProcessCoord(pipe=0, data=12, model=1): 25, ProcessCoord(pipe=0, data=13, model=0): 26, ProcessCoord(pipe=0, data=13, model=1): 27, ProcessCoord(pipe=0, data=14, model=0): 28, ProcessCoord(pipe=0, data=14, model=1): 29, ProcessCoord(pipe=0, data=15, model=0): 30, ProcessCoord(pipe=0, data=15, model=1): 31, ProcessCoord(pipe=0, data=16, model=0): 32, ProcessCoord(pipe=0, data=16, model=1): 33, ProcessCoord(pipe=0, data=17, model=0): 34, ProcessCoord(pipe=0, data=17, model=1): 35, ProcessCoord(pipe=0, data=18, model=0): 36, ProcessCoord(pipe=0, data=18, model=1): 37, ProcessCoord(pipe=0, data=19, model=0): 38, ProcessCoord(pipe=0, data=19, model=1): 39, ProcessCoord(pipe=0, data=20, model=0): 40, ProcessCoord(pipe=0, data=20, model=1): 41, ProcessCoord(pipe=0, data=21, model=0): 42, ProcessCoord(pipe=0, data=21, model=1): 43, ProcessCoord(pipe=0, data=22, model=0): 44, ProcessCoord(pipe=0, data=22, model=1): 45, ProcessCoord(pipe=0, data=23, model=0 + 0: ): 46, ProcessCoord(pipe=0, data=23, model=1): 47, ProcessCoord(pipe=0, data=24, model=0): 48, ProcessCoord(pipe=0, data=24, model=1): 49, ProcessCoord(pipe=0, data=25, model=0): 50, ProcessCoord(pipe=0, data=25, model=1): 51, ProcessCoord(pipe=0, data=26, model=0): 52, ProcessCoord(pipe=0, data=26, model=1): 53, ProcessCoord(pipe=0, data=27, model=0): 54, ProcessCoord(pipe=0, data=27, model=1): 55, ProcessCoord(pipe=0, data=28, model=0): 56, ProcessCoord(pipe=0, data=28, model=1): 57, ProcessCoord(pipe=0, data=29, model=0): 58, ProcessCoord(pipe=0, data=29, model=1): 59, ProcessCoord(pipe=0, data=30, model=0): 60, ProcessCoord(pipe=0, data=30, model=1): 61, ProcessCoord(pipe=0, data=31, model=0): 62, ProcessCoord(pipe=0, data=31, model=1): 63, ProcessCoord(pipe=0, data=32, model=0): 64, ProcessCoord(pipe=0, data=32, model=1): 65, ProcessCoord(pipe=0, data=33, model=0): 66, ProcessCoord(pipe=0, data=33, model=1): 67, ProcessCoord(pipe=0, data=34, model=0): 68, ProcessCoord(pipe=0, data=34, model=1): 69, Proce + 0: ssCoord(pipe=0, data=35, model=0): 70, ProcessCoord(pipe=0, data=35, model=1): 71, ProcessCoord(pipe=0, data=36, model=0): 72, ProcessCoord(pipe=0, data=36, model=1): 73, ProcessCoord(pipe=0, data=37, model=0): 74, ProcessCoord(pipe=0, data=37, model=1): 75, ProcessCoord(pipe=0, data=38, model=0): 76, ProcessCoord(pipe=0, data=38, model=1): 77, ProcessCoord(pipe=0, data=39, model=0): 78, ProcessCoord(pipe=0, data=39, model=1): 79, ProcessCoord(pipe=0, data=40, model=0): 80, ProcessCoord(pipe=0, data=40, model=1): 81, ProcessCoord(pipe=0, data=41, model=0): 82, ProcessCoord(pipe=0, data=41, model=1): 83, ProcessCoord(pipe=0, data=42, model=0): 84, ProcessCoord(pipe=0, data=42, model=1): 85, ProcessCoord(pipe=0, data=43, model=0): 86, ProcessCoord(pipe=0, data=43, model=1): 87, ProcessCoord(pipe=0, data=44, model=0): 88, ProcessCoord(pipe=0, data=44, model=1): 89, ProcessCoord(pipe=0, data=45, model=0): 90, ProcessCoord(pipe=0, data=45, model=1): 91, ProcessCoord(pipe=0, data=46, model=0): 92, ProcessCoord(pipe + 0: =0, data=46, model=1): 93, ProcessCoord(pipe=0, data=47, model=0): 94, ProcessCoord(pipe=0, data=47, model=1): 95, ProcessCoord(pipe=0, data=48, model=0): 96, ProcessCoord(pipe=0, data=48, model=1): 97, ProcessCoord(pipe=0, data=49, model=0): 98, ProcessCoord(pipe=0, data=49, model=1): 99, ProcessCoord(pipe=0, data=50, model=0): 100, ProcessCoord(pipe=0, data=50, model=1): 101, ProcessCoord(pipe=0, data=51, model=0): 102, ProcessCoord(pipe=0, data=51, model=1): 103, ProcessCoord(pipe=0, data=52, model=0): 104, ProcessCoord(pipe=0, data=52, model=1): 105, ProcessCoord(pipe=0, data=53, model=0): 106, ProcessCoord(pipe=0, data=53, model=1): 107, ProcessCoord(pipe=0, data=54, model=0): 108, ProcessCoord(pipe=0, data=54, model=1): 109, ProcessCoord(pipe=0, data=55, model=0): 110, ProcessCoord(pipe=0, data=55, model=1): 111, ProcessCoord(pipe=0, data=56, model=0): 112, ProcessCoord(pipe=0, data=56, model=1): 113, ProcessCoord(pipe=0, data=57, model=0): 114, ProcessCoord(pipe=0, data=57, model=1): 115, ProcessCoord( + 0: pipe=0, data=58, model=0): 116, ProcessCoord(pipe=0, data=58, model=1): 117, ProcessCoord(pipe=0, data=59, model=0): 118, ProcessCoord(pipe=0, data=59, model=1): 119, ProcessCoord(pipe=0, data=60, model=0): 120, ProcessCoord(pipe=0, data=60, model=1): 121, ProcessCoord(pipe=0, data=61, model=0): 122, ProcessCoord(pipe=0, data=61, model=1): 123, ProcessCoord(pipe=0, data=62, model=0): 124, ProcessCoord(pipe=0, data=62, model=1): 125, ProcessCoord(pipe=0, data=63, model=0): 126, ProcessCoord(pipe=0, data=63, model=1): 127, ProcessCoord(pipe=0, data=64, model=0): 128, ProcessCoord(pipe=0, data=64, model=1): 129, ProcessCoord(pipe=0, data=65, model=0): 130, ProcessCoord(pipe=0, data=65, model=1): 131, ProcessCoord(pipe=0, data=66, model=0): 132, ProcessCoord(pipe=0, data=66, model=1): 133, ProcessCoord(pipe=0, data=67, model=0): 134, ProcessCoord(pipe=0, data=67, model=1): 135, ProcessCoord(pipe=0, data=68, model=0): 136, ProcessCoord(pipe=0, data=68, model=1): 137, ProcessCoord(pipe=0, data=69, model=0): 138, Pr + 0: ocessCoord(pipe=0, data=69, model=1): 139, ProcessCoord(pipe=0, data=70, model=0): 140, ProcessCoord(pipe=0, data=70, model=1): 141, ProcessCoord(pipe=0, data=71, model=0): 142, ProcessCoord(pipe=0, data=71, model=1): 143, ProcessCoord(pipe=0, data=72, model=0): 144, ProcessCoord(pipe=0, data=72, model=1): 145, ProcessCoord(pipe=0, data=73, model=0): 146, ProcessCoord(pipe=0, data=73, model=1): 147, ProcessCoord(pipe=0, data=74, model=0): 148, ProcessCoord(pipe=0, data=74, model=1): 149, ProcessCoord(pipe=0, data=75, model=0): 150, ProcessCoord(pipe=0, data=75, model=1): 151, ProcessCoord(pipe=0, data=76, model=0): 152, ProcessCoord(pipe=0, data=76, model=1): 153, ProcessCoord(pipe=0, data=77, model=0): 154, ProcessCoord(pipe=0, data=77, model=1): 155, ProcessCoord(pipe=0, data=78, model=0): 156, ProcessCoord(pipe=0, data=78, model=1): 157, ProcessCoord(pipe=0, data=79, model=0): 158, ProcessCoord(pipe=0, data=79, model=1): 159, ProcessCoord(pipe=0, data=80, model=0): 160, ProcessCoord(pipe=0, data=80, model= + 0: 1): 161, ProcessCoord(pipe=0, data=81, model=0): 162, ProcessCoord(pipe=0, data=81, model=1): 163, ProcessCoord(pipe=0, data=82, model=0): 164, ProcessCoord(pipe=0, data=82, model=1): 165, ProcessCoord(pipe=0, data=83, model=0): 166, ProcessCoord(pipe=0, data=83, model=1): 167, ProcessCoord(pipe=0, data=84, model=0): 168, ProcessCoord(pipe=0, data=84, model=1): 169, ProcessCoord(pipe=0, data=85, model=0): 170, ProcessCoord(pipe=0, data=85, model=1): 171, ProcessCoord(pipe=0, data=86, model=0): 172, ProcessCoord(pipe=0, data=86, model=1): 173, ProcessCoord(pipe=0, data=87, model=0): 174, ProcessCoord(pipe=0, data=87, model=1): 175, ProcessCoord(pipe=0, data=88, model=0): 176, ProcessCoord(pipe=0, data=88, model=1): 177, ProcessCoord(pipe=0, data=89, model=0): 178, ProcessCoord(pipe=0, data=89, model=1): 179, ProcessCoord(pipe=0, data=90, model=0): 180, ProcessCoord(pipe=0, data=90, model=1): 181, ProcessCoord(pipe=0, data=91, model=0): 182, ProcessCoord(pipe=0, data=91, model=1): 183, ProcessCoord(pipe=0, data + 0: =92, model=0): 184, ProcessCoord(pipe=0, data=92, model=1): 185, ProcessCoord(pipe=0, data=93, model=0): 186, ProcessCoord(pipe=0, data=93, model=1): 187, ProcessCoord(pipe=0, data=94, model=0): 188, ProcessCoord(pipe=0, data=94, model=1): 189, ProcessCoord(pipe=0, data=95, model=0): 190, ProcessCoord(pipe=0, data=95, model=1): 191, ProcessCoord(pipe=0, data=96, model=0): 192, ProcessCoord(pipe=0, data=96, model=1): 193, ProcessCoord(pipe=0, data=97, model=0): 194, ProcessCoord(pipe=0, data=97, model=1): 195, ProcessCoord(pipe=0, data=98, model=0): 196, ProcessCoord(pipe=0, data=98, model=1): 197, ProcessCoord(pipe=0, data=99, model=0): 198, ProcessCoord(pipe=0, data=99, model=1): 199, ProcessCoord(pipe=0, data=100, model=0): 200, ProcessCoord(pipe=0, data=100, model=1): 201, ProcessCoord(pipe=0, data=101, model=0): 202, ProcessCoord(pipe=0, data=101, model=1): 203, ProcessCoord(pipe=0, data=102, model=0): 204, ProcessCoord(pipe=0, data=102, model=1): 205, ProcessCoord(pipe=0, data=103, model=0): 206, Process + 0: Coord(pipe=0, data=103, model=1): 207, ProcessCoord(pipe=0, data=104, model=0): 208, ProcessCoord(pipe=0, data=104, model=1): 209, ProcessCoord(pipe=0, data=105, model=0): 210, ProcessCoord(pipe=0, data=105, model=1): 211, ProcessCoord(pipe=0, data=106, model=0): 212, ProcessCoord(pipe=0, data=106, model=1): 213, ProcessCoord(pipe=0, data=107, model=0): 214, ProcessCoord(pipe=0, data=107, model=1): 215, ProcessCoord(pipe=0, data=108, model=0): 216, ProcessCoord(pipe=0, data=108, model=1): 217, ProcessCoord(pipe=0, data=109, model=0): 218, ProcessCoord(pipe=0, data=109, model=1): 219, ProcessCoord(pipe=0, data=110, model=0): 220, ProcessCoord(pipe=0, data=110, model=1): 221, ProcessCoord(pipe=0, data=111, model=0): 222, ProcessCoord(pipe=0, data=111, model=1): 223, ProcessCoord(pipe=0, data=112, model=0): 224, ProcessCoord(pipe=0, data=112, model=1): 225, ProcessCoord(pipe=0, data=113, model=0): 226, ProcessCoord(pipe=0, data=113, model=1): 227, ProcessCoord(pipe=0, data=114, model=0): 228, ProcessCoord(pipe=0 + 0: , data=114, model=1): 229, ProcessCoord(pipe=0, data=115, model=0): 230, ProcessCoord(pipe=0, data=115, model=1): 231, ProcessCoord(pipe=0, data=116, model=0): 232, ProcessCoord(pipe=0, data=116, model=1): 233, ProcessCoord(pipe=0, data=117, model=0): 234, ProcessCoord(pipe=0, data=117, model=1): 235, ProcessCoord(pipe=0, data=118, model=0): 236, ProcessCoord(pipe=0, data=118, model=1): 237, ProcessCoord(pipe=0, data=119, model=0): 238, ProcessCoord(pipe=0, data=119, model=1): 239, ProcessCoord(pipe=0, data=120, model=0): 240, ProcessCoord(pipe=0, data=120, model=1): 241, ProcessCoord(pipe=0, data=121, model=0): 242, ProcessCoord(pipe=0, data=121, model=1): 243, ProcessCoord(pipe=0, data=122, model=0): 244, ProcessCoord(pipe=0, data=122, model=1): 245, ProcessCoord(pipe=0, data=123, model=0): 246, ProcessCoord(pipe=0, data=123, model=1): 247, ProcessCoord(pipe=0, data=124, model=0): 248, ProcessCoord(pipe=0, data=124, model=1): 249, ProcessCoord(pipe=0, data=125, model=0): 250, ProcessCoord(pipe=0, data=125, + 0: model=1): 251, ProcessCoord(pipe=0, data=126, model=0): 252, ProcessCoord(pipe=0, data=126, model=1): 253, ProcessCoord(pipe=0, data=127, model=0): 254, ProcessCoord(pipe=0, data=127, model=1): 255} + 0: [2023-05-08 11:22:44,608] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer + 0: stage=0 layers=41 + 0: 0: _to_float16 + 0: 1: EmbeddingPipe + 0: 2: + 0: 3: ParallelTransformerLayerPipe + 0: 4: ParallelTransformerLayerPipe + 0: 5: ParallelTransformerLayerPipe + 0: 6: ParallelTransformerLayerPipe + 0: 7: ParallelTransformerLayerPipe + 0: 8: ParallelTransformerLayerPipe + 0: 9: ParallelTransformerLayerPipe + 0: 10: ParallelTransformerLayerPipe + 0: 11: ParallelTransformerLayerPipe + 0: 12: ParallelTransformerLayerPipe + 0: 13: ParallelTransformerLayerPipe + 0: 14: ParallelTransformerLayerPipe + 0: 15: ParallelTransformerLayerPipe + 0: 16: ParallelTransformerLayerPipe + 0: 17: ParallelTransformerLayerPipe + 0: 18: ParallelTransformerLayerPipe + 0: 19: ParallelTransformerLayerPipe + 0: 20: ParallelTransformerLayerPipe + 0: 21: ParallelTransformerLayerPipe + 0: 22: ParallelTransformerLayerPipe + 0: 23: ParallelTransformerLayerPipe + 0: 24: ParallelTransformerLayerPipe + 0: 25: ParallelTransformerLayerPipe + 0: 26: ParallelTransformerLayerPipe + 0: 27: ParallelTransformerLayerPipe + 0: 28: ParallelTransformerLayerPipe + 0: 29: ParallelTransformerLayerPipe + 0: 30: ParallelTransformerLayerPipe + 0: 31: ParallelTransformerLayerPipe + 0: 32: ParallelTransformerLayerPipe + 0: 33: ParallelTransformerLayerPipe + 0: 34: ParallelTransformerLayerPipe + 0: 35: ParallelTransformerLayerPipe + 0: 36: ParallelTransformerLayerPipe + 0: 37: undo + 0: 38: MixedFusedLayerNorm + 0: 39: EmbeddingPipe + 0: 40: float16_to_fp32 + 0: loss: CrossEntropy + 0: [2023-05-08 11:22:45,198] [INFO] [utils.py:827:see_memory_usage] After Building Model + 0: [2023-05-08 11:22:45,199] [INFO] [utils.py:828:see_memory_usage] MA 2.7 GB Max_MA 2.7 GB CA 2.77 GB Max_CA 3 GB + 0: [2023-05-08 11:22:45,199] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.45 GB, percent = 7.8% + 0: setting training iterations to 0 + 0: > learning rate decay style: cosine + 0: DeepSpeed is enabled. + 0: [2023-05-08 11:22:45,202] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.5, git-hash=unknown, git-branch=unknown + 0: [2023-05-08 11:22:50,619] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False + 0: [2023-05-08 11:22:50,620] [INFO] [logging.py:68:log_dist] [Rank 0] Removing param_group that has no 'params' in the client Optimizer + 0: [2023-05-08 11:22:50,620] [INFO] [logging.py:68:log_dist] [Rank 0] Using client Optimizer as basic optimizer + 0: [2023-05-08 11:22:50,639] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam + 0: [2023-05-08 11:22:50,639] [INFO] [logging.py:68:log_dist] [Rank 0] Creating BF16 optimizer + 0: [2023-05-08 11:22:50,781] [INFO] [utils.py:827:see_memory_usage] begin bf16_optimizer + 0: [2023-05-08 11:22:50,781] [INFO] [utils.py:828:see_memory_usage] MA 2.69 GB Max_MA 2.71 GB CA 2.77 GB Max_CA 3 GB + 0: [2023-05-08 11:22:50,781] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.11 GB, percent = 8.0% + 1: ninja: no work to do. +10: Time to load utils op: 0.28797340393066406 secondsTime to load utils op: 0.2879829406738281 seconds +10: +10: Time to load utils op: 0.28798532485961914 seconds +10: Time to load utils op: 0.28800249099731445 seconds + 1: Time to load utils op: 0.2973482608795166 secondsTime to load utils op: 0.29820775985717773 seconds + 1: Time to load utils op: 0.29821276664733887 seconds + 1: + 1: Time to load utils op: 0.2987549304962158 seconds +11: Time to load utils op: 0.29308152198791504 seconds +11: Time to load utils op: 0.29260921478271484 secondsTime to load utils op: 0.293229341506958 seconds +11: +11: Time to load utils op: 0.2926445007324219 seconds + 2: Time to load utils op: 0.29709553718566895 seconds + 2: Time to load utils op: 0.2971014976501465 secondsTime to load utils op: 0.29710888862609863 seconds + 2: + 2: Time to load utils op: 0.2969858646392822 seconds +16: Time to load utils op: 0.29004502296447754 secondsTime to load utils op: 0.29126691818237305 seconds +16: +16: Time to load utils op: 0.2914597988128662 seconds +16: Time to load utils op: 0.29042959213256836 seconds +14: Time to load utils op: 0.289060115814209 seconds +14: Time to load utils op: 0.289064884185791 seconds +14: Time to load utils op: 0.289090633392334 seconds +14: Time to load utils op: 0.2890963554382324 seconds + 0: Time to load utils op: 0.17525768280029297 seconds + 0: Time to load utils op: 0.2998166084289551 seconds + 0: Time to load utils op: 0.30032920837402344 seconds + 0: Time to load utils op: 0.2998671531677246 seconds +28: Time to load utils op: 0.2847445011138916 seconds +28: Time to load utils op: 0.2845616340637207 seconds +28: Time to load utils op: 0.28545331954956055 seconds +28: Time to load utils op: 0.2843942642211914 seconds +18: Time to load utils op: 0.28762125968933105 seconds +18: Time to load utils op: 0.28766608238220215 seconds +18: Time to load utils op: 0.28766751289367676 seconds +18: Time to load utils op: 0.28767919540405273 seconds + 9: Time to load utils op: 0.293459415435791 seconds + 9: Time to load utils op: 0.293468713760376 seconds + 9: Time to load utils op: 0.29348158836364746 seconds + 9: Time to load utils op: 0.2934854030609131 seconds + 4: Time to load utils op: 0.3007235527038574 secondsTime to load utils op: 0.30055737495422363 seconds + 4: + 4: Time to load utils op: 0.3004453182220459 seconds + 4: Time to load utils op: 0.30063748359680176 seconds +15: Time to load utils op: 0.2952260971069336 seconds +15: Time to load utils op: 0.2946500778198242 secondsTime to load utils op: 0.29619312286376953 secondsTime to load utils op: 0.29600048065185547 seconds +15: +15: +12: Time to load utils op: 0.28758931159973145 secondsTime to load utils op: 0.28758883476257324 seconds +12: +12: Time to load utils op: 0.28759193420410156 seconds +12: Time to load utils op: 0.28760814666748047 seconds + 8: Time to load utils op: 0.294905424118042 seconds + 8: Time to load utils op: 0.2949233055114746 secondsTime to load utils op: 0.2949209213256836 seconds + 8: + 8: Time to load utils op: 0.2949404716491699 seconds + 5: Time to load utils op: 0.3018147945404053 secondsTime to load utils op: 0.30196094512939453 seconds + 5: + 5: Time to load utils op: 0.30160093307495117 seconds + 5: Time to load utils op: 0.30175280570983887 seconds + 6: Time to load utils op: 0.29686784744262695 secondsTime to load utils op: 0.29686832427978516 seconds + 6: + 6: Time to load utils op: 0.29688382148742676 seconds + 6: Time to load utils op: 0.2968926429748535 seconds +22: Time to load utils op: 0.28917694091796875 seconds +22: Time to load utils op: 0.28919196128845215 seconds +22: Time to load utils op: 0.289198637008667 seconds +22: Time to load utils op: 0.28918910026550293 seconds +13: Time to load utils op: 0.29920196533203125 secondsTime to load utils op: 0.29905128479003906 seconds +13: +13: Time to load utils op: 0.29930996894836426 seconds +13: Time to load utils op: 0.29884791374206543 seconds +25: Time to load utils op: 0.289050817489624 secondsTime to load utils op: 0.2890605926513672 secondsTime to load utils op: 0.28905558586120605 seconds +25: +25: +25: Time to load utils op: 0.2890784740447998 seconds +21: Time to load utils op: 0.290834903717041 secondsTime to load utils op: 0.29085278511047363 secondsTime to load utils op: 0.29085278511047363 seconds +21: +21: +21: Time to load utils op: 0.29086732864379883 seconds +30: Time to load utils op: 0.2868468761444092 secondsTime to load utils op: 0.2868499755859375 seconds +30: +30: Time to load utils op: 0.2868633270263672 seconds +30: Time to load utils op: 0.2868812084197998 seconds +20: Time to load utils op: 0.292325496673584 seconds +20: Time to load utils op: 0.29233336448669434 seconds +20: Time to load utils op: 0.29233670234680176 secondsTime to load utils op: 0.29233670234680176 seconds +20: + 7: Time to load utils op: 0.30431652069091797 secondsTime to load utils op: 0.30424070358276367 seconds + 7: + 7: Time to load utils op: 0.3043205738067627 seconds + 7: Time to load utils op: 0.30525732040405273 seconds +19: Time to load utils op: 0.2928273677825928 secondsTime to load utils op: 0.2928168773651123 seconds +19: +19: Time to load utils op: 0.2928292751312256 seconds +19: Time to load utils op: 0.29283857345581055 seconds +29: Time to load utils op: 0.2940068244934082 secondsTime to load utils op: 0.29289746284484863 seconds +29: +29: Time to load utils op: 0.2928180694580078 secondsTime to load utils op: 0.29282498359680176 seconds +29: +31: Time to load utils op: 0.2927682399749756 seconds +31: Time to load utils op: 0.29253554344177246 seconds +31: Time to load utils op: 0.2922627925872803 seconds +31: Time to load utils op: 0.29173922538757324 seconds + 3: Time to load utils op: 0.2824385166168213 secondsTime to load utils op: 0.28243565559387207 seconds + 3: + 3: Time to load utils op: 0.2824440002441406 secondsTime to load utils op: 0.2824115753173828 seconds + 3: +27: Time to load utils op: 0.2646474838256836 seconds +24: Time to load utils op: 0.29378342628479004 secondsTime to load utils op: 0.2937886714935303 seconds +24: +27: Time to load utils op: 0.26007699966430664 seconds +27: Time to load utils op: 0.2555701732635498 seconds +27: Time to load utils op: 0.25867605209350586 seconds +24: Time to load utils op: 0.2938120365142822 seconds +24: Time to load utils op: 0.29381704330444336 seconds +26: Time to load utils op: 0.29293179512023926 secondsTime to load utils op: 0.2929401397705078 seconds +26: +26: Time to load utils op: 0.2929508686065674 seconds +26: Time to load utils op: 0.29294681549072266 seconds +17: Time to load utils op: 0.3018944263458252 seconds +17: Time to load utils op: 0.30167603492736816 seconds +17: Time to load utils op: 0.30168652534484863 seconds +17: Time to load utils op: 0.30229711532592773 seconds +23: Time to load utils op: 0.2951340675354004 secondsTime to load utils op: 0.2951390743255615 secondsTime to load utils op: 0.2951319217681885 seconds +23: +23: +23: Time to load utils op: 0.29511570930480957 seconds + 0: [2023-05-08 11:22:51,090] [INFO] [utils.py:827:see_memory_usage] before initializing group 0 + 0: [2023-05-08 11:22:51,091] [INFO] [utils.py:828:see_memory_usage] MA 2.69 GB Max_MA 2.69 GB CA 2.77 GB Max_CA 3 GB + 0: [2023-05-08 11:22:51,091] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.13 GB, percent = 8.0% + 0: ninja: no work to do. + 0: Time to load utils op: 0.17389583587646484 seconds + 0: Time to load utils op: 0.20305275917053223 seconds + 0: Time to load utils op: 0.2025318145751953 seconds + 0: Time to load utils op: 0.20235681533813477 seconds + 2: Time to load utils op: 0.20322179794311523 seconds + 2: Time to load utils op: 0.20378661155700684 seconds + 2: Time to load utils op: 0.20272350311279297 seconds + 1: Time to load utils op: 0.20454621315002441 seconds + 1: Time to load utils op: 0.20395898818969727 seconds + 1: Time to load utils op: 0.20419859886169434 seconds + 4: Time to load utils op: 0.20339584350585938 seconds + 2: Time to load utils op: 0.20223116874694824 seconds + 4: Time to load utils op: 0.20395421981811523 seconds + 4: Time to load utils op: 0.20411300659179688 seconds + 4: Time to load utils op: 0.2039041519165039 seconds + 1: Time to load utils op: 0.2019822597503662 seconds + 7: Time to load utils op: 0.20249319076538086 seconds + 7: Time to load utils op: 0.20333218574523926 seconds + 7: Time to load utils op: 0.20255112648010254 seconds + 7: Time to load utils op: 0.20374059677124023 seconds + 5: Time to load utils op: 0.20409345626831055 seconds + 5: Time to load utils op: 0.20448851585388184 seconds + 5: Time to load utils op: 0.20472145080566406 seconds + 5: Time to load utils op: 0.204817533493042 seconds + 6: Time to load utils op: 0.20319414138793945 seconds + 6: Time to load utils op: 0.20374107360839844 seconds + 6: Time to load utils op: 0.2036149501800537 seconds + 6: Time to load utils op: 0.20342707633972168 seconds + 9: Time to load utils op: 0.2032167911529541 seconds + 9: Time to load utils op: 0.20279669761657715 seconds + 9: Time to load utils op: 0.2034146785736084 seconds + 8: Time to load utils op: 0.20391845703125 seconds + 9: Time to load utils op: 0.20285916328430176 seconds + 8: Time to load utils op: 0.2034900188446045 seconds + 8: Time to load utils op: 0.20405173301696777 seconds + 8: Time to load utils op: 0.20391511917114258 seconds +11: Time to load utils op: 0.2036747932434082 secondsTime to load utils op: 0.20361018180847168 seconds +11: +11: Time to load utils op: 0.20356249809265137 seconds +13: Time to load utils op: 0.20202112197875977 seconds +10: Time to load utils op: 0.20371484756469727 seconds +10: Time to load utils op: 0.20380854606628418 seconds +10: Time to load utils op: 0.20385456085205078 seconds +13: Time to load utils op: 0.20271563529968262 seconds +11: Time to load utils op: 0.2034318447113037 seconds +13: Time to load utils op: 0.2026069164276123 seconds +13: Time to load utils op: 0.20329952239990234 seconds +10: Time to load utils op: 0.2038125991821289 seconds +12: Time to load utils op: 0.203338623046875 seconds +15: Time to load utils op: 0.20320582389831543 seconds +16: Time to load utils op: 0.20318078994750977 seconds +12: Time to load utils op: 0.20345449447631836 seconds +12: Time to load utils op: 0.2040576934814453 seconds +15: Time to load utils op: 0.2033100128173828 seconds +15: Time to load utils op: 0.20385360717773438 seconds +15: Time to load utils op: 0.2025587558746338 seconds +16: Time to load utils op: 0.20322585105895996 seconds +16: Time to load utils op: 0.20238351821899414 secondsTime to load utils op: 0.20318341255187988 seconds +16: +14: Time to load utils op: 0.20363116264343262 seconds +17: Time to load utils op: 0.2025599479675293 seconds +12: Time to load utils op: 0.20212578773498535 seconds +17: Time to load utils op: 0.20379948616027832 seconds +14: Time to load utils op: 0.20395612716674805 seconds +14: Time to load utils op: 0.20383381843566895 seconds +14: Time to load utils op: 0.20397090911865234 seconds +17: Time to load utils op: 0.20354795455932617 seconds +17: Time to load utils op: 0.20328140258789062 seconds +18: Time to load utils op: 0.20380258560180664 seconds +18: Time to load utils op: 0.2038581371307373 seconds +18: Time to load utils op: 0.20390558242797852 seconds +19: Time to load utils op: 0.20314645767211914 seconds +18: Time to load utils op: 0.2043008804321289 seconds +20: Time to load utils op: 0.20328807830810547 seconds +20: Time to load utils op: 0.2024245262145996 seconds +19: Time to load utils op: 0.20333576202392578 secondsTime to load utils op: 0.20334100723266602 seconds +19: +19: Time to load utils op: 0.2035541534423828 seconds +20: Time to load utils op: 0.2035200595855713 seconds +20: Time to load utils op: 0.20356082916259766 seconds +22: Time to load utils op: 0.2028353214263916 seconds +22: Time to load utils op: 0.20275282859802246 seconds +22: Time to load utils op: 0.20235943794250488 seconds +22: Time to load utils op: 0.20231938362121582 seconds +21: Time to load utils op: 0.2042217254638672 seconds +21: Time to load utils op: 0.20450758934020996 seconds +21: Time to load utils op: 0.20446133613586426 seconds +21: Time to load utils op: 0.20467734336853027 seconds +28: Time to load utils op: 0.2030942440032959 seconds +28: Time to load utils op: 0.2025747299194336 seconds +23: Time to load utils op: 0.20404291152954102 seconds +28: Time to load utils op: 0.2025315761566162 seconds +28: Time to load utils op: 0.20231986045837402 seconds +23: Time to load utils op: 0.20477986335754395 seconds +23: Time to load utils op: 0.20448803901672363 seconds +23: Time to load utils op: 0.20498418807983398 seconds +29: Time to load utils op: 0.203751802444458 seconds +29: Time to load utils op: 0.2025456428527832 seconds +24: Time to load utils op: 0.20468473434448242 seconds +29: Time to load utils op: 0.20284795761108398 seconds +24: Time to load utils op: 0.20510387420654297 seconds +29: Time to load utils op: 0.20287752151489258 seconds +24: Time to load utils op: 0.2049424648284912 seconds + 3: Time to load utils op: 0.2039949893951416 secondsTime to load utils op: 0.20390844345092773 seconds + 3: +24: Time to load utils op: 0.20530104637145996 seconds +31: Time to load utils op: 0.20333147048950195 seconds +25: Time to load utils op: 0.2049260139465332 seconds +31: Time to load utils op: 0.20264673233032227 seconds +25: Time to load utils op: 0.2052619457244873 seconds +25: Time to load utils op: 0.20502805709838867 seconds +26: Time to load utils op: 0.2051088809967041 seconds +26: Time to load utils op: 0.20517420768737793 seconds +31: Time to load utils op: 0.20291399955749512 seconds +31: Time to load utils op: 0.20270228385925293 seconds +25: Time to load utils op: 0.20526862144470215 seconds +26: Time to load utils op: 0.20515990257263184 seconds +26: Time to load utils op: 0.20530486106872559 seconds + 3: Time to load utils op: 0.20233368873596191 seconds +30: Time to load utils op: 0.20361804962158203 seconds +30: Time to load utils op: 0.2040870189666748 seconds +30: Time to load utils op: 0.20390081405639648 seconds +30: Time to load utils op: 0.2044074535369873 seconds + 3: Time to load utils op: 0.20198464393615723 seconds +27: Time to load utils op: 0.20299816131591797 seconds +27: Time to load utils op: 0.2026810646057129 seconds +27: Time to load utils op: 0.20293641090393066 seconds +27: Time to load utils op: 0.20216035842895508 seconds + 7: Time to load utils op: 0.0004971027374267578 seconds + 7: Time to load utils op: 0.0005509853363037109 seconds + 7: Time to load utils op: 0.0005295276641845703 seconds + 7: Time to load utils op: 0.0005135536193847656 seconds + 7: Time to load utils op: 0.0005702972412109375 seconds + 7: Time to load utils op: 0.0005819797515869141 secondsTime to load utils op: 0.0005986690521240234 seconds + 7: + 7: Time to load utils op: 0.0006172657012939453 seconds + 1: Time to load utils op: 0.0006458759307861328 secondsTime to load utils op: 0.0007371902465820312 seconds + 1: Time to load utils op: 0.0007357597351074219 seconds + 1: + 2: Time to load utils op: 0.0006933212280273438 seconds +11: Time to load utils op: 0.0005433559417724609 seconds +16: Time to load utils op: 0.0006244182586669922 seconds +13: Time to load utils op: 0.0005524158477783203 seconds +13: Time to load utils op: 0.0005173683166503906 seconds +28: Time to load utils op: 0.0008063316345214844 seconds +15: Time to load utils op: 0.0005509853363037109 seconds +16: Time to load utils op: 0.0005769729614257812 seconds +16: Time to load utils op: 0.00069427490234375 seconds +16: Time to load utils op: 0.0005750656127929688 seconds + 2: Time to load utils op: 0.0006885528564453125 seconds +11: Time to load utils op: 0.0005733966827392578 seconds +15: Time to load utils op: 0.0006031990051269531 seconds + 1: Time to load utils op: 0.0009667873382568359 seconds + 2: Time to load utils op: 0.0006773471832275391 seconds + 2: Time to load utils op: 0.0007169246673583984 seconds +13: Time to load utils op: 0.0005233287811279297 seconds +28: Time to load utils op: 0.0007379055023193359 secondsTime to load utils op: 0.0007619857788085938 seconds +28: + 6: Time to load utils op: 0.0007059574127197266 seconds +16: Time to load utils op: 0.0006558895111083984 seconds + 1: Time to load utils op: 0.0009453296661376953 seconds + 9: Time to load utils op: 0.0008399486541748047 seconds +11: Time to load utils op: 0.0005962848663330078 seconds +15: Time to load utils op: 0.0006208419799804688 seconds + 6: Time to load utils op: 0.00067901611328125 seconds + 2: Time to load utils op: 0.0006997585296630859 seconds + 2: Time to load utils op: 0.0007526874542236328 seconds + 2: Time to load utils op: 0.0007224082946777344 seconds +11: Time to load utils op: 0.0006017684936523438 seconds +10: Time to load utils op: 0.0007796287536621094 seconds +16: Time to load utils op: 0.0008275508880615234 secondsTime to load utils op: 0.000820159912109375 seconds +16: + 1: Time to load utils op: 0.0005221366882324219 seconds +13: Time to load utils op: 0.0005967617034912109 seconds +28: Time to load utils op: 0.0008842945098876953 seconds +15: Time to load utils op: 0.0006816387176513672 seconds +11: Time to load utils op: 0.0007240772247314453 secondsTime to load utils op: 0.000759124755859375 seconds +11: +13: Time to load utils op: 0.0005960464477539062 seconds +15: Time to load utils op: 0.0006890296936035156 seconds + 1: Time to load utils op: 0.0009601116180419922 secondsTime to load utils op: 0.0009136199951171875 seconds + 1: +28: Time to load utils op: 0.0008635520935058594 seconds + 6: Time to load utils op: 0.0008184909820556641 seconds + 6: Time to load utils op: 0.0007023811340332031 secondsTime to load utils op: 0.0007181167602539062 seconds + 6: +25: Time to load utils op: 0.0008678436279296875 seconds + 0: Time to load utils op: 0.0010509490966796875 seconds +13: Time to load utils op: 0.0006442070007324219 seconds +28: Time to load utils op: 0.0008950233459472656 seconds +28: Time to load utils op: 0.0008502006530761719 seconds +15: Time to load utils op: 0.0006778240203857422 seconds +30: Time to load utils op: 0.0008401870727539062 secondsTime to load utils op: 0.0007665157318115234 seconds +30: +30: Time to load utils op: 0.0007963180541992188 seconds + 0: Time to load utils op: 0.0010831356048583984 seconds + 8: Time to load utils op: 0.0009469985961914062 secondsTime to load utils op: 0.0009474754333496094 secondsTime to load utils op: 0.0008368492126464844 secondsTime to load utils op: 0.0009152889251708984 seconds + 8: + 8: + 8: + 8: Time to load utils op: 0.0008208751678466797 seconds + 9: Time to load utils op: 0.0007927417755126953 secondsTime to load utils op: 0.0007734298706054688 seconds + 9: +16: Time to load utils op: 0.0012118816375732422 seconds + 8: Time to load utils op: 0.0008406639099121094 seconds + 8: Time to load utils op: 0.0008993148803710938 seconds + 9: Time to load utils op: 0.0007915496826171875 secondsTime to load utils op: 0.0008530616760253906 seconds + 9: +11: Time to load utils op: 0.0012044906616210938 seconds +13: Time to load utils op: 0.0009567737579345703 secondsTime to load utils op: 0.0008969306945800781 seconds +13: +28: Time to load utils op: 0.0008726119995117188 seconds +10: Time to load utils op: 0.0011222362518310547 seconds +29: Time to load utils op: 0.0010037422180175781 seconds + 3: Time to load utils op: 0.0007417201995849609 secondsTime to load utils op: 0.0006721019744873047 secondsTime to load utils op: 0.0006570816040039062 seconds + 3: + 3: + 9: Time to load utils op: 0.0009019374847412109 seconds + 9: Time to load utils op: 0.0008895397186279297 seconds + 9: Time to load utils op: 0.0008099079132080078 seconds +11: Time to load utils op: 0.0012278556823730469 seconds +10: Time to load utils op: 0.0011696815490722656 secondsTime to load utils op: 0.00116729736328125 seconds +10: Time to load utils op: 0.0011103153228759766 seconds +10: +29: Time to load utils op: 0.001039743423461914 seconds +29: Time to load utils op: 0.001012563705444336 seconds +15: Time to load utils op: 0.0007293224334716797 seconds + 6: Time to load utils op: 0.0009219646453857422 seconds +25: Time to load utils op: 0.0007760524749755859 seconds +25: Time to load utils op: 0.000896453857421875 seconds +25: Time to load utils op: 0.0008733272552490234 secondsTime to load utils op: 0.0008819103240966797 seconds +25: +25: Time to load utils op: 0.0008893013000488281 seconds +25: Time to load utils op: 0.0008785724639892578 seconds + 2: Time to load utils op: 0.0012793540954589844 seconds +15: Time to load utils op: 0.0007789134979248047 seconds + 6: Time to load utils op: 0.0009427070617675781 seconds + 6: Time to load utils op: 0.0008816719055175781 seconds +30: Time to load utils op: 0.0008709430694580078 secondsTime to load utils op: 0.0009720325469970703 seconds +30: + 0: Time to load utils op: 0.0013608932495117188 secondsTime to load utils op: 0.0013644695281982422 secondsTime to load utils op: 0.0013625621795654297 secondsTime to load utils op: 0.0013532638549804688 seconds + 0: + 0: + 0: +31: Time to load utils op: 0.0011320114135742188 seconds + 0: Time to load utils op: 0.0011742115020751953 seconds + 8: Time to load utils op: 0.0009794235229492188 seconds +30: Time to load utils op: 0.0008285045623779297 seconds +25: Time to load utils op: 0.0010180473327636719 seconds +19: Time to load utils op: 0.001367807388305664 seconds +10: Time to load utils op: 0.0012631416320800781 seconds +10: Time to load utils op: 0.0012099742889404297 secondsTime to load utils op: 0.0011932849884033203 seconds +10: + 5: Time to load utils op: 0.0015101432800292969 seconds +30: Time to load utils op: 0.0009331703186035156 seconds + 3: Time to load utils op: 0.0011153221130371094 secondsTime to load utils op: 0.0010952949523925781 seconds + 3: +18: Time to load utils op: 0.001447916030883789 seconds + 3: Time to load utils op: 0.0009126663208007812 seconds +29: Time to load utils op: 0.001079559326171875 secondsTime to load utils op: 0.0010902881622314453 seconds +29: +31: Time to load utils op: 0.0012543201446533203 seconds +31: Time to load utils op: 0.0013282299041748047 seconds +31: Time to load utils op: 0.0012662410736083984 secondsTime to load utils op: 0.0012514591217041016 seconds +31: +31: Time to load utils op: 0.0012302398681640625 seconds + 3: Time to load utils op: 0.0009744167327880859 seconds + 3: Time to load utils op: 0.001116037368774414 seconds + 5: Time to load utils op: 0.0016818046569824219 seconds +31: Time to load utils op: 0.0011670589447021484 seconds +31: Time to load utils op: 0.0012240409851074219 seconds +30: Time to load utils op: 0.0010607242584228516 seconds +29: Time to load utils op: 0.0011188983917236328 seconds +29: Time to load utils op: 0.0011491775512695312 seconds +29: Time to load utils op: 0.0010890960693359375 seconds +18: Time to load utils op: 0.0018258094787597656 seconds +18: Time to load utils op: 0.0019061565399169922 seconds +19: Time to load utils op: 0.0021126270294189453 seconds +18: Time to load utils op: 0.0021178722381591797 seconds +18: Time to load utils op: 0.0021734237670898438 seconds + 5: Time to load utils op: 0.0022928714752197266 seconds +19: Time to load utils op: 0.0020895004272460938 seconds +19: Time to load utils op: 0.0021359920501708984 seconds +18: Time to load utils op: 0.002202749252319336 seconds +18: Time to load utils op: 0.002187490463256836 seconds +18: Time to load utils op: 0.0021817684173583984 seconds + 5: Time to load utils op: 0.0022788047790527344 seconds + 5: Time to load utils op: 0.0022766590118408203 seconds + 5: Time to load utils op: 0.0023441314697265625 seconds + 5: Time to load utils op: 0.0022978782653808594 seconds + 5: Time to load utils op: 0.002318143844604492 seconds +19: Time to load utils op: 0.0021817684173583984 seconds +19: Time to load utils op: 0.002110004425048828 seconds +19: Time to load utils op: 0.002124309539794922 seconds +19: Time to load utils op: 0.002290964126586914 seconds +14: Time to load utils op: 0.0005383491516113281 seconds +14: Time to load utils op: 0.00045561790466308594 secondsTime to load utils op: 0.0004596710205078125 seconds +14: +14: Time to load utils op: 0.0004172325134277344 seconds +14: Time to load utils op: 0.0005564689636230469 secondsTime to load utils op: 0.0004222393035888672 seconds +14: +14: Time to load utils op: 0.0006251335144042969 seconds +14: Time to load utils op: 0.0006122589111328125 seconds +24: Time to load utils op: 0.0005013942718505859 seconds +24: Time to load utils op: 0.00047850608825683594 seconds +24: Time to load utils op: 0.0004611015319824219 secondsTime to load utils op: 0.00042438507080078125 seconds +24: +24: Time to load utils op: 0.0005195140838623047 seconds +12: Time to load utils op: 0.0004918575286865234 seconds +23: Time to load utils op: 0.0006072521209716797 seconds +12: Time to load utils op: 0.0005061626434326172 seconds +24: Time to load utils op: 0.0005860328674316406 seconds +12: Time to load utils op: 0.00047469139099121094 secondsTime to load utils op: 0.0004718303680419922 seconds +12: +24: Time to load utils op: 0.0005471706390380859 seconds +24: Time to load utils op: 0.0006015300750732422 seconds +12: Time to load utils op: 0.0005402565002441406 seconds +12: Time to load utils op: 0.0005037784576416016 seconds +12: Time to load utils op: 0.0005731582641601562 secondsTime to load utils op: 0.0005655288696289062 seconds +12: +21: Time to load utils op: 0.0005106925964355469 seconds +23: Time to load utils op: 0.000942230224609375 seconds +21: Time to load utils op: 0.0005509853363037109 seconds +21: Time to load utils op: 0.0004589557647705078 seconds +21: Time to load utils op: 0.0004477500915527344 secondsTime to load utils op: 0.0004634857177734375 secondsTime to load utils op: 0.0004470348358154297 secondsTime to load utils op: 0.00054168701171875 seconds +21: +21: +21: +21: Time to load utils op: 0.0005056858062744141 seconds +23: Time to load utils op: 0.0012416839599609375 seconds +23: Time to load utils op: 0.0012922286987304688 seconds +23: Time to load utils op: 0.0012090206146240234 seconds +23: Time to load utils op: 0.0012500286102294922 seconds +23: Time to load utils op: 0.0012001991271972656 seconds +23: Time to load utils op: 0.001251220703125 seconds +20: Time to load utils op: 0.0004813671112060547 seconds +20: Time to load utils op: 0.000408172607421875 secondsTime to load utils op: 0.0004012584686279297 secondsTime to load utils op: 0.0005178451538085938 seconds +20: +20: + 4: Time to load utils op: 0.0011126995086669922 seconds +20: Time to load utils op: 0.0005533695220947266 seconds +20: Time to load utils op: 0.0005540847778320312 seconds +20: Time to load utils op: 0.0005810260772705078 seconds +20: Time to load utils op: 0.0005588531494140625 seconds + 4: Time to load utils op: 0.0013039112091064453 seconds + 4: Time to load utils op: 0.0012977123260498047 seconds + 4: Time to load utils op: 0.0012869834899902344 seconds + 4: Time to load utils op: 0.0013022422790527344 seconds + 4: Time to load utils op: 0.001316070556640625 seconds + 4: Time to load utils op: 0.0013446807861328125 seconds +17: Time to load utils op: 0.0005371570587158203 seconds + 4: Time to load utils op: 0.0013666152954101562 seconds +17: Time to load utils op: 0.0005424022674560547 seconds +17: Time to load utils op: 0.0005130767822265625 seconds +17: Time to load utils op: 0.0005235671997070312 seconds +22: Time to load utils op: 0.0005352497100830078 seconds +22: Time to load utils op: 0.0004150867462158203 seconds +22: Time to load utils op: 0.0005018711090087891 seconds +22: Time to load utils op: 0.0005536079406738281 secondsTime to load utils op: 0.0005714893341064453 seconds +22: +17: Time to load utils op: 0.0005614757537841797 seconds +17: Time to load utils op: 0.0005979537963867188 seconds +17: Time to load utils op: 0.0006136894226074219 seconds +22: Time to load utils op: 0.0005769729614257812 seconds +22: Time to load utils op: 0.0005764961242675781 secondsTime to load utils op: 0.0005848407745361328 seconds +22: +17: Time to load utils op: 0.0006501674652099609 seconds +26: Time to load utils op: 0.0008456707000732422 seconds +27: Time to load utils op: 0.0005373954772949219 seconds +27: Time to load utils op: 0.000545501708984375 seconds +27: Time to load utils op: 0.0004391670227050781 secondsTime to load utils op: 0.0004343986511230469 seconds +27: +27: Time to load utils op: 0.0004162788391113281 secondsTime to load utils op: 0.0004215240478515625 seconds +27: +26: Time to load utils op: 0.001230478286743164 seconds +27: Time to load utils op: 0.00043392181396484375 seconds +27: Time to load utils op: 0.00043082237243652344 seconds +26: Time to load utils op: 0.0014719963073730469 seconds +26: Time to load utils op: 0.0013179779052734375 seconds +26: Time to load utils op: 0.0013625621795654297 seconds +26: Time to load utils op: 0.0014193058013916016 seconds +26: Time to load utils op: 0.0013675689697265625 seconds +26: Time to load utils op: 0.0014312267303466797 seconds + 0: [2023-05-08 11:22:51,553] [INFO] [utils.py:827:see_memory_usage] after initializing group 0 + 0: [2023-05-08 11:22:51,554] [INFO] [utils.py:828:see_memory_usage] MA 5.38 GB Max_MA 5.38 GB CA 6.82 GB Max_CA 7 GB + 0: [2023-05-08 11:22:51,554] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.25 GB, percent = 8.0% + 0: [2023-05-08 11:22:51,685] [INFO] [utils.py:827:see_memory_usage] before initializing group 1 + 0: [2023-05-08 11:22:51,685] [INFO] [utils.py:828:see_memory_usage] MA 5.38 GB Max_MA 5.38 GB CA 6.82 GB Max_CA 7 GB + 0: [2023-05-08 11:22:51,686] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.23 GB, percent = 8.0% + 0: [2023-05-08 11:22:51,813] [INFO] [utils.py:827:see_memory_usage] after initializing group 1 + 0: [2023-05-08 11:22:51,813] [INFO] [utils.py:828:see_memory_usage] MA 7.9 GB Max_MA 7.9 GB CA 10.63 GB Max_CA 11 GB + 0: [2023-05-08 11:22:51,813] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.25 GB, percent = 8.0% + 0: [2023-05-08 11:22:51,936] [INFO] [utils.py:827:see_memory_usage] before initializing group 2 + 0: [2023-05-08 11:22:51,936] [INFO] [utils.py:828:see_memory_usage] MA 7.9 GB Max_MA 7.9 GB CA 10.63 GB Max_CA 11 GB + 0: [2023-05-08 11:22:51,936] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.22 GB, percent = 8.0% + 0: [2023-05-08 11:22:52,065] [INFO] [utils.py:827:see_memory_usage] after initializing group 2 + 0: [2023-05-08 11:22:52,066] [INFO] [utils.py:828:see_memory_usage] MA 7.91 GB Max_MA 7.91 GB CA 10.63 GB Max_CA 11 GB + 0: [2023-05-08 11:22:52,066] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.24 GB, percent = 8.0% + 0: [2023-05-08 11:22:52,190] [INFO] [utils.py:827:see_memory_usage] before initialize_optimizer + 0: [2023-05-08 11:22:52,190] [INFO] [utils.py:828:see_memory_usage] MA 7.91 GB Max_MA 7.91 GB CA 10.63 GB Max_CA 11 GB + 0: [2023-05-08 11:22:52,191] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.27 GB, percent = 8.0% + 0: [2023-05-08 11:22:52,320] [INFO] [utils.py:827:see_memory_usage] end initialize_optimizer + 0: [2023-05-08 11:22:52,321] [INFO] [utils.py:828:see_memory_usage] MA 7.99 GB Max_MA 7.99 GB CA 10.63 GB Max_CA 11 GB + 0: [2023-05-08 11:22:52,321] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.25 GB, percent = 8.0% + 0: [2023-05-08 11:22:52,446] [INFO] [utils.py:827:see_memory_usage] end bf16_optimizer + 0: [2023-05-08 11:22:52,446] [INFO] [utils.py:828:see_memory_usage] MA 7.99 GB Max_MA 7.99 GB CA 10.63 GB Max_CA 11 GB + 0: [2023-05-08 11:22:52,446] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 40.2 GB, percent = 8.0% + 0: [2023-05-08 11:22:52,447] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam + 0: [2023-05-08 11:22:52,447] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using client LR scheduler + 0: [2023-05-08 11:22:52,447] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = + 0: [2023-05-08 11:22:52,447] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0002, 0.0002, 0.0002], mom=[(0.9, 0.999), (0.9, 0.999), (0.9, 0.999)] + 0: [2023-05-08 11:22:52,447] [INFO] [config.py:1007:print] DeepSpeedEngine configuration: + 0: [2023-05-08 11:22:52,448] [INFO] [config.py:1011:print] activation_checkpointing_config { + 0: "partition_activations": false, + 0: "contiguous_memory_optimization": false, + 0: "cpu_checkpointing": false, + 0: "number_checkpoints": null, + 0: "synchronize_checkpoint_boundary": false, + 0: "profile": false + 0: } + 0: [2023-05-08 11:22:52,448] [INFO] [config.py:1011:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} + 0: [2023-05-08 11:22:52,448] [INFO] [config.py:1011:print] amp_enabled .................. False + 0: [2023-05-08 11:22:52,448] [INFO] [config.py:1011:print] amp_params ................... False + 0: [2023-05-08 11:22:52,448] [INFO] [config.py:1011:print] autotuning_config ............ { + 0: "enabled": false, + 0: "start_step": null, + 0: "end_step": null, + 0: "metric_path": null, + 0: "arg_mappings": null, + 0: "metric": "throughput", + 0: "model_info": null, + 0: "results_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_results", + 0: "exps_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_exps", + 0: "overwrite": true, + 0: "fast": true, + 0: "start_profile_step": 3, + 0: "end_profile_step": 5, + 0: "tuner_type": "gridsearch", + 0: "tuner_early_stopping": 5, + 0: "tuner_num_trials": 50, + 0: "model_info_path": null, + 0: "mp_size": 1, + 0: "max_train_batch_size": null, + 0: "min_train_batch_size": 1, + 0: "max_train_micro_batch_size_per_gpu": 1.024000e+03, + 0: "min_train_micro_batch_size_per_gpu": 1, + 0: "num_tuning_micro_batch_sizes": 3 + 0: } + 0: [2023-05-08 11:22:52,448] [INFO] [config.py:1011:print] bfloat16_enabled ............. True + 0: [2023-05-08 11:22:52,448] [INFO] [config.py:1011:print] checkpoint_parallel_write_pipeline False + 0: [2023-05-08 11:22:52,448] [INFO] [config.py:1011:print] checkpoint_tag_validation_enabled True + 0: [2023-05-08 11:22:52,448] [INFO] [config.py:1011:print] checkpoint_tag_validation_fail False + 0: [2023-05-08 11:22:52,448] [INFO] [config.py:1011:print] comms_config ................. + 0: [2023-05-08 11:22:52,448] [INFO] [config.py:1011:print] communication_data_type ...... None + 0: [2023-05-08 11:22:52,448] [INFO] [config.py:1011:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_pa + 0: rameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} + 0: [2023-05-08 11:22:52,448] [INFO] [config.py:1011:print] curriculum_enabled ........... False + 0: [2023-05-08 11:22:52,448] [INFO] [config.py:1011:print] curriculum_params ............ False + 0: [2023-05-08 11:22:52,448] [INFO] [config.py:1011:print] dataloader_drop_last ......... False + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] disable_allgather ............ False + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] dump_state ................... False + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] dynamic_loss_scale_args ...... None + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] eigenvalue_enabled ........... False + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] eigenvalue_gas_boundary_resolution 1 + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] eigenvalue_layer_name ........ bert.encoder.layer + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] eigenvalue_layer_num ......... 0 + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] eigenvalue_max_iter .......... 100 + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] eigenvalue_stability ......... 1e-06 + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] eigenvalue_tol ............... 0.01 + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] eigenvalue_verbose ........... False + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] elasticity_enabled ........... False + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] flops_profiler_config ........ { + 0: "enabled": false, + 0: "profile_step": 1, + 0: "module_depth": -1, + 0: "top_modules": 1, + 0: "detailed": true, + 0: "output_file": null + 0: } + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] fp16_auto_cast ............... None + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] fp16_enabled ................. False + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] fp16_master_weights_and_gradients False + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] global_rank .................. 0 + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] gradient_accumulation_steps .. 4 + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] gradient_clipping ............ 1.0 + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] gradient_predivide_factor .... 1.0 + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] initial_dynamic_scale ........ 1 + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] load_universal_checkpoint .... False + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] loss_scale ................... 1.0 + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] memory_breakdown ............. False + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] monitor_config ............... + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] nebula_config ................ { + 0: "enabled": false, + 0: "persistent_storage_path": null, + 0: "persistent_time_interval": 100, + 0: "num_of_version_in_retention": 2, + 0: "enable_nebula_load": true, + 0: "load_path": null + 0: } + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] optimizer_legacy_fusion ...... False + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] optimizer_name ............... None + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] optimizer_params ............. None + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] pld_enabled .................. False + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] pld_params ................... False + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] prescale_gradients ........... False + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] scheduler_name ............... None + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] scheduler_params ............. None + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] sparse_attention ............. None + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] sparse_gradients_enabled ..... False + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] steps_per_print .............. 2000 + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] train_batch_size ............. 1024 + 0: [2023-05-08 11:22:52,449] [INFO] [config.py:1011:print] train_micro_batch_size_per_gpu 2 + 0: [2023-05-08 11:22:52,450] [INFO] [config.py:1011:print] use_node_local_storage ....... False + 0: [2023-05-08 11:22:52,450] [INFO] [config.py:1011:print] wall_clock_breakdown ......... False + 0: [2023-05-08 11:22:52,450] [INFO] [config.py:1011:print] world_size ................... 128 + 0: [2023-05-08 11:22:52,450] [INFO] [config.py:1011:print] zero_allow_untested_optimizer False + 0: [2023-05-08 11:22:52,450] [INFO] [config.py:1011:print] zero_config .................. stage=0 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False + 0: [2023-05-08 11:22:52,450] [INFO] [config.py:1011:print] zero_enabled ................. False + 0: [2023-05-08 11:22:52,450] [INFO] [config.py:1011:print] zero_optimization_stage ...... 0 + 0: [2023-05-08 11:22:52,450] [INFO] [config.py:996:print_user_config] json = { + 0: "train_micro_batch_size_per_gpu": 2, + 0: "train_batch_size": 1.024000e+03, + 0: "gradient_clipping": 1.0, + 0: "zero_optimization": { + 0: "stage": 0 + 0: }, + 0: "bf16": { + 0: "enabled": true + 0: }, + 0: "steps_per_print": 2.000000e+03, + 0: "wall_clock_breakdown": false + 0: } + 0: Time to load utils op: 0.0004355907440185547 seconds + 0: [2023-05-08 11:22:52,450] [INFO] [engine.py:87:__init__] CONFIG: micro_batches=4 micro_batch_size=2 + 0: [2023-05-08 11:22:52,474] [INFO] [engine.py:145:__init__] RANK=0 STAGE=0 LAYERS=41 [0, 41) STAGE_PARAMS=1407562240 (1407.562M) TOTAL_PARAMS=2815124480 (2815.124M) UNIQUE_PARAMS=2815124480 (2815.124M) + 0: [2023-05-08 11:22:52,474] [INFO] [engine.py:145:__init__] RANK=1 STAGE=0 LAYERS=41 [0, 41) STAGE_PARAMS=1407562240 (1407.562M) TOTAL_PARAMS=2815124480 (2815.124M) UNIQUE_PARAMS=2815124480 (2815.124M) +16: [2023-05-08 11:22:54,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +16: [2023-05-08 11:22:54,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +16: [2023-05-08 11:22:54,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +16: [2023-05-08 11:22:54,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +16: [2023-05-08 11:22:54,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +16: [2023-05-08 11:22:54,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +16: [2023-05-08 11:22:54,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +16: [2023-05-08 11:22:54,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +24: [2023-05-08 11:22:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +24: [2023-05-08 11:22:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +24: [2023-05-08 11:22:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +24: [2023-05-08 11:22:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +24: [2023-05-08 11:22:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +24: [2023-05-08 11:22:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +24: [2023-05-08 11:22:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +27: [2023-05-08 11:22:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +27: [2023-05-08 11:22:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +27: [2023-05-08 11:22:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +27: [2023-05-08 11:22:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +27: [2023-05-08 11:22:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +27: [2023-05-08 11:22:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +27: [2023-05-08 11:22:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 0: [2023-05-08 11:22:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 0: [2023-05-08 11:22:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 0: [2023-05-08 11:22:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 0: [2023-05-08 11:22:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 0: [2023-05-08 11:22:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 0: [2023-05-08 11:22:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 0: [2023-05-08 11:22:54,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +24: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +13: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +13: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +13: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +13: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +28: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +28: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +28: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +28: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +28: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +27: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +13: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +13: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +13: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +28: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +28: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +30: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +28: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +30: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +30: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +30: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +30: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +30: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +30: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 0: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:22:54,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +29: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +10: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +10: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +10: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +29: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +18: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +10: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +29: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +29: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +29: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +10: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +10: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +18: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +18: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +10: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +29: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +29: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +20: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +18: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +18: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +18: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 6: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +18: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +10: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 6: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 6: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +15: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 6: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 6: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +12: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 6: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +20: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +13: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +20: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +20: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +14: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 6: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +20: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +20: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +15: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +15: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +15: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +15: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +23: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +30: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +20: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +15: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +15: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +12: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +12: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +12: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +12: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +12: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +12: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +23: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +23: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 7: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +14: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +14: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +14: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +14: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +23: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +14: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +14: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +22: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +23: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +23: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +23: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 7: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 7: [2023-05-08 11:22:54,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +22: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +22: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +22: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +22: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +22: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 7: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 7: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 7: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 7: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +23: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +22: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +18: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 6: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +22: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +20: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +29: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +19: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +12: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +19: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +15: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +19: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +31: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +31: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +11: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +31: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +31: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +31: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +19: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +19: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +19: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +19: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +31: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +31: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +14: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +19: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +25: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +25: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +25: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 1: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 1: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +25: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +25: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +25: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +25: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 1: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 1: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 1: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +21: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +21: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +21: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 1: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 1: [2023-05-08 11:22:54,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +21: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +11: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +21: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +21: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +21: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 7: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 3: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 3: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 3: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 3: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +25: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 3: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 3: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 3: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 9: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +26: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +26: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +26: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +31: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +21: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 9: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 9: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +11: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +11: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 1: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +17: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +17: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +17: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +17: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +11: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +17: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +17: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +17: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 3: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 9: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +11: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +11: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +26: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +26: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +26: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +26: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 9: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 9: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 9: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +11: [2023-05-08 11:22:54,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +17: [2023-05-08 11:22:54,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +26: [2023-05-08 11:22:54,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 9: [2023-05-08 11:22:54,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +16: [2023-05-08 11:22:54,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +16: [2023-05-08 11:22:54,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +16: [2023-05-08 11:22:54,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +16: [2023-05-08 11:22:54,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +16: [2023-05-08 11:22:54,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +16: [2023-05-08 11:22:54,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +16: [2023-05-08 11:22:54,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +16: [2023-05-08 11:22:54,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +16: [2023-05-08 11:22:54,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +16: [2023-05-08 11:22:54,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +16: [2023-05-08 11:22:54,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +16: [2023-05-08 11:22:54,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +16: [2023-05-08 11:22:54,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +16: [2023-05-08 11:22:54,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +16: [2023-05-08 11:22:54,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +16: [2023-05-08 11:22:54,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +16: [2023-05-08 11:22:54,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +16: [2023-05-08 11:22:54,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +16: [2023-05-08 11:22:54,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +16: [2023-05-08 11:22:54,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +16: [2023-05-08 11:22:54,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +16: [2023-05-08 11:22:54,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +16: [2023-05-08 11:22:54,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +16: [2023-05-08 11:22:54,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +27: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +27: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +27: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +27: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +27: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +27: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +28: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +28: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +28: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +28: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +28: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +27: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +27: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +27: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +27: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +28: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +28: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +28: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +28: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +28: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +28: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +27: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +28: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +27: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +27: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +27: [2023-05-08 11:22:54,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +27: [2023-05-08 11:22:54,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +27: [2023-05-08 11:22:54,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 4: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:22:54,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:22:54,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +28: [2023-05-08 11:22:54,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +28: [2023-05-08 11:22:54,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +28: [2023-05-08 11:22:54,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +28: [2023-05-08 11:22:54,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 4: [2023-05-08 11:22:54,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 4: [2023-05-08 11:22:54,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 4: [2023-05-08 11:22:54,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 4: [2023-05-08 11:22:54,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +27: [2023-05-08 11:22:54,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +27: [2023-05-08 11:22:54,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +27: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +27: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +28: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +28: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +13: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +13: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +13: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +13: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +13: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +13: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +13: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +28: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +13: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +13: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +28: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +27: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +27: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 0: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +13: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 0: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +13: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +27: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +13: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +27: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 0: [2023-05-08 11:22:54,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +13: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 4: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 4: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +13: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 0: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +13: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +13: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 4: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +28: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 0: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +29: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +29: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +29: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 0: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 0: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 4: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +11: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +11: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +11: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +11: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +11: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +29: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +29: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +28: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +29: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +29: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +11: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +11: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +29: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +29: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +11: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +11: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +11: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +11: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +28: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +29: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +29: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +28: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +29: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +11: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +11: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +29: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +29: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +29: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 2: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +11: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +11: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +29: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 2: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +11: [2023-05-08 11:22:54,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +13: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +24: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +24: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +24: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +24: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +24: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +24: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +24: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +13: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +10: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +10: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +10: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +10: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +10: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 2: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +24: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +24: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +24: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +10: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +10: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +15: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +15: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +15: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +15: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +15: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +13: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +24: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +10: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +15: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +10: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +10: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +15: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +12: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +12: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +12: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +12: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +12: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +24: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +13: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +10: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +15: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +15: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +15: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +12: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +15: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +12: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +24: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +24: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +10: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +15: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +24: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 0: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +12: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +12: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +12: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +12: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +24: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +11: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +29: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +12: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +13: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 5: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 8: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 8: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 8: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 8: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 8: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +29: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +13: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 5: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +11: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +15: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +15: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +15: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +15: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 0: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 8: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +23: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +23: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +23: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +23: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +23: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 8: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +10: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +10: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +10: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +10: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 5: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +11: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +13: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 5: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +23: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +23: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 8: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +12: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +29: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +11: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +13: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +23: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +23: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:22:54,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +12: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +12: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +12: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +19: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +19: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +19: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +19: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +19: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +29: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +23: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +19: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +19: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +23: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +19: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +19: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +19: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +19: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +14: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +14: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +14: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +14: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +14: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +19: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +14: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +14: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +14: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +14: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +23: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +23: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 0: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +11: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 8: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 8: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 8: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +14: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +14: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 5: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 5: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 5: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +11: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +14: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +23: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +23: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 8: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +11: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +29: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +29: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 2: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +11: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +19: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +19: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +19: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +19: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +29: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 3: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 3: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 3: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 3: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +29: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +14: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +14: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +14: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 7: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 7: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 7: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 7: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 7: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 3: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 3: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +10: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 7: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 7: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +24: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +14: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 3: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 3: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 3: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +15: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 3: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +10: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 7: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 7: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 0: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +24: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 3: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +17: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +17: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +17: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +17: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +17: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 2: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 7: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 1: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +12: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +12: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +24: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +10: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +17: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 7: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +15: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +17: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +25: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +10: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +17: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +17: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +17: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +25: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +25: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 3: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 3: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 3: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 3: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 8: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +17: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +25: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +25: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +25: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +25: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +17: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 1: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 1: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +25: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +25: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 7: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 1: [2023-05-08 11:22:54,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +12: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +18: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +18: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +18: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +18: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +25: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +25: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 7: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 7: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +23: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +25: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 8: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +19: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +18: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +18: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +18: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +23: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +10: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +17: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +17: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +17: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +17: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +31: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +31: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +31: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +31: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +31: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 8: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +18: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +24: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +18: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +18: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +10: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +25: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +25: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +19: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +18: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +18: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +23: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +31: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +31: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 1: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +10: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +25: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +25: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +24: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +10: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 1: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 8: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 6: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 6: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 6: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 6: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 6: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +19: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +31: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +31: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +31: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +19: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +14: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +31: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 6: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 6: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +15: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 8: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 6: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +12: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 8: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +18: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +18: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +31: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +31: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 6: [2023-05-08 11:22:54,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +20: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +20: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +20: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +20: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +20: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 3: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +12: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +31: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +19: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +20: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +20: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 6: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +20: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +20: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +23: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +20: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +30: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +30: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +30: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +20: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +30: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +30: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +30: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +30: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 8: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +14: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +19: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +20: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +30: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +30: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 3: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +30: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 9: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 9: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 9: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 9: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 9: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +14: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +17: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +23: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +30: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 9: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +14: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 5: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +23: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +21: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +21: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +21: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 9: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +30: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +20: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +20: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 6: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 6: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 6: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 9: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 9: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +14: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +23: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +21: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +17: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +21: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +21: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 9: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +20: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +20: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +30: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +30: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +30: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +21: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 9: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +17: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 3: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 3: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +26: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +26: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +26: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +26: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +21: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +21: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 9: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +17: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +21: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +22: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +22: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +22: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +22: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +22: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +26: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +26: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +21: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +21: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +14: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +14: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +22: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +26: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +25: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +26: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +26: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +26: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +22: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +22: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:22:54,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +25: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +22: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +25: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 9: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 9: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +22: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +21: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +21: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 9: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... + 9: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +17: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 5: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +21: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +21: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +22: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +26: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt... +17: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +25: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +17: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +14: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +17: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +22: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +26: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +26: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +26: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +26: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +22: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +22: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt... +20: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +20: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +20: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +31: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +31: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +20: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 7: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 7: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +31: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +25: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 7: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +30: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 7: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +21: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +25: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 1: [2023-05-08 11:22:54,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +25: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +20: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +20: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +25: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +20: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +21: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 9: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +18: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +20: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 1: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +21: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +31: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 6: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +22: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +18: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +18: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 6: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 9: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +22: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +30: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +21: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +21: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +21: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +21: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 7: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 1: [2023-05-08 11:22:54,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 1: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +26: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +26: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. +22: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +21: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt. + 6: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:22:54,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:22:54,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:22:54,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:22:54,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:22:54,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:22:54,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:22:54,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +16: [2023-05-08 11:22:54,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +16: [2023-05-08 11:22:54,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +16: [2023-05-08 11:22:54,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +16: [2023-05-08 11:22:54,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +16: [2023-05-08 11:22:54,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +16: [2023-05-08 11:22:54,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +16: [2023-05-08 11:22:54,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +27: [2023-05-08 11:22:54,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +27: [2023-05-08 11:22:54,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +16: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +27: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +27: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 4: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 2: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 2: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +11: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 4: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +13: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +13: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +13: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 2: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 4: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +23: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +12: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +12: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +11: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +11: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +24: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +28: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +29: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +27: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +27: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 2: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +12: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +13: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +19: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +24: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +23: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 8: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 8: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +19: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +28: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +28: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +29: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +29: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +27: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 4: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +19: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +10: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +10: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 8: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +12: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +11: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +10: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +17: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +17: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +23: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +24: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +29: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +17: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 4: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +28: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +27: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 2: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 2: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 8: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +11: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +13: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +13: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 4: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 2: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +10: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 4: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +13: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +23: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +11: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +11: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +20: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +17: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 2: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +13: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:22:54,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +23: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +20: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +20: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +29: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 8: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +28: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +29: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +29: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +31: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 4: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 8: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +11: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +10: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +21: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 3: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +24: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +17: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +17: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +25: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 8: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +22: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +20: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +28: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +28: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +10: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +10: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +29: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +30: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +21: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 3: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 3: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 8: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +17: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +21: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +22: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +24: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +25: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +28: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +10: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +15: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +21: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +22: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +17: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 3: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +20: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +20: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 0: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 0: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +20: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 0: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +25: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 9: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +15: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +21: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +20: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +21: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +21: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +25: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +14: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 0: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +22: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 5: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +25: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 9: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +14: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +14: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +30: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +25: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +21: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +14: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 5: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 5: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +25: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 6: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 1: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 1: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 1: [2023-05-08 11:22:54,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 9: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 7: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 5: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +25: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +26: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +26: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 7: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 1: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +18: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +26: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 7: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +18: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 0: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 1: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 0: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 0: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 9: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 5: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. +26: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt. + 9: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 0: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +14: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +14: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +14: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 1: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 1: [2023-05-08 11:22:54,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +14: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 5: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 5: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 5: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 1: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:22:54,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:22:54,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:22:54,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 8: [2023-05-08 11:22:55,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 8: [2023-05-08 11:22:55,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 8: [2023-05-08 11:22:55,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 8: [2023-05-08 11:22:55,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:22:55,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:22:55,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:22:55,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:22:55,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:22:55,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:22:55,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:22:55,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:22:55,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:22:55,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:22:55,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:22:55,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:22:55,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:22:55,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:22:55,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:22:55,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:22:55,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:22:55,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:22:55,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +22: [2023-05-08 11:22:55,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +21: [2023-05-08 11:22:55,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:22:55,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 3: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 3: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +21: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +17: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:22:55,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +23: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +23: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +23: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +23: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +23: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +23: [2023-05-08 11:22:55,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:22:55,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +23: [2023-05-08 11:22:55,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:22:55,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +21: [2023-05-08 11:22:55,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +17: [2023-05-08 11:22:55,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:22:55,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:22:55,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:22:55,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:22:55,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:22:55,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:22:55,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:22:55,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:22:55,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +13: [2023-05-08 11:22:55,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:22:55,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:22:55,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:22:55,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +17: [2023-05-08 11:22:55,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +17: [2023-05-08 11:22:55,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:22:55,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:22:55,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +23: [2023-05-08 11:22:55,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:22:55,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +17: [2023-05-08 11:22:55,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:22:55,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:22:55,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +17: [2023-05-08 11:22:55,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 3: [2023-05-08 11:22:55,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +23: [2023-05-08 11:22:55,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +23: [2023-05-08 11:22:55,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 3: [2023-05-08 11:22:55,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 3: [2023-05-08 11:22:55,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +13: [2023-05-08 11:22:55,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +13: [2023-05-08 11:22:55,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +17: [2023-05-08 11:22:55,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +13: [2023-05-08 11:22:55,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +17: [2023-05-08 11:22:55,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +13: [2023-05-08 11:22:55,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 3: [2023-05-08 11:22:55,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +13: [2023-05-08 11:22:55,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +17: [2023-05-08 11:22:55,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +17: [2023-05-08 11:22:55,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +13: [2023-05-08 11:22:55,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +13: [2023-05-08 11:22:55,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +13: [2023-05-08 11:22:55,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +30: [2023-05-08 11:22:55,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +30: [2023-05-08 11:22:55,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +30: [2023-05-08 11:22:55,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +30: [2023-05-08 11:22:55,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +30: [2023-05-08 11:22:55,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +30: [2023-05-08 11:22:55,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +30: [2023-05-08 11:22:55,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +30: [2023-05-08 11:22:55,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +30: [2023-05-08 11:22:55,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 8: [2023-05-08 11:22:55,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +30: [2023-05-08 11:22:55,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:22:55,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 8: [2023-05-08 11:22:55,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +30: [2023-05-08 11:22:55,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +30: [2023-05-08 11:22:55,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:22:55,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:22:55,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:22:55,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +30: [2023-05-08 11:22:55,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +19: [2023-05-08 11:22:55,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:22:55,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:22:55,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:22:55,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:22:55,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:22:55,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +30: [2023-05-08 11:22:55,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:22:55,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +19: [2023-05-08 11:22:55,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +19: [2023-05-08 11:22:55,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 8: [2023-05-08 11:22:55,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +19: [2023-05-08 11:22:55,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +19: [2023-05-08 11:22:55,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:22:55,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:22:55,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:22:55,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +19: [2023-05-08 11:22:55,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +21: [2023-05-08 11:22:55,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +22: [2023-05-08 11:22:55,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:22:55,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:22:55,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:22:55,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:22:55,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:22:55,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:22:55,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +23: [2023-05-08 11:22:55,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +23: [2023-05-08 11:22:55,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:22:55,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:22:55,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:22:55,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:22:55,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +23: [2023-05-08 11:22:55,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +23: [2023-05-08 11:22:55,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:22:55,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:22:55,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:22:55,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:22:55,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:22:55,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:22:55,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +14: [2023-05-08 11:22:55,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +14: [2023-05-08 11:22:55,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:22:55,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:22:55,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:22:55,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +14: [2023-05-08 11:22:55,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +14: [2023-05-08 11:22:55,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +14: [2023-05-08 11:22:55,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +14: [2023-05-08 11:22:55,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:22:55,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:22:55,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:22:55,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:22:55,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:22:55,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:22:55,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:22:55,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:22:55,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:22:55,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 3: [2023-05-08 11:22:55,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +22: [2023-05-08 11:22:55,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +22: [2023-05-08 11:22:55,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 3: [2023-05-08 11:22:55,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 3: [2023-05-08 11:22:55,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:22:55,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:22:55,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 3: [2023-05-08 11:22:55,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:22:55,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:22:55,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:22:55,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:22:55,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:22:55,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +16: [2023-05-08 11:22:55,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +16: [2023-05-08 11:22:55,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +16: [2023-05-08 11:22:55,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +13: [2023-05-08 11:22:55,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:22:55,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:22:55,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:22:55,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:22:55,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:22:55,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +14: [2023-05-08 11:22:55,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +16: [2023-05-08 11:22:55,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +14: [2023-05-08 11:22:55,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +16: [2023-05-08 11:22:55,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +16: [2023-05-08 11:22:55,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +17: [2023-05-08 11:22:55,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:22:55,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:22:55,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:22:55,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +13: [2023-05-08 11:22:55,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:22:55,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +30: [2023-05-08 11:22:55,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +30: [2023-05-08 11:22:55,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +14: [2023-05-08 11:22:55,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +14: [2023-05-08 11:22:55,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +14: [2023-05-08 11:22:55,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:22:55,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:22:55,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:22:55,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:22:55,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:22:55,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:22:55,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:22:55,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:22:55,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:22:55,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:22:55,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:22:55,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +18: [2023-05-08 11:22:55,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:22:55,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +18: [2023-05-08 11:22:55,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +18: [2023-05-08 11:22:55,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +18: [2023-05-08 11:22:55,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:22:55,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +18: [2023-05-08 11:22:55,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:22:55,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 9: [2023-05-08 11:22:55,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 9: [2023-05-08 11:22:55,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 9: [2023-05-08 11:22:55,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 9: [2023-05-08 11:22:55,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 9: [2023-05-08 11:22:55,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 9: [2023-05-08 11:22:55,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 9: [2023-05-08 11:22:55,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:22:55,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:22:55,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:22:55,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 6: [2023-05-08 11:22:55,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 6: [2023-05-08 11:22:55,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 6: [2023-05-08 11:22:55,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 6: [2023-05-08 11:22:55,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 6: [2023-05-08 11:22:55,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 6: [2023-05-08 11:22:55,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 6: [2023-05-08 11:22:55,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 6: [2023-05-08 11:22:55,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:22:55,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:22:55,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:22:55,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:22:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:22:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:22:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:22:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:22:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:22:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:22:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:22:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:22:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:22:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:22:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:22:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:22:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:22:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:22:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +11: [2023-05-08 11:22:55,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:22:55,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:22:55,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:22:55,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:22:55,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:22:55,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:22:55,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +11: [2023-05-08 11:22:55,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:22:55,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +11: [2023-05-08 11:22:55,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +26: [2023-05-08 11:22:55,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +26: [2023-05-08 11:22:55,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +26: [2023-05-08 11:22:55,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +26: [2023-05-08 11:22:55,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +26: [2023-05-08 11:22:55,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +26: [2023-05-08 11:22:55,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +26: [2023-05-08 11:22:55,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +26: [2023-05-08 11:22:55,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +11: [2023-05-08 11:22:55,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:22:55,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +11: [2023-05-08 11:22:55,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +11: [2023-05-08 11:22:55,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +11: [2023-05-08 11:22:55,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:22:55,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +29: [2023-05-08 11:22:55,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +10: [2023-05-08 11:22:55,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +20: [2023-05-08 11:22:55,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +20: [2023-05-08 11:22:55,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +10: [2023-05-08 11:22:55,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:22:55,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +20: [2023-05-08 11:22:55,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +10: [2023-05-08 11:22:55,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +10: [2023-05-08 11:22:55,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +11: [2023-05-08 11:22:55,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +11: [2023-05-08 11:22:55,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 2: [2023-05-08 11:22:55,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 2: [2023-05-08 11:22:55,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +11: [2023-05-08 11:22:55,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 2: [2023-05-08 11:22:55,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 2: [2023-05-08 11:22:55,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +20: [2023-05-08 11:22:55,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +11: [2023-05-08 11:22:55,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 2: [2023-05-08 11:22:55,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +20: [2023-05-08 11:22:55,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 2: [2023-05-08 11:22:55,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +29: [2023-05-08 11:22:55,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 6: [2023-05-08 11:22:55,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +20: [2023-05-08 11:22:55,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 2: [2023-05-08 11:22:55,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +29: [2023-05-08 11:22:55,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:22:55,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 9: [2023-05-08 11:22:55,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 9: [2023-05-08 11:22:55,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +29: [2023-05-08 11:22:55,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +10: [2023-05-08 11:22:55,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +29: [2023-05-08 11:22:55,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 9: [2023-05-08 11:22:55,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:22:55,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +29: [2023-05-08 11:22:55,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +29: [2023-05-08 11:22:55,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +10: [2023-05-08 11:22:55,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 6: [2023-05-08 11:22:55,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:22:55,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 2: [2023-05-08 11:22:55,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 9: [2023-05-08 11:22:55,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +10: [2023-05-08 11:22:55,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 7: [2023-05-08 11:22:55,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 9: [2023-05-08 11:22:55,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +10: [2023-05-08 11:22:55,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 6: [2023-05-08 11:22:55,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:22:55,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:22:55,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +20: [2023-05-08 11:22:55,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:22:55,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 7: [2023-05-08 11:22:55,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:22:55,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 9: [2023-05-08 11:22:55,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:22:55,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:22:55,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +24: [2023-05-08 11:22:55,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +24: [2023-05-08 11:22:55,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +24: [2023-05-08 11:22:55,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +24: [2023-05-08 11:22:55,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +24: [2023-05-08 11:22:55,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +24: [2023-05-08 11:22:55,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 6: [2023-05-08 11:22:55,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 7: [2023-05-08 11:22:55,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 7: [2023-05-08 11:22:55,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:22:55,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +11: [2023-05-08 11:22:55,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +11: [2023-05-08 11:22:55,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 6: [2023-05-08 11:22:55,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 7: [2023-05-08 11:22:55,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +11: [2023-05-08 11:22:55,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:22:55,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 7: [2023-05-08 11:22:55,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:22:55,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +11: [2023-05-08 11:22:55,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +11: [2023-05-08 11:22:55,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +26: [2023-05-08 11:22:55,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:22:55,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:22:55,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:22:55,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:22:55,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +24: [2023-05-08 11:22:55,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:22:55,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +24: [2023-05-08 11:22:55,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +24: [2023-05-08 11:22:55,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +30: [2023-05-08 11:22:55,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:22:55,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:22:55,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +24: [2023-05-08 11:22:55,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:22:55,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:22:55,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:22:55,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +19: [2023-05-08 11:22:55,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +19: [2023-05-08 11:22:55,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:22:55,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:22:55,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:22:55,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +28: [2023-05-08 11:22:55,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +28: [2023-05-08 11:22:55,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +28: [2023-05-08 11:22:55,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +28: [2023-05-08 11:22:55,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +28: [2023-05-08 11:22:55,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +28: [2023-05-08 11:22:55,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +28: [2023-05-08 11:22:55,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 4: [2023-05-08 11:22:55,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 4: [2023-05-08 11:22:55,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 4: [2023-05-08 11:22:55,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 4: [2023-05-08 11:22:55,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 4: [2023-05-08 11:22:55,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 4: [2023-05-08 11:22:55,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 4: [2023-05-08 11:22:55,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +28: [2023-05-08 11:22:55,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +14: [2023-05-08 11:22:55,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:22:55,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:22:55,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:22:55,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +28: [2023-05-08 11:22:55,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +28: [2023-05-08 11:22:55,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +28: [2023-05-08 11:22:55,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +28: [2023-05-08 11:22:55,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 4: [2023-05-08 11:22:55,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +28: [2023-05-08 11:22:55,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 4: [2023-05-08 11:22:55,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +28: [2023-05-08 11:22:55,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +28: [2023-05-08 11:22:55,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 4: [2023-05-08 11:22:55,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +16: [2023-05-08 11:22:55,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +14: [2023-05-08 11:22:55,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:22:55,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:22:55,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:22:55,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +14: [2023-05-08 11:22:55,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +14: [2023-05-08 11:22:55,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:22:55,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:22:55,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:22:55,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:22:55,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:22:55,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +20: [2023-05-08 11:22:55,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:22:55,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:22:55,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +11: [2023-05-08 11:22:55,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +11: [2023-05-08 11:22:55,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:22:55,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:22:55,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:22:55,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:22:55,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:22:55,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:22:55,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:22:55,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:22:55,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:22:55,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:22:55,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:22:55,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:22:55,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:22:55,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:22:55,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:22:55,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:22:55,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:22:55,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:22:55,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:22:55,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:22:55,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:22:55,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:22:55,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:22:55,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:22:55,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:22:55,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:22:55,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:22:55,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:22:55,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +11: [2023-05-08 11:22:55,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 5: [2023-05-08 11:22:55,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +11: [2023-05-08 11:22:55,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:22:55,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:22:55,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 9: [2023-05-08 11:22:55,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 9: [2023-05-08 11:22:55,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 5: [2023-05-08 11:22:55,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 5: [2023-05-08 11:22:55,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 5: [2023-05-08 11:22:55,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 5: [2023-05-08 11:22:55,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 5: [2023-05-08 11:22:55,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:22:55,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:22:55,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:22:55,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:22:55,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +25: [2023-05-08 11:22:55,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:22:55,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:22:55,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 5: [2023-05-08 11:22:55,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:22:55,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +15: [2023-05-08 11:22:55,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +14: [2023-05-08 11:22:55,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 9: [2023-05-08 11:22:55,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:22:55,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 1: [2023-05-08 11:22:55,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 1: [2023-05-08 11:22:55,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:22:55,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 6: [2023-05-08 11:22:55,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:22:55,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:22:55,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:22:55,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:22:55,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:22:55,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 1: [2023-05-08 11:22:55,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:22:55,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:22:55,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 1: [2023-05-08 11:22:55,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 1: [2023-05-08 11:22:55,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 1: [2023-05-08 11:22:55,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:22:55,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 0: [2023-05-08 11:22:55,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +24: [2023-05-08 11:22:55,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:22:55,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:22:55,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:22:55,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:22:55,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:22:55,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:22:55,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:22:55,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:22:55,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:22:55,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 6: [2023-05-08 11:22:55,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 6: [2023-05-08 11:22:55,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 1: [2023-05-08 11:22:55,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +24: [2023-05-08 11:22:55,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:22:55,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:22:55,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +10: [2023-05-08 11:22:55,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:22:55,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:22:55,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:22:55,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 7: [2023-05-08 11:22:55,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:22:55,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 7: [2023-05-08 11:22:55,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 9: [2023-05-08 11:22:55,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +24: [2023-05-08 11:22:55,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:22:55,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:22:55,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 5: [2023-05-08 11:22:55,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +25: [2023-05-08 11:22:55,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +27: [2023-05-08 11:22:55,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 5: [2023-05-08 11:22:55,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 5: [2023-05-08 11:22:55,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 5: [2023-05-08 11:22:55,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +10: [2023-05-08 11:22:55,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:22:55,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +25: [2023-05-08 11:22:55,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +27: [2023-05-08 11:22:55,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 5: [2023-05-08 11:22:55,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +27: [2023-05-08 11:22:55,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +27: [2023-05-08 11:22:55,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:22:55,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +11: [2023-05-08 11:22:55,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:22:55,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +26: [2023-05-08 11:22:55,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +26: [2023-05-08 11:22:55,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +11: [2023-05-08 11:22:55,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:22:55,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:22:55,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:22:55,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:22:55,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +14: [2023-05-08 11:22:55,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:22:55,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:22:55,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +25: [2023-05-08 11:22:55,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 0: [2023-05-08 11:22:55,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:22:55,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:22:55,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:22:55,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:22:55,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 5: [2023-05-08 11:22:55,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +25: [2023-05-08 11:22:55,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +28: [2023-05-08 11:22:55,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:22:55,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:22:55,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:22:55,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:22:55,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 2: [2023-05-08 11:22:55,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:22:55,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +25: [2023-05-08 11:22:55,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +25: [2023-05-08 11:22:55,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +27: [2023-05-08 11:22:55,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +31: [2023-05-08 11:22:55,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 1: [2023-05-08 11:22:55,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 5: [2023-05-08 11:22:55,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 0: [2023-05-08 11:22:55,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 2: [2023-05-08 11:22:55,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +31: [2023-05-08 11:22:55,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +25: [2023-05-08 11:22:55,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +29: [2023-05-08 11:22:55,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:22:55,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 2: [2023-05-08 11:22:55,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +12: [2023-05-08 11:22:55,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +12: [2023-05-08 11:22:55,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 2: [2023-05-08 11:22:55,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +12: [2023-05-08 11:22:55,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:22:55,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +12: [2023-05-08 11:22:55,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 0: [2023-05-08 11:22:55,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:22:55,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +28: [2023-05-08 11:22:55,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +11: [2023-05-08 11:22:55,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +11: [2023-05-08 11:22:55,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:22:55,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +20: [2023-05-08 11:22:55,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +20: [2023-05-08 11:22:55,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 0: [2023-05-08 11:22:55,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 0: [2023-05-08 11:22:55,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 1: [2023-05-08 11:22:55,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 0: [2023-05-08 11:22:55,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 1: [2023-05-08 11:22:55,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... + 0: [2023-05-08 11:22:55,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... + 7: [2023-05-08 11:22:55,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 0: [2023-05-08 11:22:55,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +24: [2023-05-08 11:22:55,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:22:55,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt... +28: [2023-05-08 11:22:55,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 1: [2023-05-08 11:22:55,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt... +28: [2023-05-08 11:22:55,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 4: [2023-05-08 11:22:55,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:22:55,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:22:55,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 9: [2023-05-08 11:22:55,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 9: [2023-05-08 11:22:55,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 4: [2023-05-08 11:22:55,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +24: [2023-05-08 11:22:55,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:22:55,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:22:55,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 6: [2023-05-08 11:22:55,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:22:55,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 9: [2023-05-08 11:22:55,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 7: [2023-05-08 11:22:55,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:22:55,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 7: [2023-05-08 11:22:55,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:22:55,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:22:55,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:22:55,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 9: [2023-05-08 11:22:55,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:22:55,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:22:55,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:22:55,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:22:55,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:22:55,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:22:55,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:22:55,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:22:55,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:22:55,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 7: [2023-05-08 11:22:55,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +15: [2023-05-08 11:22:55,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:22:55,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:22:55,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:22:55,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 5: [2023-05-08 11:22:55,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:22:55,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:22:55,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:22:55,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:22:55,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:22:55,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 5: [2023-05-08 11:22:55,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 5: [2023-05-08 11:22:55,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 5: [2023-05-08 11:22:55,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:22:55,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:22:55,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:22:55,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +15: [2023-05-08 11:22:55,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:22:55,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:22:55,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:22:55,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +25: [2023-05-08 11:22:55,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:22:55,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:22:55,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:22:55,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 5: [2023-05-08 11:22:55,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +31: [2023-05-08 11:22:55,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:22:55,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:22:55,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 1: [2023-05-08 11:22:55,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 1: [2023-05-08 11:22:55,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 1: [2023-05-08 11:22:55,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:22:55,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +12: [2023-05-08 11:22:55,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:22:55,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:22:55,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 0: [2023-05-08 11:22:55,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:22:55,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +25: [2023-05-08 11:22:55,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:22:55,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:22:55,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 0: [2023-05-08 11:22:55,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:22:55,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:22:55,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:22:55,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +25: [2023-05-08 11:22:55,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +31: [2023-05-08 11:22:55,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:22:55,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:22:55,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +12: [2023-05-08 11:22:55,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +12: [2023-05-08 11:22:55,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:22:55,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:22:55,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +31: [2023-05-08 11:22:55,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +31: [2023-05-08 11:22:55,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:22:55,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:22:55,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +12: [2023-05-08 11:22:55,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +31: [2023-05-08 11:22:55,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 0: [2023-05-08 11:22:55,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 0: [2023-05-08 11:22:55,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 0: [2023-05-08 11:22:55,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 3: [2023-05-08 11:22:55,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 3: [2023-05-08 11:22:55,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 3: [2023-05-08 11:22:55,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 3: [2023-05-08 11:22:55,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:22:55,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:22:55,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:22:55,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:22:55,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:22:55,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:22:55,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:22:55,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:22:55,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:22:55,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +13: [2023-05-08 11:22:55,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +13: [2023-05-08 11:22:55,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +13: [2023-05-08 11:22:55,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +13: [2023-05-08 11:22:55,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:22:55,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:22:55,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:22:55,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:22:55,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +23: [2023-05-08 11:22:55,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +23: [2023-05-08 11:22:55,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +23: [2023-05-08 11:22:55,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +23: [2023-05-08 11:22:55,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:22:55,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:22:55,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:22:55,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 3: [2023-05-08 11:22:55,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:22:55,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:22:55,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:22:55,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:22:55,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:22:55,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:22:55,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:22:55,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +13: [2023-05-08 11:22:55,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +13: [2023-05-08 11:22:55,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +13: [2023-05-08 11:22:55,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:22:55,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +23: [2023-05-08 11:22:55,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +23: [2023-05-08 11:22:55,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +23: [2023-05-08 11:22:55,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +23: [2023-05-08 11:22:55,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:22:55,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:22:55,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:22:55,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:22:55,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:22:55,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +30: [2023-05-08 11:22:55,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:22:55,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:22:55,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:22:55,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:22:55,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:22:55,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:22:55,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:22:55,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +30: [2023-05-08 11:22:55,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:22:55,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:22:55,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +23: [2023-05-08 11:22:55,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +23: [2023-05-08 11:22:55,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +23: [2023-05-08 11:22:55,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:22:55,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +22: [2023-05-08 11:22:55,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +22: [2023-05-08 11:22:55,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +22: [2023-05-08 11:22:55,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:22:55,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:22:55,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:22:55,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:22:55,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +22: [2023-05-08 11:22:55,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +22: [2023-05-08 11:22:55,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:22:55,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +22: [2023-05-08 11:22:55,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:22:55,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:22:55,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:22:55,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +22: [2023-05-08 11:22:55,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:22:55,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:22:55,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:22:55,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:22:55,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +22: [2023-05-08 11:22:55,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +22: [2023-05-08 11:22:55,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +22: [2023-05-08 11:22:55,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:22:55,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:22:55,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +22: [2023-05-08 11:22:55,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:22:55,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:22:55,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:22:55,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +16: [2023-05-08 11:22:55,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:22:55,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:22:55,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:22:55,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:22:55,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:22:55,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:22:55,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:22:55,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:22:55,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +30: [2023-05-08 11:22:55,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:22:55,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +30: [2023-05-08 11:22:55,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:22:55,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:22:55,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:22:55,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +30: [2023-05-08 11:22:55,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:22:55,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:22:55,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:22:55,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +19: [2023-05-08 11:22:55,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:22:55,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:22:55,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:22:55,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:22:55,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +19: [2023-05-08 11:22:55,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +19: [2023-05-08 11:22:55,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:22:55,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:22:55,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 8: [2023-05-08 11:22:55,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 8: [2023-05-08 11:22:55,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +16: [2023-05-08 11:22:55,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:22:55,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:22:55,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +22: [2023-05-08 11:22:55,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +22: [2023-05-08 11:22:55,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +22: [2023-05-08 11:22:55,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +22: [2023-05-08 11:22:55,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +16: [2023-05-08 11:22:55,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:22:55,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +13: [2023-05-08 11:22:55,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +13: [2023-05-08 11:22:55,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +13: [2023-05-08 11:22:55,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +30: [2023-05-08 11:22:55,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +30: [2023-05-08 11:22:55,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +30: [2023-05-08 11:22:55,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +30: [2023-05-08 11:22:55,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 8: [2023-05-08 11:22:55,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:22:55,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:22:55,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +16: [2023-05-08 11:22:55,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +18: [2023-05-08 11:22:55,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:22:55,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:22:55,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:22:55,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +18: [2023-05-08 11:22:55,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +18: [2023-05-08 11:22:55,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +18: [2023-05-08 11:22:55,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +18: [2023-05-08 11:22:55,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +11: [2023-05-08 11:22:55,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:22:55,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:22:55,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:22:55,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:22:55,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:22:55,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +11: [2023-05-08 11:22:55,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:22:55,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +11: [2023-05-08 11:22:55,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:22:55,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:22:55,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +11: [2023-05-08 11:22:55,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:22:55,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +19: [2023-05-08 11:22:55,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:22:55,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +22: [2023-05-08 11:22:55,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +19: [2023-05-08 11:22:55,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:22:55,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +13: [2023-05-08 11:22:55,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 2: [2023-05-08 11:22:55,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +13: [2023-05-08 11:22:55,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:22:55,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +22: [2023-05-08 11:22:55,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +22: [2023-05-08 11:22:55,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +22: [2023-05-08 11:22:55,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 2: [2023-05-08 11:22:55,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:22:55,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 2: [2023-05-08 11:22:55,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 2: [2023-05-08 11:22:55,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 2: [2023-05-08 11:22:55,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:22:55,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:22:55,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:22:55,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +19: [2023-05-08 11:22:55,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +19: [2023-05-08 11:22:55,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +19: [2023-05-08 11:22:55,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +19: [2023-05-08 11:22:55,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +24: [2023-05-08 11:22:55,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +24: [2023-05-08 11:22:55,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +24: [2023-05-08 11:22:55,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +24: [2023-05-08 11:22:55,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:22:55,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +17: [2023-05-08 11:22:55,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +17: [2023-05-08 11:22:55,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +17: [2023-05-08 11:22:55,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +21: [2023-05-08 11:22:55,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +21: [2023-05-08 11:22:55,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +24: [2023-05-08 11:22:55,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:22:55,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:22:55,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:22:55,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:22:55,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 7: [2023-05-08 11:22:55,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +23: [2023-05-08 11:22:55,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:22:55,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:22:55,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:22:55,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:22:55,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +14: [2023-05-08 11:22:55,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 7: [2023-05-08 11:22:55,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:22:55,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 7: [2023-05-08 11:22:55,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:22:55,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:22:55,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:22:55,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +19: [2023-05-08 11:22:55,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +14: [2023-05-08 11:22:55,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:22:55,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +19: [2023-05-08 11:22:55,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +14: [2023-05-08 11:22:55,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +18: [2023-05-08 11:22:55,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +14: [2023-05-08 11:22:55,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 3: [2023-05-08 11:22:55,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 3: [2023-05-08 11:22:55,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +18: [2023-05-08 11:22:55,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 8: [2023-05-08 11:22:55,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 8: [2023-05-08 11:22:55,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:22:55,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:22:55,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:22:55,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:22:55,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 8: [2023-05-08 11:22:55,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +11: [2023-05-08 11:22:55,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:22:55,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +19: [2023-05-08 11:22:55,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:22:55,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:22:55,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 2: [2023-05-08 11:22:55,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +22: [2023-05-08 11:22:55,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:22:55,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:22:55,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:22:55,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +19: [2023-05-08 11:22:55,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +19: [2023-05-08 11:22:55,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:22:55,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +22: [2023-05-08 11:22:55,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +21: [2023-05-08 11:22:55,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:22:55,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:22:55,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:22:55,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +22: [2023-05-08 11:22:55,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:22:55,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:22:55,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 2: [2023-05-08 11:22:55,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:22:55,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:22:55,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +22: [2023-05-08 11:22:55,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +22: [2023-05-08 11:22:55,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +22: [2023-05-08 11:22:55,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:22:55,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:22:55,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +13: [2023-05-08 11:22:55,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +13: [2023-05-08 11:22:55,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:22:55,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:22:55,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 3: [2023-05-08 11:22:55,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +22: [2023-05-08 11:22:55,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 3: [2023-05-08 11:22:55,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:22:55,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:22:55,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:22:55,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:22:55,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 7: [2023-05-08 11:22:55,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +13: [2023-05-08 11:22:55,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +24: [2023-05-08 11:22:55,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:22:55,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:22:55,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +30: [2023-05-08 11:22:55,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +30: [2023-05-08 11:22:55,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +11: [2023-05-08 11:22:55,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +11: [2023-05-08 11:22:55,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +30: [2023-05-08 11:22:55,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 3: [2023-05-08 11:22:55,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 7: [2023-05-08 11:22:55,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:22:55,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +24: [2023-05-08 11:22:55,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:22:55,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +14: [2023-05-08 11:22:55,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +14: [2023-05-08 11:22:55,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +14: [2023-05-08 11:22:55,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +23: [2023-05-08 11:22:55,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +24: [2023-05-08 11:22:55,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:22:55,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +23: [2023-05-08 11:22:55,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +23: [2023-05-08 11:22:55,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +23: [2023-05-08 11:22:55,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +23: [2023-05-08 11:22:55,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +23: [2023-05-08 11:22:55,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +14: [2023-05-08 11:22:55,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +23: [2023-05-08 11:22:55,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +23: [2023-05-08 11:22:55,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 2: [2023-05-08 11:22:55,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:22:55,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:22:55,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:22:55,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:22:55,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:22:55,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:22:55,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:22:55,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +20: [2023-05-08 11:22:55,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +20: [2023-05-08 11:22:55,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +24: [2023-05-08 11:22:55,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:22:55,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +20: [2023-05-08 11:22:55,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:22:55,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 7: [2023-05-08 11:22:55,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 7: [2023-05-08 11:22:55,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:22:55,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +28: [2023-05-08 11:22:55,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +28: [2023-05-08 11:22:55,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +24: [2023-05-08 11:22:55,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:22:55,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +28: [2023-05-08 11:22:55,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:22:55,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:22:55,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 7: [2023-05-08 11:22:55,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +14: [2023-05-08 11:22:55,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:22:55,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:22:55,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +14: [2023-05-08 11:22:55,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +14: [2023-05-08 11:22:55,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +19: [2023-05-08 11:22:55,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:22:55,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:22:55,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:22:55,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:22:55,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:22:55,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:22:55,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:22:55,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:22:55,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +29: [2023-05-08 11:22:55,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +29: [2023-05-08 11:22:55,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:22:55,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +19: [2023-05-08 11:22:55,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +29: [2023-05-08 11:22:55,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:22:55,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +19: [2023-05-08 11:22:55,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +21: [2023-05-08 11:22:55,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +17: [2023-05-08 11:22:55,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +17: [2023-05-08 11:22:55,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +17: [2023-05-08 11:22:55,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +17: [2023-05-08 11:22:55,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +29: [2023-05-08 11:22:55,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +29: [2023-05-08 11:22:55,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:22:55,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:22:55,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +29: [2023-05-08 11:22:55,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:22:55,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:22:55,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:22:55,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:22:55,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +21: [2023-05-08 11:22:55,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +14: [2023-05-08 11:22:55,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:22:55,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 3: [2023-05-08 11:22:55,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 3: [2023-05-08 11:22:55,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 3: [2023-05-08 11:22:55,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 3: [2023-05-08 11:22:55,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +15: [2023-05-08 11:22:55,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:22:55,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:22:55,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:22:55,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:22:55,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +15: [2023-05-08 11:22:55,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +15: [2023-05-08 11:22:55,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +15: [2023-05-08 11:22:55,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:22:55,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:22:55,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:22:55,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:22:55,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:22:55,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +28: [2023-05-08 11:22:55,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +28: [2023-05-08 11:22:55,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +28: [2023-05-08 11:22:55,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +28: [2023-05-08 11:22:55,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:22:55,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +29: [2023-05-08 11:22:55,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:22:55,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:22:55,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:22:55,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +29: [2023-05-08 11:22:55,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:22:55,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:22:55,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:22:55,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +16: [2023-05-08 11:22:55,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +16: [2023-05-08 11:22:55,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +15: [2023-05-08 11:22:55,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:22:55,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:22:55,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +15: [2023-05-08 11:22:55,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:22:55,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:22:55,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +28: [2023-05-08 11:22:55,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:22:55,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:22:55,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:22:55,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +25: [2023-05-08 11:22:55,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +25: [2023-05-08 11:22:55,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +25: [2023-05-08 11:22:55,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +25: [2023-05-08 11:22:55,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +29: [2023-05-08 11:22:55,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +12: [2023-05-08 11:22:55,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:22:55,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +25: [2023-05-08 11:22:55,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +12: [2023-05-08 11:22:55,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +25: [2023-05-08 11:22:55,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:22:55,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +26: [2023-05-08 11:22:55,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +26: [2023-05-08 11:22:55,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +25: [2023-05-08 11:22:55,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +12: [2023-05-08 11:22:55,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:22:55,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:22:55,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +25: [2023-05-08 11:22:55,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:22:55,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +29: [2023-05-08 11:22:55,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:22:55,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 5: [2023-05-08 11:22:55,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:22:55,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:22:55,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:22:55,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:22:55,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:22:55,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:22:55,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:22:55,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:22:55,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:22:55,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:22:55,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:22:55,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:22:55,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:22:55,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:22:55,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +10: [2023-05-08 11:22:55,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +31: [2023-05-08 11:22:55,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +31: [2023-05-08 11:22:55,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 9: [2023-05-08 11:22:55,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 9: [2023-05-08 11:22:55,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 9: [2023-05-08 11:22:55,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +10: [2023-05-08 11:22:55,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:22:55,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +15: [2023-05-08 11:22:55,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:22:55,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:22:55,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 3: [2023-05-08 11:22:55,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 6: [2023-05-08 11:22:55,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:22:55,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:22:55,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 9: [2023-05-08 11:22:55,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +10: [2023-05-08 11:22:55,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +31: [2023-05-08 11:22:55,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:22:55,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 6: [2023-05-08 11:22:55,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:22:55,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +31: [2023-05-08 11:22:55,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 9: [2023-05-08 11:22:55,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +31: [2023-05-08 11:22:55,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:22:55,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 9: [2023-05-08 11:22:55,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:22:55,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +31: [2023-05-08 11:22:55,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:22:55,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:22:55,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:22:55,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:22:55,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:22:55,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:22:55,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:22:55,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:22:55,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:22:55,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:22:55,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 0: [2023-05-08 11:22:55,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:22:55,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 0: [2023-05-08 11:22:55,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:22:55,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 9: [2023-05-08 11:22:55,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:22:55,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +31: [2023-05-08 11:22:55,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:22:55,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:22:55,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:22:55,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:22:55,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:22:55,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +31: [2023-05-08 11:22:55,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:22:55,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:22:55,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:22:55,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:22:55,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:22:55,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:22:55,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +20: [2023-05-08 11:22:55,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +20: [2023-05-08 11:22:55,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +20: [2023-05-08 11:22:55,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +27: [2023-05-08 11:22:55,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:22:55,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +20: [2023-05-08 11:22:55,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 1: [2023-05-08 11:22:55,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:22:55,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:22:55,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +25: [2023-05-08 11:22:55,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +13: [2023-05-08 11:22:55,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:22:55,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:22:55,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:22:55,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 2: [2023-05-08 11:22:55,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +13: [2023-05-08 11:22:55,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:22:55,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +13: [2023-05-08 11:22:55,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:22:55,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +25: [2023-05-08 11:22:55,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +25: [2023-05-08 11:22:55,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +25: [2023-05-08 11:22:55,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:22:55,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:22:55,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:22:55,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 2: [2023-05-08 11:22:55,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +12: [2023-05-08 11:22:55,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:22:55,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:22:55,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:22:55,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:22:55,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 9: [2023-05-08 11:22:55,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:22:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 9: [2023-05-08 11:22:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:22:55,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +31: [2023-05-08 11:22:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:22:55,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 5: [2023-05-08 11:22:55,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 5: [2023-05-08 11:22:55,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:22:55,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:22:55,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 9: [2023-05-08 11:22:55,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 9: [2023-05-08 11:22:55,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +31: [2023-05-08 11:22:55,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:22:55,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +31: [2023-05-08 11:22:55,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +31: [2023-05-08 11:22:55,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +10: [2023-05-08 11:22:55,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:22:55,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:22:55,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +26: [2023-05-08 11:22:55,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +26: [2023-05-08 11:22:55,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +26: [2023-05-08 11:22:55,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:22:55,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 0: [2023-05-08 11:22:55,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +27: [2023-05-08 11:22:55,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +10: [2023-05-08 11:22:55,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +10: [2023-05-08 11:22:55,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +10: [2023-05-08 11:22:55,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 1: [2023-05-08 11:22:55,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +27: [2023-05-08 11:22:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +27: [2023-05-08 11:22:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +27: [2023-05-08 11:22:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:22:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 3: [2023-05-08 11:22:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:22:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:22:55,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:22:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:22:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 6: [2023-05-08 11:22:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:22:55,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:22:55,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +16: [2023-05-08 11:22:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +25: [2023-05-08 11:22:55,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:22:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +12: [2023-05-08 11:22:55,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +31: [2023-05-08 11:22:55,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:22:55,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:22:55,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +12: [2023-05-08 11:22:55,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:22:55,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +29: [2023-05-08 11:22:55,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +29: [2023-05-08 11:22:55,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 9: [2023-05-08 11:22:55,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:22:55,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 4: [2023-05-08 11:22:55,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +12: [2023-05-08 11:22:55,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +12: [2023-05-08 11:22:55,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:22:55,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +25: [2023-05-08 11:22:55,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:22:55,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:22:55,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:22:55,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:22:55,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 3: [2023-05-08 11:22:55,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:22:55,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +25: [2023-05-08 11:22:55,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +25: [2023-05-08 11:22:55,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:22:55,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 3: [2023-05-08 11:22:55,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +13: [2023-05-08 11:22:55,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:22:55,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:22:55,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +31: [2023-05-08 11:22:55,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 6: [2023-05-08 11:22:55,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:22:55,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:22:55,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +31: [2023-05-08 11:22:55,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +31: [2023-05-08 11:22:55,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:22:55,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:22:55,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:22:55,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:22:55,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:22:55,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:22:55,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:22:55,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 0: [2023-05-08 11:22:55,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:22:55,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:22:55,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 0: [2023-05-08 11:22:55,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:22:55,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:22:55,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:22:55,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +10: [2023-05-08 11:22:55,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +10: [2023-05-08 11:22:55,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 6: [2023-05-08 11:22:55,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +10: [2023-05-08 11:22:55,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 0: [2023-05-08 11:22:55,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 0: [2023-05-08 11:22:55,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 6: [2023-05-08 11:22:55,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:22:55,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:22:55,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:22:55,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:22:55,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:22:55,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:22:55,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:22:55,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +15: [2023-05-08 11:22:55,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +15: [2023-05-08 11:22:55,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +15: [2023-05-08 11:22:55,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +15: [2023-05-08 11:22:55,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 6: [2023-05-08 11:22:55,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:22:55,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +13: [2023-05-08 11:22:55,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +13: [2023-05-08 11:22:55,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +20: [2023-05-08 11:22:55,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +20: [2023-05-08 11:22:55,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:22:55,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:22:55,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:22:55,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 1: [2023-05-08 11:22:55,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:22:55,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +20: [2023-05-08 11:22:55,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:22:55,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 2: [2023-05-08 11:22:55,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:22:55,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +20: [2023-05-08 11:22:55,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +21: [2023-05-08 11:22:55,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:22:55,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:22:55,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:22:55,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:22:55,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +15: [2023-05-08 11:22:55,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:22:55,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:22:55,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 2: [2023-05-08 11:22:55,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:22:55,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +15: [2023-05-08 11:22:55,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:22:55,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:22:55,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:22:55,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:22:55,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 2: [2023-05-08 11:22:55,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +12: [2023-05-08 11:22:55,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +12: [2023-05-08 11:22:55,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +12: [2023-05-08 11:22:55,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +29: [2023-05-08 11:22:55,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:22:55,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +29: [2023-05-08 11:22:55,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +29: [2023-05-08 11:22:55,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +29: [2023-05-08 11:22:55,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +29: [2023-05-08 11:22:55,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:22:55,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +18: [2023-05-08 11:22:55,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +18: [2023-05-08 11:22:55,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +29: [2023-05-08 11:22:55,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +29: [2023-05-08 11:22:55,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +29: [2023-05-08 11:22:55,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +18: [2023-05-08 11:22:55,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +21: [2023-05-08 11:22:55,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:22:55,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:22:55,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:22:55,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:22:55,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +23: [2023-05-08 11:22:55,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:22:55,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:22:55,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:22:55,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:22:55,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:22:55,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +12: [2023-05-08 11:22:55,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +15: [2023-05-08 11:22:55,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:22:55,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +23: [2023-05-08 11:22:55,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +15: [2023-05-08 11:22:55,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:22:55,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +12: [2023-05-08 11:22:55,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:22:55,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:22:55,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:22:55,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:22:55,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:22:55,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +30: [2023-05-08 11:22:55,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:22:55,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +15: [2023-05-08 11:22:55,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +12: [2023-05-08 11:22:55,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +12: [2023-05-08 11:22:55,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:22:55,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +30: [2023-05-08 11:22:55,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +30: [2023-05-08 11:22:55,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +30: [2023-05-08 11:22:55,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:22:55,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +21: [2023-05-08 11:22:55,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +28: [2023-05-08 11:22:55,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +28: [2023-05-08 11:22:55,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +28: [2023-05-08 11:22:55,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +18: [2023-05-08 11:22:55,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:22:55,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +18: [2023-05-08 11:22:55,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:22:55,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +18: [2023-05-08 11:22:55,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:22:55,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +18: [2023-05-08 11:22:55,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 0: [2023-05-08 11:22:55,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 0: [2023-05-08 11:22:55,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 0: [2023-05-08 11:22:55,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 0: [2023-05-08 11:22:55,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 4: [2023-05-08 11:22:55,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +23: [2023-05-08 11:22:55,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 4: [2023-05-08 11:22:55,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 4: [2023-05-08 11:22:55,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 4: [2023-05-08 11:22:55,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +23: [2023-05-08 11:22:55,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:22:55,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:22:55,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:22:55,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:22:55,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:22:55,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:22:55,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:22:55,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +30: [2023-05-08 11:22:55,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:22:55,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +16: [2023-05-08 11:22:55,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +12: [2023-05-08 11:22:55,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +28: [2023-05-08 11:22:55,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:22:55,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:22:55,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:22:55,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:22:55,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +23: [2023-05-08 11:22:55,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +28: [2023-05-08 11:22:55,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:22:55,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:22:55,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:22:55,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:22:55,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:22:55,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:22:55,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:22:55,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:22:55,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:22:55,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:22:55,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:22:55,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +12: [2023-05-08 11:22:55,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 0: [2023-05-08 11:22:55,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:22:55,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +14: [2023-05-08 11:22:55,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +14: [2023-05-08 11:22:55,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +23: [2023-05-08 11:22:55,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:22:55,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:22:55,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:22:55,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +17: [2023-05-08 11:22:55,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:22:55,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:22:55,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:22:55,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:22:55,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:22:55,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:22:55,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +18: [2023-05-08 11:22:55,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +18: [2023-05-08 11:22:55,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:22:55,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:22:55,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 0: [2023-05-08 11:22:55,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:22:55,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:22:55,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +18: [2023-05-08 11:22:55,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:22:55,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +30: [2023-05-08 11:22:55,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:22:55,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +31: [2023-05-08 11:22:55,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +31: [2023-05-08 11:22:55,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +31: [2023-05-08 11:22:55,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +18: [2023-05-08 11:22:55,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +18: [2023-05-08 11:22:55,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 8: [2023-05-08 11:22:55,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +18: [2023-05-08 11:22:55,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 8: [2023-05-08 11:22:55,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:22:55,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:22:55,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:22:55,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:22:55,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:22:55,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:22:55,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:22:55,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:22:55,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:22:55,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:22:55,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +16: [2023-05-08 11:22:55,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:22:55,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:22:55,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +14: [2023-05-08 11:22:55,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:22:55,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:22:55,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:22:55,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +14: [2023-05-08 11:22:55,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +25: [2023-05-08 11:22:55,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +16: [2023-05-08 11:22:55,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +28: [2023-05-08 11:22:55,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +28: [2023-05-08 11:22:55,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +28: [2023-05-08 11:22:55,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +25: [2023-05-08 11:22:55,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +31: [2023-05-08 11:22:55,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +31: [2023-05-08 11:22:55,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +25: [2023-05-08 11:22:55,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +28: [2023-05-08 11:22:55,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +25: [2023-05-08 11:22:55,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +31: [2023-05-08 11:22:55,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:22:55,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:22:55,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +28: [2023-05-08 11:22:55,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +28: [2023-05-08 11:22:55,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +28: [2023-05-08 11:22:55,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +17: [2023-05-08 11:22:55,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:22:55,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +31: [2023-05-08 11:22:55,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:22:55,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:22:55,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:22:55,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:22:55,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +11: [2023-05-08 11:22:55,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +11: [2023-05-08 11:22:55,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +11: [2023-05-08 11:22:55,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:22:55,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:22:55,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:22:55,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +11: [2023-05-08 11:22:55,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 4: [2023-05-08 11:22:55,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 4: [2023-05-08 11:22:55,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 4: [2023-05-08 11:22:55,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 8: [2023-05-08 11:22:55,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +17: [2023-05-08 11:22:55,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:22:55,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:22:55,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +11: [2023-05-08 11:22:55,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 0: [2023-05-08 11:22:55,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:22:55,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 0: [2023-05-08 11:22:55,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:22:55,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 0: [2023-05-08 11:22:55,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 0: [2023-05-08 11:22:55,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 8: [2023-05-08 11:22:55,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:22:55,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +24: [2023-05-08 11:22:55,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +24: [2023-05-08 11:22:55,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +24: [2023-05-08 11:22:55,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 8: [2023-05-08 11:22:55,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:22:55,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:22:55,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +17: [2023-05-08 11:22:55,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:22:55,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:22:55,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +25: [2023-05-08 11:22:55,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:22:55,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +22: [2023-05-08 11:22:55,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:22:55,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:22:55,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:22:55,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +22: [2023-05-08 11:22:55,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:22:55,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 8: [2023-05-08 11:22:55,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:22:55,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 7: [2023-05-08 11:22:55,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:22:55,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:22:55,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:22:55,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +22: [2023-05-08 11:22:55,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:22:55,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:22:55,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +11: [2023-05-08 11:22:55,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 7: [2023-05-08 11:22:55,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +14: [2023-05-08 11:22:55,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +14: [2023-05-08 11:22:55,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +14: [2023-05-08 11:22:55,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +11: [2023-05-08 11:22:55,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:22:55,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 7: [2023-05-08 11:22:55,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +24: [2023-05-08 11:22:55,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +31: [2023-05-08 11:22:55,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +31: [2023-05-08 11:22:55,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +24: [2023-05-08 11:22:55,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:22:55,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +24: [2023-05-08 11:22:55,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:22:55,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:22:55,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:22:55,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:22:55,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:22:55,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +31: [2023-05-08 11:22:55,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +10: [2023-05-08 11:22:55,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +10: [2023-05-08 11:22:55,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +10: [2023-05-08 11:22:55,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +31: [2023-05-08 11:22:55,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +31: [2023-05-08 11:22:55,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +26: [2023-05-08 11:22:55,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +10: [2023-05-08 11:22:55,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +24: [2023-05-08 11:22:55,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:22:55,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +26: [2023-05-08 11:22:55,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +26: [2023-05-08 11:22:55,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +22: [2023-05-08 11:22:55,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:22:55,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 5: [2023-05-08 11:22:55,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 5: [2023-05-08 11:22:55,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +14: [2023-05-08 11:22:55,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +14: [2023-05-08 11:22:55,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +14: [2023-05-08 11:22:55,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 9: [2023-05-08 11:22:55,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 9: [2023-05-08 11:22:55,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +14: [2023-05-08 11:22:55,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 9: [2023-05-08 11:22:55,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 5: [2023-05-08 11:22:55,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 9: [2023-05-08 11:22:55,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +31: [2023-05-08 11:22:55,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +31: [2023-05-08 11:22:55,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 6: [2023-05-08 11:22:55,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 6: [2023-05-08 11:22:55,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 6: [2023-05-08 11:22:55,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +24: [2023-05-08 11:22:55,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:22:55,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +27: [2023-05-08 11:22:55,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +27: [2023-05-08 11:22:55,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +27: [2023-05-08 11:22:55,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +27: [2023-05-08 11:22:55,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 7: [2023-05-08 11:22:55,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 7: [2023-05-08 11:22:55,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 7: [2023-05-08 11:22:55,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 7: [2023-05-08 11:22:55,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 1: [2023-05-08 11:22:55,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 1: [2023-05-08 11:22:55,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 1: [2023-05-08 11:22:55,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. + 1: [2023-05-08 11:22:55,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt. +25: [2023-05-08 11:22:55,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +25: [2023-05-08 11:22:55,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:22:55,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:22:55,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:22:55,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +25: [2023-05-08 11:22:55,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +25: [2023-05-08 11:22:55,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +25: [2023-05-08 11:22:55,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 5: [2023-05-08 11:22:55,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:22:55,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +25: [2023-05-08 11:22:55,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +25: [2023-05-08 11:22:55,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +10: [2023-05-08 11:22:55,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:22:55,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 9: [2023-05-08 11:22:55,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:22:55,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 9: [2023-05-08 11:22:55,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:22:55,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:22:55,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:22:55,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 9: [2023-05-08 11:22:55,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:22:55,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 9: [2023-05-08 11:22:55,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:22:55,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:22:55,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:22:55,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:22:55,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:22:55,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:22:55,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:22:55,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:22:55,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +11: [2023-05-08 11:22:55,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:22:55,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +27: [2023-05-08 11:22:55,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +11: [2023-05-08 11:22:55,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:22:55,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:22:55,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 7: [2023-05-08 11:22:55,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:22:55,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:22:55,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 7: [2023-05-08 11:22:55,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:22:55,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +11: [2023-05-08 11:22:55,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 7: [2023-05-08 11:22:55,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:22:55,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 1: [2023-05-08 11:22:55,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:22:55,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:22:55,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:22:55,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... + 7: [2023-05-08 11:22:55,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:22:55,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:22:55,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:22:55,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +24: [2023-05-08 11:22:55,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +24: [2023-05-08 11:22:55,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +24: [2023-05-08 11:22:55,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +24: [2023-05-08 11:22:55,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +24: [2023-05-08 11:22:55,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +24: [2023-05-08 11:22:55,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +24: [2023-05-08 11:22:55,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +24: [2023-05-08 11:22:55,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 8: [2023-05-08 11:22:55,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 8: [2023-05-08 11:22:55,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 8: [2023-05-08 11:22:55,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 8: [2023-05-08 11:22:55,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +11: [2023-05-08 11:22:55,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:22:55,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:22:55,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 7: [2023-05-08 11:22:55,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:22:55,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:22:55,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:22:55,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 2: [2023-05-08 11:22:55,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:22:55,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 2: [2023-05-08 11:22:55,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:22:55,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +24: [2023-05-08 11:22:55,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:22:55,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 2: [2023-05-08 11:22:55,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:22:55,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:22:55,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:22:55,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +24: [2023-05-08 11:22:55,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 7: [2023-05-08 11:22:55,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:22:55,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +27: [2023-05-08 11:22:55,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +26: [2023-05-08 11:22:55,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +26: [2023-05-08 11:22:55,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 5: [2023-05-08 11:22:55,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +27: [2023-05-08 11:22:55,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +26: [2023-05-08 11:22:55,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 9: [2023-05-08 11:22:55,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 9: [2023-05-08 11:22:55,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:22:55,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +10: [2023-05-08 11:22:55,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 5: [2023-05-08 11:22:55,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 5: [2023-05-08 11:22:55,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +26: [2023-05-08 11:22:55,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +14: [2023-05-08 11:22:56,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:22:56,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:22:56,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +27: [2023-05-08 11:22:56,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +14: [2023-05-08 11:22:56,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +27: [2023-05-08 11:22:56,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 6: [2023-05-08 11:22:56,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +27: [2023-05-08 11:22:56,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +10: [2023-05-08 11:22:56,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 9: [2023-05-08 11:22:56,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 9: [2023-05-08 11:22:56,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +27: [2023-05-08 11:22:56,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 5: [2023-05-08 11:22:56,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +26: [2023-05-08 11:22:56,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +26: [2023-05-08 11:22:56,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +10: [2023-05-08 11:22:56,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +10: [2023-05-08 11:22:56,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +10: [2023-05-08 11:22:56,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +27: [2023-05-08 11:22:56,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +14: [2023-05-08 11:22:56,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +14: [2023-05-08 11:22:56,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:22:56,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +14: [2023-05-08 11:22:56,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 9: [2023-05-08 11:22:56,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +18: [2023-05-08 11:22:56,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +18: [2023-05-08 11:22:56,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +18: [2023-05-08 11:22:56,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +26: [2023-05-08 11:22:56,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 6: [2023-05-08 11:22:56,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 6: [2023-05-08 11:22:56,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 6: [2023-05-08 11:22:56,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +26: [2023-05-08 11:22:56,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +18: [2023-05-08 11:22:56,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +14: [2023-05-08 11:22:56,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +10: [2023-05-08 11:22:56,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +10: [2023-05-08 11:22:56,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 1: [2023-05-08 11:22:56,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. +10: [2023-05-08 11:22:56,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 8: [2023-05-08 11:22:56,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +18: [2023-05-08 11:22:56,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 8: [2023-05-08 11:22:56,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +18: [2023-05-08 11:22:56,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +18: [2023-05-08 11:22:56,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +18: [2023-05-08 11:22:56,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 6: [2023-05-08 11:22:56,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 1: [2023-05-08 11:22:56,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 7: [2023-05-08 11:22:56,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 7: [2023-05-08 11:22:56,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 2: [2023-05-08 11:22:56,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 1: [2023-05-08 11:22:56,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... + 1: [2023-05-08 11:22:56,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt... +20: [2023-05-08 11:22:56,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +20: [2023-05-08 11:22:56,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +20: [2023-05-08 11:22:56,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +20: [2023-05-08 11:22:56,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 2: [2023-05-08 11:22:56,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:22:56,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +23: [2023-05-08 11:22:56,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +23: [2023-05-08 11:22:56,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +23: [2023-05-08 11:22:56,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +13: [2023-05-08 11:22:56,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +13: [2023-05-08 11:22:56,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +13: [2023-05-08 11:22:56,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +13: [2023-05-08 11:22:56,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 2: [2023-05-08 11:22:56,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:22:56,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:22:56,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:22:56,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +18: [2023-05-08 11:22:56,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:22:56,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +14: [2023-05-08 11:22:56,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 2: [2023-05-08 11:22:56,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 2: [2023-05-08 11:22:56,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 2: [2023-05-08 11:22:56,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +18: [2023-05-08 11:22:56,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +18: [2023-05-08 11:22:56,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:22:56,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +23: [2023-05-08 11:22:56,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +18: [2023-05-08 11:22:56,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:22:56,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:22:56,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:22:56,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:22:56,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:22:56,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:22:56,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +14: [2023-05-08 11:22:56,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:22:56,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +20: [2023-05-08 11:22:56,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:22:56,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:22:56,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:22:56,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +19: [2023-05-08 11:22:56,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +19: [2023-05-08 11:22:56,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +19: [2023-05-08 11:22:56,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +18: [2023-05-08 11:22:56,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:22:56,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 8: [2023-05-08 11:22:56,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 3: [2023-05-08 11:22:56,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 3: [2023-05-08 11:22:56,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 3: [2023-05-08 11:22:56,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 3: [2023-05-08 11:22:56,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 8: [2023-05-08 11:22:56,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 8: [2023-05-08 11:22:56,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +14: [2023-05-08 11:22:56,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +18: [2023-05-08 11:22:56,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +18: [2023-05-08 11:22:56,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +18: [2023-05-08 11:22:56,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:22:56,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +17: [2023-05-08 11:22:56,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +17: [2023-05-08 11:22:56,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +17: [2023-05-08 11:22:56,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +19: [2023-05-08 11:22:56,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +19: [2023-05-08 11:22:56,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +19: [2023-05-08 11:22:56,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +19: [2023-05-08 11:22:56,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:22:56,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +21: [2023-05-08 11:22:56,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +21: [2023-05-08 11:22:56,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +21: [2023-05-08 11:22:56,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +23: [2023-05-08 11:22:56,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:22:56,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +29: [2023-05-08 11:22:56,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +29: [2023-05-08 11:22:56,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +29: [2023-05-08 11:22:56,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +30: [2023-05-08 11:22:56,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +30: [2023-05-08 11:22:56,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +30: [2023-05-08 11:22:56,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +23: [2023-05-08 11:22:56,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +30: [2023-05-08 11:22:56,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +23: [2023-05-08 11:22:56,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:22:56,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:22:56,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +16: [2023-05-08 11:22:56,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +16: [2023-05-08 11:22:56,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +16: [2023-05-08 11:22:56,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +23: [2023-05-08 11:22:56,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:22:56,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +23: [2023-05-08 11:22:56,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +13: [2023-05-08 11:22:56,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:22:56,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +23: [2023-05-08 11:22:56,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:22:56,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:22:56,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +17: [2023-05-08 11:22:56,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:22:56,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:22:56,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:22:56,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:22:56,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +23: [2023-05-08 11:22:56,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +13: [2023-05-08 11:22:56,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +13: [2023-05-08 11:22:56,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 2: [2023-05-08 11:22:56,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +21: [2023-05-08 11:22:56,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:22:56,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:22:56,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:22:56,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +30: [2023-05-08 11:22:56,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +30: [2023-05-08 11:22:56,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +30: [2023-05-08 11:22:56,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:22:56,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +16: [2023-05-08 11:22:56,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +16: [2023-05-08 11:22:56,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:22:56,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +22: [2023-05-08 11:22:56,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +16: [2023-05-08 11:22:56,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:22:56,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +30: [2023-05-08 11:22:56,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +16: [2023-05-08 11:22:56,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:22:56,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:22:56,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:22:56,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +29: [2023-05-08 11:22:56,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 2: [2023-05-08 11:22:56,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +25: [2023-05-08 11:22:56,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:22:56,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:22:56,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:22:56,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:22:56,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:22:56,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:22:56,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:22:56,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +30: [2023-05-08 11:22:56,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +30: [2023-05-08 11:22:56,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:22:56,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +25: [2023-05-08 11:22:56,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +25: [2023-05-08 11:22:56,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +12: [2023-05-08 11:22:56,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:22:56,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +12: [2023-05-08 11:22:56,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:22:56,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:22:56,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:22:56,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:22:56,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:22:56,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:22:56,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +22: [2023-05-08 11:22:56,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +19: [2023-05-08 11:22:56,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:22:56,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:22:56,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:22:56,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:22:56,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:22:56,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:22:56,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +19: [2023-05-08 11:22:56,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:22:56,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:22:56,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 3: [2023-05-08 11:22:56,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 3: [2023-05-08 11:22:56,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 3: [2023-05-08 11:22:56,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +12: [2023-05-08 11:22:56,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:22:56,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:22:56,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:22:56,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:22:56,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +15: [2023-05-08 11:22:56,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:22:56,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +15: [2023-05-08 11:22:56,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:22:56,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +19: [2023-05-08 11:22:56,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +17: [2023-05-08 11:22:56,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +15: [2023-05-08 11:22:56,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +15: [2023-05-08 11:22:56,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +15: [2023-05-08 11:22:56,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +15: [2023-05-08 11:22:56,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:22:56,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:22:56,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +28: [2023-05-08 11:22:56,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +28: [2023-05-08 11:22:56,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +28: [2023-05-08 11:22:56,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:22:56,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +28: [2023-05-08 11:22:56,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +19: [2023-05-08 11:22:56,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:22:56,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:22:56,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +31: [2023-05-08 11:22:56,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +31: [2023-05-08 11:22:56,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +31: [2023-05-08 11:22:56,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:22:56,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +31: [2023-05-08 11:22:56,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:22:56,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 9: [2023-05-08 11:22:56,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:22:56,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +19: [2023-05-08 11:22:56,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +27: [2023-05-08 11:22:56,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +27: [2023-05-08 11:22:56,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +27: [2023-05-08 11:22:56,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:22:56,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +28: [2023-05-08 11:22:56,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:22:56,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:22:56,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +31: [2023-05-08 11:22:56,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:22:56,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:22:56,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:22:56,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:22:56,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:22:56,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:22:56,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:22:56,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:22:56,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +31: [2023-05-08 11:22:56,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:22:56,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +31: [2023-05-08 11:22:56,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +31: [2023-05-08 11:22:56,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:22:56,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:22:56,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:22:56,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:22:56,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:22:56,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +13: [2023-05-08 11:22:56,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +21: [2023-05-08 11:22:56,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:22:56,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:22:56,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:22:56,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:22:56,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:22:56,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:22:56,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +17: [2023-05-08 11:22:56,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +21: [2023-05-08 11:22:56,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:22:56,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:22:56,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +17: [2023-05-08 11:22:56,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +21: [2023-05-08 11:22:56,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +21: [2023-05-08 11:22:56,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +16: [2023-05-08 11:22:56,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:22:56,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:22:56,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +17: [2023-05-08 11:22:56,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +30: [2023-05-08 11:22:56,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +16: [2023-05-08 11:22:56,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:22:56,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:22:56,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +21: [2023-05-08 11:22:56,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +16: [2023-05-08 11:22:56,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +16: [2023-05-08 11:22:56,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +16: [2023-05-08 11:22:56,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +16: [2023-05-08 11:22:56,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +30: [2023-05-08 11:22:56,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:22:56,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:22:56,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +30: [2023-05-08 11:22:56,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:22:56,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:22:56,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:22:56,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +30: [2023-05-08 11:22:56,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 3: [2023-05-08 11:22:56,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:22:56,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:22:56,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:22:56,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:22:56,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:22:56,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:22:56,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:22:56,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:22:56,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:22:56,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +22: [2023-05-08 11:22:56,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:22:56,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +30: [2023-05-08 11:22:56,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +13: [2023-05-08 11:22:56,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +22: [2023-05-08 11:22:56,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +22: [2023-05-08 11:22:56,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:22:56,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:22:56,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:22:56,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:22:56,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:22:56,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:22:56,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +26: [2023-05-08 11:22:56,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:22:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:22:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:22:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:22:56,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +26: [2023-05-08 11:22:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:22:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:22:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:22:56,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +15: [2023-05-08 11:22:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +15: [2023-05-08 11:22:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +20: [2023-05-08 11:22:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:22:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:22:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:22:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +25: [2023-05-08 11:22:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:22:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:22:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:22:56,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +20: [2023-05-08 11:22:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:22:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:22:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:22:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +15: [2023-05-08 11:22:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:22:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:22:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:22:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:22:56,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:22:56,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +12: [2023-05-08 11:22:56,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +22: [2023-05-08 11:22:56,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +10: [2023-05-08 11:22:56,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:22:56,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +12: [2023-05-08 11:22:56,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:22:56,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +27: [2023-05-08 11:22:56,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:22:56,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +26: [2023-05-08 11:22:56,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:22:56,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +26: [2023-05-08 11:22:56,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:22:56,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:22:56,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:22:56,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +26: [2023-05-08 11:22:56,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:22:56,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +21: [2023-05-08 11:22:56,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:22:56,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +21: [2023-05-08 11:22:56,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +21: [2023-05-08 11:22:56,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:22:56,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:22:56,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +20: [2023-05-08 11:22:56,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +15: [2023-05-08 11:22:56,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +10: [2023-05-08 11:22:56,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:22:56,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 3: [2023-05-08 11:22:56,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +31: [2023-05-08 11:22:56,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +10: [2023-05-08 11:22:56,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:22:56,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +10: [2023-05-08 11:22:56,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +10: [2023-05-08 11:22:56,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +15: [2023-05-08 11:22:56,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:22:56,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:22:56,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:22:56,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:22:56,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:22:56,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:22:56,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 9: [2023-05-08 11:22:56,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:22:56,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:22:56,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:22:56,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:22:56,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:22:56,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:22:56,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +28: [2023-05-08 11:22:56,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:22:56,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:22:56,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 4: [2023-05-08 11:22:56,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +27: [2023-05-08 11:22:56,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +27: [2023-05-08 11:22:56,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:22:56,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +23: [2023-05-08 11:22:56,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:22:56,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:22:56,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:22:56,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:22:56,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:22:56,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +23: [2023-05-08 11:22:56,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:22:56,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:22:56,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:22:56,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +12: [2023-05-08 11:22:56,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 4: [2023-05-08 11:22:56,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:22:56,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:22:56,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +31: [2023-05-08 11:22:56,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +31: [2023-05-08 11:22:56,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +27: [2023-05-08 11:22:56,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:22:56,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:22:56,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:22:56,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:22:56,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:22:56,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +29: [2023-05-08 11:22:56,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:22:56,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:22:56,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +28: [2023-05-08 11:22:56,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +31: [2023-05-08 11:22:56,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +28: [2023-05-08 11:22:56,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:22:56,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:22:56,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:22:56,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 4: [2023-05-08 11:22:56,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 4: [2023-05-08 11:22:56,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 4: [2023-05-08 11:22:56,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 3: [2023-05-08 11:22:56,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:22:56,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +21: [2023-05-08 11:22:56,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:22:56,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:22:56,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +15: [2023-05-08 11:22:56,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:22:56,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +27: [2023-05-08 11:22:56,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +27: [2023-05-08 11:22:56,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:22:56,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:22:56,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:22:56,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:22:56,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:22:56,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:22:56,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +13: [2023-05-08 11:22:56,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:22:56,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +15: [2023-05-08 11:22:56,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:22:56,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +28: [2023-05-08 11:22:56,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +22: [2023-05-08 11:22:56,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +22: [2023-05-08 11:22:56,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +22: [2023-05-08 11:22:56,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +22: [2023-05-08 11:22:56,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +22: [2023-05-08 11:22:56,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +22: [2023-05-08 11:22:56,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +22: [2023-05-08 11:22:56,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:22:56,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +22: [2023-05-08 11:22:56,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:22:56,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:22:56,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:22:56,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:22:56,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:22:56,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +28: [2023-05-08 11:22:56,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +27: [2023-05-08 11:22:56,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:22:56,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:22:56,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:22:56,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:22:56,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +18: [2023-05-08 11:22:56,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +18: [2023-05-08 11:22:56,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +18: [2023-05-08 11:22:56,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +18: [2023-05-08 11:22:56,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 5: [2023-05-08 11:22:56,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +21: [2023-05-08 11:22:56,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:22:56,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:22:56,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:22:56,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:22:56,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:22:56,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +21: [2023-05-08 11:22:56,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:22:56,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:22:56,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +31: [2023-05-08 11:22:56,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +31: [2023-05-08 11:22:56,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +31: [2023-05-08 11:22:56,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 4: [2023-05-08 11:22:56,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +12: [2023-05-08 11:22:56,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:22:56,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +22: [2023-05-08 11:22:56,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:22:56,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +10: [2023-05-08 11:22:56,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:22:56,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:22:56,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +28: [2023-05-08 11:22:56,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +22: [2023-05-08 11:22:56,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +22: [2023-05-08 11:22:56,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +28: [2023-05-08 11:22:56,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +28: [2023-05-08 11:22:56,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +28: [2023-05-08 11:22:56,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 0: [2023-05-08 11:22:56,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +26: [2023-05-08 11:22:56,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +26: [2023-05-08 11:22:56,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +26: [2023-05-08 11:22:56,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +26: [2023-05-08 11:22:56,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:22:56,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:22:56,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:22:56,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:22:56,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +24: [2023-05-08 11:22:56,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +20: [2023-05-08 11:22:56,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:22:56,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +20: [2023-05-08 11:22:56,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:22:56,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:22:56,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +21: [2023-05-08 11:22:56,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:22:56,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 0: [2023-05-08 11:22:56,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 0: [2023-05-08 11:22:56,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +18: [2023-05-08 11:22:56,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +11: [2023-05-08 11:22:56,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +11: [2023-05-08 11:22:56,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +11: [2023-05-08 11:22:56,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +18: [2023-05-08 11:22:56,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +11: [2023-05-08 11:22:56,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +31: [2023-05-08 11:22:56,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +18: [2023-05-08 11:22:56,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +31: [2023-05-08 11:22:56,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +18: [2023-05-08 11:22:56,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 6: [2023-05-08 11:22:56,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +10: [2023-05-08 11:22:56,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:22:56,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +31: [2023-05-08 11:22:56,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:22:56,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:22:56,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:22:56,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:22:56,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +26: [2023-05-08 11:22:56,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +26: [2023-05-08 11:22:56,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +22: [2023-05-08 11:22:56,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:22:56,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:22:56,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +26: [2023-05-08 11:22:56,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:22:56,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:22:56,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:22:56,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +15: [2023-05-08 11:22:56,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +28: [2023-05-08 11:22:56,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +15: [2023-05-08 11:22:56,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +28: [2023-05-08 11:22:56,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:22:56,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +10: [2023-05-08 11:22:56,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:22:56,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +28: [2023-05-08 11:22:56,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +10: [2023-05-08 11:22:56,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +24: [2023-05-08 11:22:56,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +12: [2023-05-08 11:22:56,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:22:56,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +24: [2023-05-08 11:22:56,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +24: [2023-05-08 11:22:56,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:22:56,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:22:56,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +16: [2023-05-08 11:22:56,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +16: [2023-05-08 11:22:56,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +16: [2023-05-08 11:22:56,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:22:56,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +12: [2023-05-08 11:22:56,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:22:56,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:22:56,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +12: [2023-05-08 11:22:56,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +20: [2023-05-08 11:22:56,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +20: [2023-05-08 11:22:56,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:22:56,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +12: [2023-05-08 11:22:56,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +16: [2023-05-08 11:22:56,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +16: [2023-05-08 11:22:56,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:22:56,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:22:56,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +26: [2023-05-08 11:22:56,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +26: [2023-05-08 11:22:56,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 0: [2023-05-08 11:22:56,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 5: [2023-05-08 11:22:56,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 5: [2023-05-08 11:22:56,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +12: [2023-05-08 11:22:56,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +26: [2023-05-08 11:22:56,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +26: [2023-05-08 11:22:56,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 5: [2023-05-08 11:22:56,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 4: [2023-05-08 11:22:56,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +11: [2023-05-08 11:22:56,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 7: [2023-05-08 11:22:56,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:22:56,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:22:56,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:22:56,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:22:56,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +17: [2023-05-08 11:22:56,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +17: [2023-05-08 11:22:56,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +17: [2023-05-08 11:22:56,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +12: [2023-05-08 11:22:56,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 0: [2023-05-08 11:22:56,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +17: [2023-05-08 11:22:56,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:22:56,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:22:56,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +11: [2023-05-08 11:22:56,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +11: [2023-05-08 11:22:56,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +11: [2023-05-08 11:22:56,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:22:56,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:22:56,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +11: [2023-05-08 11:22:56,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:22:56,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +11: [2023-05-08 11:22:56,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:22:56,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +31: [2023-05-08 11:22:56,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:22:56,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +31: [2023-05-08 11:22:56,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +31: [2023-05-08 11:22:56,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +31: [2023-05-08 11:22:56,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:22:56,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:22:56,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +18: [2023-05-08 11:22:56,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +31: [2023-05-08 11:22:56,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +18: [2023-05-08 11:22:56,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +16: [2023-05-08 11:22:56,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +16: [2023-05-08 11:22:56,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:22:56,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:22:56,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:22:56,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:22:56,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:22:56,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:22:56,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:22:56,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +18: [2023-05-08 11:22:56,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:22:56,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +28: [2023-05-08 11:22:56,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +18: [2023-05-08 11:22:56,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 8: [2023-05-08 11:22:56,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 9: [2023-05-08 11:22:56,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 8: [2023-05-08 11:22:56,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:22:56,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:22:56,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 9: [2023-05-08 11:22:56,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +28: [2023-05-08 11:22:56,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +28: [2023-05-08 11:22:56,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +28: [2023-05-08 11:22:56,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +18: [2023-05-08 11:22:56,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +28: [2023-05-08 11:22:56,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +16: [2023-05-08 11:22:56,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:22:56,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +24: [2023-05-08 11:22:56,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +24: [2023-05-08 11:22:56,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +28: [2023-05-08 11:22:56,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +17: [2023-05-08 11:22:56,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:22:56,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:22:56,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:22:56,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:22:56,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +28: [2023-05-08 11:22:56,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +11: [2023-05-08 11:22:56,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:22:56,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +11: [2023-05-08 11:22:56,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:22:56,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +24: [2023-05-08 11:22:56,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +24: [2023-05-08 11:22:56,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +11: [2023-05-08 11:22:56,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +16: [2023-05-08 11:22:56,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +16: [2023-05-08 11:22:56,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +14: [2023-05-08 11:22:56,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +14: [2023-05-08 11:22:56,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +14: [2023-05-08 11:22:56,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +14: [2023-05-08 11:22:56,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +25: [2023-05-08 11:22:56,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +25: [2023-05-08 11:22:56,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +25: [2023-05-08 11:22:56,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +25: [2023-05-08 11:22:56,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 0: [2023-05-08 11:22:56,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:22:56,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 6: [2023-05-08 11:22:56,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 6: [2023-05-08 11:22:56,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 6: [2023-05-08 11:22:56,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 8: [2023-05-08 11:22:56,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +11: [2023-05-08 11:22:56,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +11: [2023-05-08 11:22:56,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:22:56,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +17: [2023-05-08 11:22:56,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +17: [2023-05-08 11:22:56,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 7: [2023-05-08 11:22:56,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 8: [2023-05-08 11:22:56,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 7: [2023-05-08 11:22:56,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 0: [2023-05-08 11:22:56,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +11: [2023-05-08 11:22:56,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +11: [2023-05-08 11:22:56,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 0: [2023-05-08 11:22:56,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 0: [2023-05-08 11:22:56,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +26: [2023-05-08 11:22:56,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 8: [2023-05-08 11:22:56,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:22:56,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:22:56,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +27: [2023-05-08 11:22:56,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 5: [2023-05-08 11:22:56,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +27: [2023-05-08 11:22:56,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +27: [2023-05-08 11:22:56,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +25: [2023-05-08 11:22:56,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +25: [2023-05-08 11:22:56,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:22:56,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 5: [2023-05-08 11:22:56,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +26: [2023-05-08 11:22:56,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +25: [2023-05-08 11:22:56,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +25: [2023-05-08 11:22:56,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:22:56,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +26: [2023-05-08 11:22:56,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +11: [2023-05-08 11:22:56,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:22:56,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +11: [2023-05-08 11:22:56,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +11: [2023-05-08 11:22:56,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +26: [2023-05-08 11:22:56,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +26: [2023-05-08 11:22:56,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +14: [2023-05-08 11:22:56,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +14: [2023-05-08 11:22:56,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:22:56,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +14: [2023-05-08 11:22:56,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +11: [2023-05-08 11:22:56,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:22:56,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:22:56,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 6: [2023-05-08 11:22:56,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +11: [2023-05-08 11:22:56,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +11: [2023-05-08 11:22:56,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 7: [2023-05-08 11:22:56,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:22:56,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 1: [2023-05-08 11:22:56,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 1: [2023-05-08 11:22:56,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +27: [2023-05-08 11:22:56,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +27: [2023-05-08 11:22:56,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:22:56,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 9: [2023-05-08 11:22:56,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 9: [2023-05-08 11:22:56,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 9: [2023-05-08 11:22:56,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 1: [2023-05-08 11:22:56,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +25: [2023-05-08 11:22:56,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:22:56,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:22:56,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:22:56,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:22:56,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:22:56,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +25: [2023-05-08 11:22:56,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +25: [2023-05-08 11:22:56,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 7: [2023-05-08 11:22:56,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:22:56,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +14: [2023-05-08 11:22:56,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +14: [2023-05-08 11:22:56,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:22:56,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:22:56,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +25: [2023-05-08 11:22:56,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 7: [2023-05-08 11:22:56,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 1: [2023-05-08 11:22:56,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +14: [2023-05-08 11:22:56,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +14: [2023-05-08 11:22:56,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 6: [2023-05-08 11:22:56,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +27: [2023-05-08 11:22:56,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +27: [2023-05-08 11:22:56,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +27: [2023-05-08 11:22:56,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 7: [2023-05-08 11:22:56,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 6: [2023-05-08 11:22:56,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 6: [2023-05-08 11:22:56,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 6: [2023-05-08 11:22:56,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +27: [2023-05-08 11:22:56,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +10: [2023-05-08 11:22:56,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +10: [2023-05-08 11:22:56,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. +10: [2023-05-08 11:22:56,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 7: [2023-05-08 11:22:56,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +10: [2023-05-08 11:22:56,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt. + 6: [2023-05-08 11:22:56,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +27: [2023-05-08 11:22:56,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +27: [2023-05-08 11:22:56,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +27: [2023-05-08 11:22:56,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +27: [2023-05-08 11:22:56,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 6: [2023-05-08 11:22:56,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 2: [2023-05-08 11:22:56,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +10: [2023-05-08 11:22:56,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:22:56,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:22:56,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +24: [2023-05-08 11:22:56,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:22:56,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:22:56,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +24: [2023-05-08 11:22:56,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:22:56,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +24: [2023-05-08 11:22:56,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 1: [2023-05-08 11:22:56,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +10: [2023-05-08 11:22:56,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:22:56,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... +10: [2023-05-08 11:22:56,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 1: [2023-05-08 11:22:56,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +24: [2023-05-08 11:22:56,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:22:56,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:22:56,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:22:56,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:22:56,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +27: [2023-05-08 11:22:56,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +24: [2023-05-08 11:22:56,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:22:56,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:22:56,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:22:56,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +27: [2023-05-08 11:22:56,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +24: [2023-05-08 11:22:56,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +24: [2023-05-08 11:22:56,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:22:56,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:22:56,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:22:56,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:22:56,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 2: [2023-05-08 11:22:56,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:22:56,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:22:56,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:22:56,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:22:56,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:22:56,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +10: [2023-05-08 11:22:56,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:22:56,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:22:56,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:22:56,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +10: [2023-05-08 11:22:56,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... +10: [2023-05-08 11:22:56,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt... + 2: [2023-05-08 11:22:56,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:22:56,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:22:56,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:22:56,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:22:56,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:22:56,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +25: [2023-05-08 11:22:56,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:22:56,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +25: [2023-05-08 11:22:56,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:22:56,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +25: [2023-05-08 11:22:56,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +25: [2023-05-08 11:22:56,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:22:56,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:22:56,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:22:56,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:22:56,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:22:56,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:22:56,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:22:56,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:22:56,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:22:56,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:22:56,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:22:56,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:22:56,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:22:56,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +20: [2023-05-08 11:22:56,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:22:56,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +20: [2023-05-08 11:22:56,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:22:56,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:22:56,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:22:56,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +20: [2023-05-08 11:22:56,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +20: [2023-05-08 11:22:56,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:22:56,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:22:56,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:22:56,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:22:56,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:22:56,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:22:56,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +12: [2023-05-08 11:22:56,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +12: [2023-05-08 11:22:56,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:22:56,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:22:56,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +25: [2023-05-08 11:22:56,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +25: [2023-05-08 11:22:56,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +25: [2023-05-08 11:22:56,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +29: [2023-05-08 11:22:56,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +29: [2023-05-08 11:22:56,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:22:56,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:22:56,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:22:56,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +12: [2023-05-08 11:22:56,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +12: [2023-05-08 11:22:56,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:22:56,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:22:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +29: [2023-05-08 11:22:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +29: [2023-05-08 11:22:56,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:22:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +14: [2023-05-08 11:22:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +14: [2023-05-08 11:22:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +14: [2023-05-08 11:22:56,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:22:56,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:22:56,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:22:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +14: [2023-05-08 11:22:56,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:22:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:22:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:22:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:22:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +18: [2023-05-08 11:22:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:22:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +18: [2023-05-08 11:22:56,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:22:56,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:22:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +19: [2023-05-08 11:22:56,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:22:56,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:22:56,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +18: [2023-05-08 11:22:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:22:56,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:22:56,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +20: [2023-05-08 11:22:56,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:22:56,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:22:56,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:22:56,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +10: [2023-05-08 11:22:56,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +10: [2023-05-08 11:22:56,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +10: [2023-05-08 11:22:56,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:22:56,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +23: [2023-05-08 11:22:56,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +23: [2023-05-08 11:22:56,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +10: [2023-05-08 11:22:56,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +10: [2023-05-08 11:22:56,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +10: [2023-05-08 11:22:56,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 2: [2023-05-08 11:22:56,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +19: [2023-05-08 11:22:56,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:22:56,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:22:56,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +14: [2023-05-08 11:22:56,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 5: [2023-05-08 11:22:56,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +12: [2023-05-08 11:22:56,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +12: [2023-05-08 11:22:56,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:22:56,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:22:56,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +23: [2023-05-08 11:22:56,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +12: [2023-05-08 11:22:56,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +12: [2023-05-08 11:22:56,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:22:56,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:22:56,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:22:56,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:22:56,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +29: [2023-05-08 11:22:56,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +29: [2023-05-08 11:22:56,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +29: [2023-05-08 11:22:56,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:22:56,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:22:56,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:22:56,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +14: [2023-05-08 11:22:56,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:22:56,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:22:56,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +14: [2023-05-08 11:22:56,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:22:56,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +19: [2023-05-08 11:22:56,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +23: [2023-05-08 11:22:56,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:22:56,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:22:56,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:22:56,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +15: [2023-05-08 11:22:56,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +15: [2023-05-08 11:22:56,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:22:56,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:22:56,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +15: [2023-05-08 11:22:56,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:22:56,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:22:56,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:22:56,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +23: [2023-05-08 11:22:56,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +14: [2023-05-08 11:22:56,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +12: [2023-05-08 11:22:56,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +12: [2023-05-08 11:22:56,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:22:56,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:22:56,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:22:56,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +20: [2023-05-08 11:22:56,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:22:56,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:22:56,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:22:56,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +12: [2023-05-08 11:22:56,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +14: [2023-05-08 11:22:56,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +12: [2023-05-08 11:22:56,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 3: [2023-05-08 11:22:56,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 3: [2023-05-08 11:22:56,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +18: [2023-05-08 11:22:56,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 4: [2023-05-08 11:22:56,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:22:56,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:22:56,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +14: [2023-05-08 11:22:56,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +20: [2023-05-08 11:22:56,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:22:56,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +30: [2023-05-08 11:22:56,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 0: [2023-05-08 11:22:56,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:22:56,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +10: [2023-05-08 11:22:56,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +10: [2023-05-08 11:22:56,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:22:56,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:22:56,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +20: [2023-05-08 11:22:56,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:22:56,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 9: [2023-05-08 11:22:56,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +20: [2023-05-08 11:22:56,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:22:56,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +28: [2023-05-08 11:22:56,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +28: [2023-05-08 11:22:56,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +28: [2023-05-08 11:22:56,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +28: [2023-05-08 11:22:56,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:22:56,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:22:56,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:22:56,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:22:56,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +28: [2023-05-08 11:22:56,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +28: [2023-05-08 11:22:56,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +26: [2023-05-08 11:22:56,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +26: [2023-05-08 11:22:56,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +26: [2023-05-08 11:22:56,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:22:56,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +28: [2023-05-08 11:22:56,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +29: [2023-05-08 11:22:56,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:22:56,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +15: [2023-05-08 11:22:56,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:22:56,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:22:56,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +18: [2023-05-08 11:22:56,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:22:56,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +23: [2023-05-08 11:22:56,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:22:56,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 3: [2023-05-08 11:22:56,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +10: [2023-05-08 11:22:56,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:22:56,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:22:56,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 6: [2023-05-08 11:22:56,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +21: [2023-05-08 11:22:56,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +21: [2023-05-08 11:22:56,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:22:56,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +21: [2023-05-08 11:22:56,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +21: [2023-05-08 11:22:56,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +21: [2023-05-08 11:22:56,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:22:56,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +30: [2023-05-08 11:22:56,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +30: [2023-05-08 11:22:56,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +30: [2023-05-08 11:22:56,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:22:56,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +16: [2023-05-08 11:22:56,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +16: [2023-05-08 11:22:56,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +21: [2023-05-08 11:22:56,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:22:56,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +16: [2023-05-08 11:22:56,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +22: [2023-05-08 11:22:56,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +30: [2023-05-08 11:22:56,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +22: [2023-05-08 11:22:56,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:22:56,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +22: [2023-05-08 11:22:56,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +22: [2023-05-08 11:22:56,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:22:56,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +10: [2023-05-08 11:22:56,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:22:56,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:22:56,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:22:56,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +23: [2023-05-08 11:22:56,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 2: [2023-05-08 11:22:56,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +10: [2023-05-08 11:22:56,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +15: [2023-05-08 11:22:56,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:22:56,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +15: [2023-05-08 11:22:56,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:22:56,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +15: [2023-05-08 11:22:56,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +18: [2023-05-08 11:22:56,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:22:56,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +30: [2023-05-08 11:22:56,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +30: [2023-05-08 11:22:56,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +30: [2023-05-08 11:22:56,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:22:56,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:22:56,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:22:56,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:22:56,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:22:56,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:22:56,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:22:56,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +13: [2023-05-08 11:22:56,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:22:56,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:22:56,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:22:56,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:22:56,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:22:56,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:22:56,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:22:56,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:22:56,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +17: [2023-05-08 11:22:56,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:22:56,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +17: [2023-05-08 11:22:56,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:22:56,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:22:56,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:22:56,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +23: [2023-05-08 11:22:56,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:22:56,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +16: [2023-05-08 11:22:56,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +21: [2023-05-08 11:22:56,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:22:56,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:22:56,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +30: [2023-05-08 11:22:56,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:22:56,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +13: [2023-05-08 11:22:56,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:22:56,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +13: [2023-05-08 11:22:56,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:22:56,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +16: [2023-05-08 11:22:56,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:22:56,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:22:56,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:22:56,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:22:56,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:22:56,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +17: [2023-05-08 11:22:56,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:22:56,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:22:56,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:22:56,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +15: [2023-05-08 11:22:56,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:22:56,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:22:56,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:22:56,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +30: [2023-05-08 11:22:56,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +21: [2023-05-08 11:22:56,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 3: [2023-05-08 11:22:56,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +15: [2023-05-08 11:22:56,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:22:56,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +22: [2023-05-08 11:22:56,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:22:56,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:22:56,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +21: [2023-05-08 11:22:56,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +21: [2023-05-08 11:22:56,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:22:56,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:22:56,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:22:56,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:22:56,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:22:56,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +28: [2023-05-08 11:22:56,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:22:56,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +23: [2023-05-08 11:22:56,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:22:56,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:22:56,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:22:56,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +16: [2023-05-08 11:22:56,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:22:56,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:22:56,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +16: [2023-05-08 11:22:56,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 3: [2023-05-08 11:22:56,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +13: [2023-05-08 11:22:56,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +30: [2023-05-08 11:22:56,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +13: [2023-05-08 11:22:56,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:22:56,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:22:56,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +21: [2023-05-08 11:22:56,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:22:56,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +16: [2023-05-08 11:22:56,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +16: [2023-05-08 11:22:56,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +16: [2023-05-08 11:22:56,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +22: [2023-05-08 11:22:56,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +22: [2023-05-08 11:22:56,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:22:56,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:22:56,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:22:56,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +16: [2023-05-08 11:22:56,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:22:56,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:22:56,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:22:56,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:22:56,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +17: [2023-05-08 11:22:56,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:22:56,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:22:56,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:22:56,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:22:56,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +16: [2023-05-08 11:22:56,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:22:56,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:22:56,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:22:56,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:22:56,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:22:56,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +29: [2023-05-08 11:22:56,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +29: [2023-05-08 11:22:56,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +22: [2023-05-08 11:22:56,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:22:56,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +21: [2023-05-08 11:22:56,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:22:56,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:22:56,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:22:56,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:22:56,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:22:56,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:22:56,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +23: [2023-05-08 11:22:56,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +22: [2023-05-08 11:22:56,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +30: [2023-05-08 11:22:56,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +30: [2023-05-08 11:22:56,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:22:56,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:22:56,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +23: [2023-05-08 11:22:56,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:22:56,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:22:56,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:22:56,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:22:56,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:22:56,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +17: [2023-05-08 11:22:56,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:22:56,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +30: [2023-05-08 11:22:56,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +30: [2023-05-08 11:22:56,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +20: [2023-05-08 11:22:56,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +20: [2023-05-08 11:22:56,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 3: [2023-05-08 11:22:56,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +20: [2023-05-08 11:22:56,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +19: [2023-05-08 11:22:56,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +19: [2023-05-08 11:22:56,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +19: [2023-05-08 11:22:56,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +20: [2023-05-08 11:22:56,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +19: [2023-05-08 11:22:56,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +21: [2023-05-08 11:22:56,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +21: [2023-05-08 11:22:56,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +21: [2023-05-08 11:22:56,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +21: [2023-05-08 11:22:56,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +17: [2023-05-08 11:22:56,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +17: [2023-05-08 11:22:56,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +17: [2023-05-08 11:22:56,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +17: [2023-05-08 11:22:56,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 3: [2023-05-08 11:22:56,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:22:56,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:22:56,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +13: [2023-05-08 11:22:56,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 8: [2023-05-08 11:22:56,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +13: [2023-05-08 11:22:56,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +16: [2023-05-08 11:22:56,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:22:56,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 7: [2023-05-08 11:22:56,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 8: [2023-05-08 11:22:56,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 7: [2023-05-08 11:22:56,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 7: [2023-05-08 11:22:56,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +19: [2023-05-08 11:22:56,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:22:56,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:22:56,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:22:56,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:22:56,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +20: [2023-05-08 11:22:56,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:22:56,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +21: [2023-05-08 11:22:56,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +21: [2023-05-08 11:22:56,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +20: [2023-05-08 11:22:56,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:22:56,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +21: [2023-05-08 11:22:56,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +21: [2023-05-08 11:22:56,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +17: [2023-05-08 11:22:56,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:22:56,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:22:56,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +17: [2023-05-08 11:22:56,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:22:56,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +17: [2023-05-08 11:22:56,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +20: [2023-05-08 11:22:56,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +13: [2023-05-08 11:22:56,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +13: [2023-05-08 11:22:56,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +13: [2023-05-08 11:22:56,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +17: [2023-05-08 11:22:56,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +22: [2023-05-08 11:22:56,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +22: [2023-05-08 11:22:56,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +24: [2023-05-08 11:22:56,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +24: [2023-05-08 11:22:56,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +24: [2023-05-08 11:22:56,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +24: [2023-05-08 11:22:56,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +24: [2023-05-08 11:22:56,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:22:56,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +24: [2023-05-08 11:22:56,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:22:56,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +24: [2023-05-08 11:22:56,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:22:56,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 7: [2023-05-08 11:22:56,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +24: [2023-05-08 11:22:56,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:22:56,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +29: [2023-05-08 11:22:56,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 8: [2023-05-08 11:22:56,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:22:56,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:22:56,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:22:56,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +29: [2023-05-08 11:22:56,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +11: [2023-05-08 11:22:56,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +11: [2023-05-08 11:22:56,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +11: [2023-05-08 11:22:56,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +22: [2023-05-08 11:22:56,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +22: [2023-05-08 11:22:56,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:22:56,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +29: [2023-05-08 11:22:56,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +29: [2023-05-08 11:22:56,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 7: [2023-05-08 11:22:56,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:22:56,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +22: [2023-05-08 11:22:56,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:22:56,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +19: [2023-05-08 11:22:56,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:22:56,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:22:56,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +19: [2023-05-08 11:22:56,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +19: [2023-05-08 11:22:56,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +13: [2023-05-08 11:22:56,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:22:56,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +13: [2023-05-08 11:22:56,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:22:56,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +20: [2023-05-08 11:22:56,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:22:56,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +21: [2023-05-08 11:22:56,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +21: [2023-05-08 11:22:56,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +21: [2023-05-08 11:22:56,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +21: [2023-05-08 11:22:56,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:22:56,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +21: [2023-05-08 11:22:56,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +21: [2023-05-08 11:22:56,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +21: [2023-05-08 11:22:56,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +21: [2023-05-08 11:22:56,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +20: [2023-05-08 11:22:56,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:22:56,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 8: [2023-05-08 11:22:56,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +17: [2023-05-08 11:22:56,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:22:56,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +19: [2023-05-08 11:22:56,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:22:56,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +17: [2023-05-08 11:22:56,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:22:56,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 8: [2023-05-08 11:22:56,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +20: [2023-05-08 11:22:56,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:22:56,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +24: [2023-05-08 11:22:56,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:22:56,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +20: [2023-05-08 11:22:56,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:22:56,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +19: [2023-05-08 11:22:56,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:22:56,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 8: [2023-05-08 11:22:56,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 8: [2023-05-08 11:22:56,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +13: [2023-05-08 11:22:56,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:22:56,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +26: [2023-05-08 11:22:56,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +26: [2023-05-08 11:22:56,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +26: [2023-05-08 11:22:56,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +26: [2023-05-08 11:22:56,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +11: [2023-05-08 11:22:56,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +17: [2023-05-08 11:22:56,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:22:56,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:22:56,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +17: [2023-05-08 11:22:56,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +11: [2023-05-08 11:22:56,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:22:56,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:22:56,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +24: [2023-05-08 11:22:56,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +22: [2023-05-08 11:22:56,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +22: [2023-05-08 11:22:56,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +22: [2023-05-08 11:22:56,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +22: [2023-05-08 11:22:56,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +22: [2023-05-08 11:22:56,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +22: [2023-05-08 11:22:56,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +22: [2023-05-08 11:22:56,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +22: [2023-05-08 11:22:56,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +24: [2023-05-08 11:22:56,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:22:56,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +24: [2023-05-08 11:22:56,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:22:56,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +26: [2023-05-08 11:22:56,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +26: [2023-05-08 11:22:56,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +24: [2023-05-08 11:22:56,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:22:56,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +11: [2023-05-08 11:22:56,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +11: [2023-05-08 11:22:56,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +11: [2023-05-08 11:22:56,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +11: [2023-05-08 11:22:56,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +11: [2023-05-08 11:22:56,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +11: [2023-05-08 11:22:56,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +11: [2023-05-08 11:22:56,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +11: [2023-05-08 11:22:56,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +11: [2023-05-08 11:22:56,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +11: [2023-05-08 11:22:56,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +24: [2023-05-08 11:22:56,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:22:56,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:22:56,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:22:56,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +26: [2023-05-08 11:22:56,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:22:56,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +25: [2023-05-08 11:22:56,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +25: [2023-05-08 11:22:56,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +25: [2023-05-08 11:22:56,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +25: [2023-05-08 11:22:56,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:22:56,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:22:56,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +25: [2023-05-08 11:22:56,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:22:56,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +11: [2023-05-08 11:22:56,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +11: [2023-05-08 11:22:56,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +11: [2023-05-08 11:22:56,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +11: [2023-05-08 11:22:56,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +26: [2023-05-08 11:22:56,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:22:56,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:22:56,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +26: [2023-05-08 11:22:56,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +15: [2023-05-08 11:22:56,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +15: [2023-05-08 11:22:56,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +15: [2023-05-08 11:22:56,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +15: [2023-05-08 11:22:56,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +31: [2023-05-08 11:22:56,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +31: [2023-05-08 11:22:56,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +31: [2023-05-08 11:22:56,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:22:56,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +31: [2023-05-08 11:22:56,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +31: [2023-05-08 11:22:56,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:22:56,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:22:56,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:22:56,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:22:56,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:22:56,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:22:56,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:22:56,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +11: [2023-05-08 11:22:56,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:22:56,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:22:56,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:22:56,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:22:56,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +25: [2023-05-08 11:22:56,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +11: [2023-05-08 11:22:56,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +11: [2023-05-08 11:22:56,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:22:56,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:22:56,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:22:56,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:22:56,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +31: [2023-05-08 11:22:56,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:22:56,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +15: [2023-05-08 11:22:56,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 8: [2023-05-08 11:22:56,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:22:56,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:22:56,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:22:56,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +25: [2023-05-08 11:22:56,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +31: [2023-05-08 11:22:56,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:22:56,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:22:56,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +15: [2023-05-08 11:22:56,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +15: [2023-05-08 11:22:56,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +15: [2023-05-08 11:22:56,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +25: [2023-05-08 11:22:56,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:22:56,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:22:56,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:22:56,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +15: [2023-05-08 11:22:56,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 8: [2023-05-08 11:22:56,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 2: [2023-05-08 11:22:56,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 7: [2023-05-08 11:22:56,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 2: [2023-05-08 11:22:56,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +31: [2023-05-08 11:22:56,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +10: [2023-05-08 11:22:56,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +10: [2023-05-08 11:22:56,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +10: [2023-05-08 11:22:56,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +10: [2023-05-08 11:22:56,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 2: [2023-05-08 11:22:56,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +19: [2023-05-08 11:22:56,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:22:56,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:22:56,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +19: [2023-05-08 11:22:56,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +19: [2023-05-08 11:22:56,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +27: [2023-05-08 11:22:56,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +27: [2023-05-08 11:22:56,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +27: [2023-05-08 11:22:56,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:22:56,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:22:56,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +31: [2023-05-08 11:22:56,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +31: [2023-05-08 11:22:56,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +31: [2023-05-08 11:22:56,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +31: [2023-05-08 11:22:56,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +19: [2023-05-08 11:22:56,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:22:56,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +28: [2023-05-08 11:22:56,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +28: [2023-05-08 11:22:56,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +28: [2023-05-08 11:22:56,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 0: [2023-05-08 11:22:56,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:22:56,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:22:56,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:22:56,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:22:56,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 6: [2023-05-08 11:22:56,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 8: [2023-05-08 11:22:56,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +27: [2023-05-08 11:22:56,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +14: [2023-05-08 11:22:56,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +14: [2023-05-08 11:22:56,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +14: [2023-05-08 11:22:56,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +14: [2023-05-08 11:22:56,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 4: [2023-05-08 11:22:56,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 4: [2023-05-08 11:22:56,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 4: [2023-05-08 11:22:56,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 4: [2023-05-08 11:22:56,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +10: [2023-05-08 11:22:56,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +10: [2023-05-08 11:22:56,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:22:56,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +25: [2023-05-08 11:22:56,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +25: [2023-05-08 11:22:56,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +10: [2023-05-08 11:22:56,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +10: [2023-05-08 11:22:56,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:22:56,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 8: [2023-05-08 11:22:56,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:22:56,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +12: [2023-05-08 11:22:56,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +27: [2023-05-08 11:22:56,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +27: [2023-05-08 11:22:56,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +27: [2023-05-08 11:22:56,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +12: [2023-05-08 11:22:56,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +12: [2023-05-08 11:22:56,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +27: [2023-05-08 11:22:56,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 0: [2023-05-08 11:22:56,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 0: [2023-05-08 11:22:56,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 0: [2023-05-08 11:22:56,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 2: [2023-05-08 11:22:56,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +31: [2023-05-08 11:22:56,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:22:56,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:22:56,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:22:56,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 9: [2023-05-08 11:22:56,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 4: [2023-05-08 11:22:56,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 5: [2023-05-08 11:22:56,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 9: [2023-05-08 11:22:56,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 5: [2023-05-08 11:22:56,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +28: [2023-05-08 11:22:56,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +28: [2023-05-08 11:22:56,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:22:56,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:22:56,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +28: [2023-05-08 11:22:56,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:22:56,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 1: [2023-05-08 11:22:56,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 4: [2023-05-08 11:22:56,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +24: [2023-05-08 11:22:56,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +24: [2023-05-08 11:22:56,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 5: [2023-05-08 11:22:56,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +14: [2023-05-08 11:22:56,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +14: [2023-05-08 11:22:56,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +24: [2023-05-08 11:22:56,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 5: [2023-05-08 11:22:56,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +24: [2023-05-08 11:22:56,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. + 1: [2023-05-08 11:22:56,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +25: [2023-05-08 11:22:56,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:22:56,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:22:56,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:22:56,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +25: [2023-05-08 11:22:56,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:22:56,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:22:56,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:22:56,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:22:56,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:22:56,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:22:56,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +27: [2023-05-08 11:22:56,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:22:56,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:22:56,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +10: [2023-05-08 11:22:56,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:22:56,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +10: [2023-05-08 11:22:56,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +10: [2023-05-08 11:22:56,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:22:56,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +14: [2023-05-08 11:22:56,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:22:56,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +14: [2023-05-08 11:22:56,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:22:56,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +18: [2023-05-08 11:22:56,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +18: [2023-05-08 11:22:56,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +27: [2023-05-08 11:22:56,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +10: [2023-05-08 11:22:56,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:22:56,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +10: [2023-05-08 11:22:56,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +18: [2023-05-08 11:22:56,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +18: [2023-05-08 11:22:56,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt. +12: [2023-05-08 11:22:56,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:22:56,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:22:56,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:22:56,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:22:56,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +24: [2023-05-08 11:22:56,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +24: [2023-05-08 11:22:56,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +24: [2023-05-08 11:22:56,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:22:56,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 7: [2023-05-08 11:22:56,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 4: [2023-05-08 11:22:56,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 0: [2023-05-08 11:22:56,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:22:56,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:22:56,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:22:56,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +31: [2023-05-08 11:22:56,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 1: [2023-05-08 11:22:56,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:22:56,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:22:56,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +12: [2023-05-08 11:22:56,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:22:56,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +12: [2023-05-08 11:22:56,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 2: [2023-05-08 11:22:56,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:22:56,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 9: [2023-05-08 11:22:56,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +10: [2023-05-08 11:22:56,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:22:56,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +10: [2023-05-08 11:22:56,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +31: [2023-05-08 11:22:56,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:22:56,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +27: [2023-05-08 11:22:56,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 1: [2023-05-08 11:22:56,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 4: [2023-05-08 11:22:56,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +25: [2023-05-08 11:22:56,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +25: [2023-05-08 11:22:56,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +25: [2023-05-08 11:22:56,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +25: [2023-05-08 11:22:56,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +18: [2023-05-08 11:22:56,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +18: [2023-05-08 11:22:56,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:22:56,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:22:56,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +18: [2023-05-08 11:22:56,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:22:56,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +12: [2023-05-08 11:22:56,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +28: [2023-05-08 11:22:56,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +28: [2023-05-08 11:22:56,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +27: [2023-05-08 11:22:56,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +14: [2023-05-08 11:22:56,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:22:56,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +12: [2023-05-08 11:22:56,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +10: [2023-05-08 11:22:56,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +10: [2023-05-08 11:22:56,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +14: [2023-05-08 11:22:56,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +10: [2023-05-08 11:22:56,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +18: [2023-05-08 11:22:56,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 6: [2023-05-08 11:22:56,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +19: [2023-05-08 11:22:56,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +25: [2023-05-08 11:22:56,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +25: [2023-05-08 11:22:56,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +25: [2023-05-08 11:22:56,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +25: [2023-05-08 11:22:56,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 2: [2023-05-08 11:22:56,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +24: [2023-05-08 11:22:56,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:22:56,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +27: [2023-05-08 11:22:56,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +27: [2023-05-08 11:22:56,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +24: [2023-05-08 11:22:56,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:22:56,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +12: [2023-05-08 11:22:56,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +12: [2023-05-08 11:22:56,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +10: [2023-05-08 11:22:56,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 0: [2023-05-08 11:22:56,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +28: [2023-05-08 11:22:56,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +28: [2023-05-08 11:22:56,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +10: [2023-05-08 11:22:56,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:22:56,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +28: [2023-05-08 11:22:56,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 4: [2023-05-08 11:22:56,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +10: [2023-05-08 11:22:56,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +19: [2023-05-08 11:22:56,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:22:56,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:22:56,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:22:56,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 9: [2023-05-08 11:22:56,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 4: [2023-05-08 11:22:56,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +27: [2023-05-08 11:22:56,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:22:56,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 5: [2023-05-08 11:22:56,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 5: [2023-05-08 11:22:56,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 1: [2023-05-08 11:22:56,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +24: [2023-05-08 11:22:56,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:22:56,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +14: [2023-05-08 11:22:56,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +24: [2023-05-08 11:22:56,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:22:56,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +27: [2023-05-08 11:22:56,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:22:56,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +14: [2023-05-08 11:22:56,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:22:56,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +14: [2023-05-08 11:22:56,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:22:56,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:22:56,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +14: [2023-05-08 11:22:56,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 5: [2023-05-08 11:22:56,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:22:56,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +18: [2023-05-08 11:22:56,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +10: [2023-05-08 11:22:56,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:22:56,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:22:56,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:22:56,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:22:56,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:22:56,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:22:56,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +18: [2023-05-08 11:22:56,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:22:56,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +14: [2023-05-08 11:22:56,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:22:56,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:22:56,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 0: [2023-05-08 11:22:56,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 0: [2023-05-08 11:22:56,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 0: [2023-05-08 11:22:56,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 0: [2023-05-08 11:22:56,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:22:56,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:22:56,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:22:56,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:22:56,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 1: [2023-05-08 11:22:56,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +27: [2023-05-08 11:22:56,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:22:56,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:22:56,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +18: [2023-05-08 11:22:56,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 9: [2023-05-08 11:22:56,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +18: [2023-05-08 11:22:56,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:22:56,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +10: [2023-05-08 11:22:56,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +27: [2023-05-08 11:22:56,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 9: [2023-05-08 11:22:56,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +14: [2023-05-08 11:22:56,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... +14: [2023-05-08 11:22:56,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:22:56,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:22:56,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +14: [2023-05-08 11:22:56,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt. +10: [2023-05-08 11:22:56,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:22:56,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +14: [2023-05-08 11:22:56,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt... + 0: [2023-05-08 11:22:56,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:22:56,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +10: [2023-05-08 11:22:56,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:22:56,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:22:56,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:22:56,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:22:56,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:22:56,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:22:56,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:22:56,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:22:56,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:22:56,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:22:56,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:22:56,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:22:56,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:22:56,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:22:56,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:22:56,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:22:56,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:22:56,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:22:56,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:22:56,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:22:56,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:22:56,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:22:56,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:22:56,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +24: [2023-05-08 11:22:56,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:22:56,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:22:56,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +24: [2023-05-08 11:22:56,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +24: [2023-05-08 11:22:56,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:22:56,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +24: [2023-05-08 11:22:56,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:22:56,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:22:56,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:22:56,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:22:56,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:22:56,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:22:56,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:22:56,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:22:56,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +28: [2023-05-08 11:22:56,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:22:56,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:22:56,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:22:56,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +13: [2023-05-08 11:22:56,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:22:56,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +13: [2023-05-08 11:22:56,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:22:56,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:22:56,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:22:56,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:22:56,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:22:56,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:22:56,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:22:56,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:22:56,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:22:56,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:22:56,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:22:56,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +24: [2023-05-08 11:22:56,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:22:56,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:22:56,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:22:56,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:22:56,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +24: [2023-05-08 11:22:56,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:22:56,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 2: [2023-05-08 11:22:56,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 2: [2023-05-08 11:22:56,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 2: [2023-05-08 11:22:56,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +29: [2023-05-08 11:22:56,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:22:56,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +24: [2023-05-08 11:22:56,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:22:56,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:22:56,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:22:56,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:22:56,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:22:56,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:22:56,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:22:56,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:22:56,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:22:56,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +22: [2023-05-08 11:22:56,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:22:56,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:22:56,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:22:56,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +28: [2023-05-08 11:22:56,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:22:56,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +20: [2023-05-08 11:22:56,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +20: [2023-05-08 11:22:56,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:22:56,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:22:56,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:22:56,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:22:56,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:22:56,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +18: [2023-05-08 11:22:56,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +18: [2023-05-08 11:22:56,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +18: [2023-05-08 11:22:56,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +18: [2023-05-08 11:22:56,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +12: [2023-05-08 11:22:56,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:22:56,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:22:56,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +12: [2023-05-08 11:22:56,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +12: [2023-05-08 11:22:56,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:22:56,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:22:56,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +18: [2023-05-08 11:22:56,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +18: [2023-05-08 11:22:56,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:22:56,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:22:56,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:22:56,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:22:56,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +22: [2023-05-08 11:22:56,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +12: [2023-05-08 11:22:56,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +12: [2023-05-08 11:22:56,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:22:56,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +22: [2023-05-08 11:22:56,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:22:56,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:22:56,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +22: [2023-05-08 11:22:56,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:22:56,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +18: [2023-05-08 11:22:56,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +23: [2023-05-08 11:22:56,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +18: [2023-05-08 11:22:56,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:22:56,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +30: [2023-05-08 11:22:56,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +30: [2023-05-08 11:22:56,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 9: [2023-05-08 11:22:56,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +30: [2023-05-08 11:22:56,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +30: [2023-05-08 11:22:56,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 2: [2023-05-08 11:22:56,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:22:56,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 2: [2023-05-08 11:22:56,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 9: [2023-05-08 11:22:56,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:22:56,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 2: [2023-05-08 11:22:56,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 1: [2023-05-08 11:22:56,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +28: [2023-05-08 11:22:56,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +18: [2023-05-08 11:22:56,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +18: [2023-05-08 11:22:56,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:22:56,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:22:56,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +23: [2023-05-08 11:22:56,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +28: [2023-05-08 11:22:56,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +28: [2023-05-08 11:22:56,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +23: [2023-05-08 11:22:56,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:22:56,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +23: [2023-05-08 11:22:56,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:22:56,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +23: [2023-05-08 11:22:56,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:22:56,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +30: [2023-05-08 11:22:56,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +20: [2023-05-08 11:22:56,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:22:56,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:22:56,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 3: [2023-05-08 11:22:56,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 3: [2023-05-08 11:22:56,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 3: [2023-05-08 11:22:56,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 4: [2023-05-08 11:22:56,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +18: [2023-05-08 11:22:56,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:22:56,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:22:56,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +18: [2023-05-08 11:22:56,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:22:56,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:22:56,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 4: [2023-05-08 11:22:56,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:22:56,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:22:56,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 1: [2023-05-08 11:22:56,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:22:56,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:22:56,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +20: [2023-05-08 11:22:56,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +12: [2023-05-08 11:22:56,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:22:56,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:22:56,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:22:56,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:22:56,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +18: [2023-05-08 11:22:56,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +20: [2023-05-08 11:22:56,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +20: [2023-05-08 11:22:56,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +23: [2023-05-08 11:22:56,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:22:56,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 7: [2023-05-08 11:22:56,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +12: [2023-05-08 11:22:56,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:22:56,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 4: [2023-05-08 11:22:56,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +18: [2023-05-08 11:22:56,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:22:56,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +16: [2023-05-08 11:22:56,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +16: [2023-05-08 11:22:56,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +30: [2023-05-08 11:22:56,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:22:56,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:22:56,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:22:56,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +30: [2023-05-08 11:22:56,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +30: [2023-05-08 11:22:56,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +26: [2023-05-08 11:22:56,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:22:56,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:22:56,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +18: [2023-05-08 11:22:56,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +30: [2023-05-08 11:22:56,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +23: [2023-05-08 11:22:56,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:22:56,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +19: [2023-05-08 11:22:56,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +19: [2023-05-08 11:22:56,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +19: [2023-05-08 11:22:56,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +16: [2023-05-08 11:22:56,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:22:56,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:22:56,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +16: [2023-05-08 11:22:56,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:22:56,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 3: [2023-05-08 11:22:56,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +30: [2023-05-08 11:22:56,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +30: [2023-05-08 11:22:56,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 7: [2023-05-08 11:22:56,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:22:56,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:22:56,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +29: [2023-05-08 11:22:56,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +29: [2023-05-08 11:22:56,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 1: [2023-05-08 11:22:56,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:22:56,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 3: [2023-05-08 11:22:56,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +18: [2023-05-08 11:22:56,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +30: [2023-05-08 11:22:56,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +19: [2023-05-08 11:22:56,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +19: [2023-05-08 11:22:56,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +19: [2023-05-08 11:22:56,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +19: [2023-05-08 11:22:56,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:22:56,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:22:56,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +16: [2023-05-08 11:22:56,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:22:56,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +29: [2023-05-08 11:22:56,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:22:56,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:22:56,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +13: [2023-05-08 11:22:56,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +13: [2023-05-08 11:22:56,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +13: [2023-05-08 11:22:56,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +13: [2023-05-08 11:22:56,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +16: [2023-05-08 11:22:56,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:22:56,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +29: [2023-05-08 11:22:56,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:22:56,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +26: [2023-05-08 11:22:56,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +26: [2023-05-08 11:22:56,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +26: [2023-05-08 11:22:56,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 3: [2023-05-08 11:22:56,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:22:56,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:22:56,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 3: [2023-05-08 11:22:56,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 3: [2023-05-08 11:22:56,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +13: [2023-05-08 11:22:56,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +13: [2023-05-08 11:22:56,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:22:56,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:22:56,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +13: [2023-05-08 11:22:56,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +13: [2023-05-08 11:22:56,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +16: [2023-05-08 11:22:56,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:22:56,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:22:56,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +16: [2023-05-08 11:22:56,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +29: [2023-05-08 11:22:56,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:22:56,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:22:56,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +29: [2023-05-08 11:22:56,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +19: [2023-05-08 11:22:56,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:22:56,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:22:56,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +19: [2023-05-08 11:22:56,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +19: [2023-05-08 11:22:56,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:22:56,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:22:56,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +19: [2023-05-08 11:22:56,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +21: [2023-05-08 11:22:56,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +21: [2023-05-08 11:22:56,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +21: [2023-05-08 11:22:56,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +21: [2023-05-08 11:22:56,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +21: [2023-05-08 11:22:56,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +21: [2023-05-08 11:22:56,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +21: [2023-05-08 11:22:56,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +21: [2023-05-08 11:22:56,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:22:56,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:22:56,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +13: [2023-05-08 11:22:56,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:22:56,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +29: [2023-05-08 11:22:56,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:22:56,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +17: [2023-05-08 11:22:56,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +17: [2023-05-08 11:22:56,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +17: [2023-05-08 11:22:56,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +17: [2023-05-08 11:22:56,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +26: [2023-05-08 11:22:56,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:22:56,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +21: [2023-05-08 11:22:56,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:22:56,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +21: [2023-05-08 11:22:56,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:22:56,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:22:56,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +13: [2023-05-08 11:22:56,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:22:56,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:22:56,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:22:56,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +21: [2023-05-08 11:22:56,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:22:56,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +21: [2023-05-08 11:22:56,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:22:56,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +26: [2023-05-08 11:22:56,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +17: [2023-05-08 11:22:56,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +17: [2023-05-08 11:22:56,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +17: [2023-05-08 11:22:56,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +17: [2023-05-08 11:22:56,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:22:56,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:22:56,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:22:56,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:22:56,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:22:56,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:22:56,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:22:56,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:22:56,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:22:56,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +21: [2023-05-08 11:22:56,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:22:56,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:22:56,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:22:56,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:22:56,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 3: [2023-05-08 11:22:56,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:22:56,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:22:56,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:22:56,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:22:56,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +17: [2023-05-08 11:22:56,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:22:56,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:22:56,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +17: [2023-05-08 11:22:56,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +17: [2023-05-08 11:22:56,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:22:56,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:22:56,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:22:56,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +25: [2023-05-08 11:22:56,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +25: [2023-05-08 11:22:56,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +25: [2023-05-08 11:22:56,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +11: [2023-05-08 11:22:56,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +11: [2023-05-08 11:22:56,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +25: [2023-05-08 11:22:56,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 3: [2023-05-08 11:22:56,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +11: [2023-05-08 11:22:56,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 3: [2023-05-08 11:22:56,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:22:56,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:22:56,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +15: [2023-05-08 11:22:56,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +15: [2023-05-08 11:22:56,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +15: [2023-05-08 11:22:56,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 3: [2023-05-08 11:22:56,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +15: [2023-05-08 11:22:56,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +17: [2023-05-08 11:22:56,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:22:56,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:22:56,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +23: [2023-05-08 11:22:56,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +23: [2023-05-08 11:22:56,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +23: [2023-05-08 11:22:56,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +23: [2023-05-08 11:22:56,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +23: [2023-05-08 11:22:56,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:22:56,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:22:56,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +23: [2023-05-08 11:22:56,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +25: [2023-05-08 11:22:56,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:22:56,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +11: [2023-05-08 11:22:56,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +11: [2023-05-08 11:22:56,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 7: [2023-05-08 11:22:56,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 7: [2023-05-08 11:22:56,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +25: [2023-05-08 11:22:56,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +11: [2023-05-08 11:22:56,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:22:56,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:22:56,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:22:56,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:22:56,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +11: [2023-05-08 11:22:56,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:22:56,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +12: [2023-05-08 11:22:56,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +12: [2023-05-08 11:22:56,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +12: [2023-05-08 11:22:56,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +30: [2023-05-08 11:22:56,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +30: [2023-05-08 11:22:56,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +30: [2023-05-08 11:22:56,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +12: [2023-05-08 11:22:56,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +30: [2023-05-08 11:22:56,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +30: [2023-05-08 11:22:56,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +30: [2023-05-08 11:22:56,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +20: [2023-05-08 11:22:56,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +20: [2023-05-08 11:22:56,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +20: [2023-05-08 11:22:56,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +30: [2023-05-08 11:22:56,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +30: [2023-05-08 11:22:56,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +20: [2023-05-08 11:22:56,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 6: [2023-05-08 11:22:56,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 6: [2023-05-08 11:22:56,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 6: [2023-05-08 11:22:56,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 6: [2023-05-08 11:22:56,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +19: [2023-05-08 11:22:56,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:22:56,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:22:56,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:22:56,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +19: [2023-05-08 11:22:56,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +19: [2023-05-08 11:22:56,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +19: [2023-05-08 11:22:56,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:22:56,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:22:56,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:22:56,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +23: [2023-05-08 11:22:56,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:22:56,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +12: [2023-05-08 11:22:56,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:22:56,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +25: [2023-05-08 11:22:56,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:22:56,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +12: [2023-05-08 11:22:56,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 5: [2023-05-08 11:22:56,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +25: [2023-05-08 11:22:56,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 5: [2023-05-08 11:22:56,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +25: [2023-05-08 11:22:56,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +23: [2023-05-08 11:22:56,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +30: [2023-05-08 11:22:56,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +12: [2023-05-08 11:22:56,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +11: [2023-05-08 11:22:56,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +11: [2023-05-08 11:22:56,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 6: [2023-05-08 11:22:56,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +20: [2023-05-08 11:22:56,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:22:56,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:22:56,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:22:56,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +15: [2023-05-08 11:22:56,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 6: [2023-05-08 11:22:56,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +20: [2023-05-08 11:22:56,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +23: [2023-05-08 11:22:56,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 9: [2023-05-08 11:22:56,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 9: [2023-05-08 11:22:56,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +20: [2023-05-08 11:22:56,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 6: [2023-05-08 11:22:56,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +11: [2023-05-08 11:22:56,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +11: [2023-05-08 11:22:56,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +23: [2023-05-08 11:22:56,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 0: [2023-05-08 11:22:56,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 0: [2023-05-08 11:22:56,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +23: [2023-05-08 11:22:56,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +30: [2023-05-08 11:22:56,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:22:56,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:22:56,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:22:56,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +31: [2023-05-08 11:22:56,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +31: [2023-05-08 11:22:56,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 5: [2023-05-08 11:22:56,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:22:56,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +30: [2023-05-08 11:22:56,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +23: [2023-05-08 11:22:56,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:22:56,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +12: [2023-05-08 11:22:56,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:22:56,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 5: [2023-05-08 11:22:56,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +11: [2023-05-08 11:22:56,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +11: [2023-05-08 11:22:56,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +15: [2023-05-08 11:22:56,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +25: [2023-05-08 11:22:56,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:22:56,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +25: [2023-05-08 11:22:56,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:22:56,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:22:56,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +25: [2023-05-08 11:22:56,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +25: [2023-05-08 11:22:56,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +25: [2023-05-08 11:22:56,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +25: [2023-05-08 11:22:56,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:22:56,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 7: [2023-05-08 11:22:56,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +15: [2023-05-08 11:22:56,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +11: [2023-05-08 11:22:56,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +25: [2023-05-08 11:22:56,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:22:56,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +25: [2023-05-08 11:22:56,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +20: [2023-05-08 11:22:56,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +28: [2023-05-08 11:22:56,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 9: [2023-05-08 11:22:56,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +19: [2023-05-08 11:22:56,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:22:56,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:22:56,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:22:56,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:22:56,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +28: [2023-05-08 11:22:56,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +28: [2023-05-08 11:22:56,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +11: [2023-05-08 11:22:56,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +25: [2023-05-08 11:22:56,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 6: [2023-05-08 11:22:56,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +31: [2023-05-08 11:22:56,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:22:56,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:22:56,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +31: [2023-05-08 11:22:56,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:22:56,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:22:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +20: [2023-05-08 11:22:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:22:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 0: [2023-05-08 11:22:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:22:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:22:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:22:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:22:56,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 7: [2023-05-08 11:22:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:22:56,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +19: [2023-05-08 11:22:56,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:22:56,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +25: [2023-05-08 11:22:56,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 7: [2023-05-08 11:22:56,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 0: [2023-05-08 11:22:56,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +28: [2023-05-08 11:22:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:22:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +28: [2023-05-08 11:22:56,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +12: [2023-05-08 11:22:56,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:22:56,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +12: [2023-05-08 11:22:56,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 9: [2023-05-08 11:22:56,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:22:56,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +19: [2023-05-08 11:22:56,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:22:56,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +20: [2023-05-08 11:22:56,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +31: [2023-05-08 11:22:56,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +31: [2023-05-08 11:22:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +25: [2023-05-08 11:22:56,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +25: [2023-05-08 11:22:56,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +25: [2023-05-08 11:22:56,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:22:56,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +25: [2023-05-08 11:22:56,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:22:56,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 9: [2023-05-08 11:22:56,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 5: [2023-05-08 11:22:56,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 5: [2023-05-08 11:22:56,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:22:56,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 6: [2023-05-08 11:22:56,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +20: [2023-05-08 11:22:56,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 0: [2023-05-08 11:22:56,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 1: [2023-05-08 11:22:56,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 1: [2023-05-08 11:22:56,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 6: [2023-05-08 11:22:56,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 1: [2023-05-08 11:22:56,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 1: [2023-05-08 11:22:56,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +31: [2023-05-08 11:22:56,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +25: [2023-05-08 11:22:56,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:22:56,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:22:56,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +31: [2023-05-08 11:22:56,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +28: [2023-05-08 11:22:56,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:22:56,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 9: [2023-05-08 11:22:56,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +25: [2023-05-08 11:22:56,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +10: [2023-05-08 11:22:56,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +10: [2023-05-08 11:22:56,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +10: [2023-05-08 11:22:56,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +10: [2023-05-08 11:22:56,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +31: [2023-05-08 11:22:56,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +31: [2023-05-08 11:22:56,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:22:56,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:22:56,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 1: [2023-05-08 11:22:56,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:22:56,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +31: [2023-05-08 11:22:56,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 1: [2023-05-08 11:22:56,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:22:56,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:22:56,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 1: [2023-05-08 11:22:56,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +10: [2023-05-08 11:22:56,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:22:56,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +10: [2023-05-08 11:22:56,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:22:56,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +14: [2023-05-08 11:22:56,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +10: [2023-05-08 11:22:56,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +14: [2023-05-08 11:22:56,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:22:56,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +14: [2023-05-08 11:22:56,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +10: [2023-05-08 11:22:56,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +14: [2023-05-08 11:22:56,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +14: [2023-05-08 11:22:56,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:22:56,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +14: [2023-05-08 11:22:56,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 1: [2023-05-08 11:22:56,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +10: [2023-05-08 11:22:56,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +10: [2023-05-08 11:22:56,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:22:56,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +10: [2023-05-08 11:22:56,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +10: [2023-05-08 11:22:56,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +10: [2023-05-08 11:22:56,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:22:56,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:22:56,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +18: [2023-05-08 11:22:56,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +18: [2023-05-08 11:22:56,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +18: [2023-05-08 11:22:56,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +18: [2023-05-08 11:22:56,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +14: [2023-05-08 11:22:56,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +14: [2023-05-08 11:22:56,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:22:56,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:22:56,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:22:56,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:22:56,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:22:56,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:22:56,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:22:56,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 8: [2023-05-08 11:22:56,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 8: [2023-05-08 11:22:56,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +15: [2023-05-08 11:22:56,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 2: [2023-05-08 11:22:56,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 8: [2023-05-08 11:22:56,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 2: [2023-05-08 11:22:56,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 8: [2023-05-08 11:22:56,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 1: [2023-05-08 11:22:56,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 1: [2023-05-08 11:22:56,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:22:56,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:22:56,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +10: [2023-05-08 11:22:56,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +27: [2023-05-08 11:22:56,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +27: [2023-05-08 11:22:56,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +24: [2023-05-08 11:22:56,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +24: [2023-05-08 11:22:56,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +24: [2023-05-08 11:22:56,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +24: [2023-05-08 11:22:56,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +27: [2023-05-08 11:22:56,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +22: [2023-05-08 11:22:56,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +22: [2023-05-08 11:22:56,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +22: [2023-05-08 11:22:56,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +22: [2023-05-08 11:22:56,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +18: [2023-05-08 11:22:56,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +18: [2023-05-08 11:22:56,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +14: [2023-05-08 11:22:56,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:22:56,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +14: [2023-05-08 11:22:56,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +18: [2023-05-08 11:22:56,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +18: [2023-05-08 11:22:56,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:22:56,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +10: [2023-05-08 11:22:56,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +14: [2023-05-08 11:22:56,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:22:56,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:22:56,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +15: [2023-05-08 11:22:56,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:22:56,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +10: [2023-05-08 11:22:56,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:22:56,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +24: [2023-05-08 11:22:56,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +24: [2023-05-08 11:22:56,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:22:56,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:22:56,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:22:56,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:22:56,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +14: [2023-05-08 11:22:56,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 5: [2023-05-08 11:22:56,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:22:56,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:22:56,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:22:56,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +14: [2023-05-08 11:22:56,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +14: [2023-05-08 11:22:56,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +14: [2023-05-08 11:22:56,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +14: [2023-05-08 11:22:56,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +31: [2023-05-08 11:22:56,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +31: [2023-05-08 11:22:56,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +31: [2023-05-08 11:22:56,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:22:56,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:22:56,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:22:56,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:22:56,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:22:56,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +31: [2023-05-08 11:22:56,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +31: [2023-05-08 11:22:56,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 4: [2023-05-08 11:22:56,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 4: [2023-05-08 11:22:56,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +27: [2023-05-08 11:22:56,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 4: [2023-05-08 11:22:56,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:22:56,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:22:56,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +14: [2023-05-08 11:22:56,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +15: [2023-05-08 11:22:56,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:22:56,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:22:56,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:22:56,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:22:56,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:22:56,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:22:56,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:22:56,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:22:56,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +18: [2023-05-08 11:22:56,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:22:56,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +18: [2023-05-08 11:22:56,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +10: [2023-05-08 11:22:56,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +27: [2023-05-08 11:22:56,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +18: [2023-05-08 11:22:56,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +18: [2023-05-08 11:22:56,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 2: [2023-05-08 11:22:56,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +18: [2023-05-08 11:22:56,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +27: [2023-05-08 11:22:56,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +27: [2023-05-08 11:22:56,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:22:56,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +10: [2023-05-08 11:22:56,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +27: [2023-05-08 11:22:56,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +18: [2023-05-08 11:22:56,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +27: [2023-05-08 11:22:56,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:22:56,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:22:56,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:22:56,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:22:56,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +27: [2023-05-08 11:22:56,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:22:56,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +16: [2023-05-08 11:22:56,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:22:56,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:22:56,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:22:56,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 5: [2023-05-08 11:22:56,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:22:56,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:22:56,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:22:56,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:22:56,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +24: [2023-05-08 11:22:56,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:22:56,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:22:56,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 2: [2023-05-08 11:22:56,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:22:56,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +10: [2023-05-08 11:22:56,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +10: [2023-05-08 11:22:56,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:22:56,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 8: [2023-05-08 11:22:56,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:22:56,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +24: [2023-05-08 11:22:56,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +24: [2023-05-08 11:22:56,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +14: [2023-05-08 11:22:56,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:22:56,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:22:56,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:22:56,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +14: [2023-05-08 11:22:56,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:22:56,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 0: [2023-05-08 11:22:56,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:22:56,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:22:56,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:22:56,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 6: [2023-05-08 11:22:56,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 6: [2023-05-08 11:22:56,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 6: [2023-05-08 11:22:56,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +18: [2023-05-08 11:22:56,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:22:56,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:22:56,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +21: [2023-05-08 11:22:56,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +21: [2023-05-08 11:22:56,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. +21: [2023-05-08 11:22:56,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt. + 0: [2023-05-08 11:22:57,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:22:57,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +12: [2023-05-08 11:22:57,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:22:57,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +12: [2023-05-08 11:22:57,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +12: [2023-05-08 11:22:57,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:22:57,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:22:57,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +12: [2023-05-08 11:22:57,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:22:57,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +18: [2023-05-08 11:22:57,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 2: [2023-05-08 11:22:57,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 5: [2023-05-08 11:22:57,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +31: [2023-05-08 11:22:57,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:22:57,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:22:57,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:22:57,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +28: [2023-05-08 11:22:57,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +28: [2023-05-08 11:22:57,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +31: [2023-05-08 11:22:57,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +31: [2023-05-08 11:22:57,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:22:57,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +28: [2023-05-08 11:22:57,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:22:57,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:22:57,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:22:57,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:22:57,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:22:57,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:22:57,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:22:57,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:22:57,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +11: [2023-05-08 11:22:57,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:22:57,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:22:57,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:22:57,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +31: [2023-05-08 11:22:57,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:22:57,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:22:57,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:22:57,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 8: [2023-05-08 11:22:57,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:22:57,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:22:57,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +24: [2023-05-08 11:22:57,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +22: [2023-05-08 11:22:57,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +14: [2023-05-08 11:22:57,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +27: [2023-05-08 11:22:57,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +22: [2023-05-08 11:22:57,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +27: [2023-05-08 11:22:57,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +27: [2023-05-08 11:22:57,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +27: [2023-05-08 11:22:57,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +27: [2023-05-08 11:22:57,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +21: [2023-05-08 11:22:57,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:22:57,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:22:57,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:22:57,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:22:57,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:22:57,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +31: [2023-05-08 11:22:57,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:22:57,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:22:57,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:22:57,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 6: [2023-05-08 11:22:57,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +31: [2023-05-08 11:22:57,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +22: [2023-05-08 11:22:57,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +12: [2023-05-08 11:22:57,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +12: [2023-05-08 11:22:57,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +21: [2023-05-08 11:22:57,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:22:57,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +22: [2023-05-08 11:22:57,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 4: [2023-05-08 11:22:57,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 9: [2023-05-08 11:22:57,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:22:57,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +28: [2023-05-08 11:22:57,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:22:57,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:22:57,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:22:57,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:22:57,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +12: [2023-05-08 11:22:57,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +22: [2023-05-08 11:22:57,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +11: [2023-05-08 11:22:57,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:22:57,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt... +11: [2023-05-08 11:22:57,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 6: [2023-05-08 11:22:57,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:22:57,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:22:57,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:22:57,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:22:57,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:22:57,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 4: [2023-05-08 11:22:57,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:22:57,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:22:57,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +16: [2023-05-08 11:22:57,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:22:57,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:22:57,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:22:57,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:22:57,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +14: [2023-05-08 11:22:57,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 8: [2023-05-08 11:22:57,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 8: [2023-05-08 11:22:57,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:22:57,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +16: [2023-05-08 11:22:57,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:22:57,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:22:57,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +28: [2023-05-08 11:22:57,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:22:57,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:22:57,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:22:57,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +27: [2023-05-08 11:22:57,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +27: [2023-05-08 11:22:57,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:22:57,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +27: [2023-05-08 11:22:57,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:22:57,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:22:57,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:22:57,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:22:57,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:22:57,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:22:57,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +28: [2023-05-08 11:22:57,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:22:57,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:22:57,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:22:57,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:22:57,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +13: [2023-05-08 11:22:57,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +13: [2023-05-08 11:22:57,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +16: [2023-05-08 11:22:57,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +13: [2023-05-08 11:22:57,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +13: [2023-05-08 11:22:57,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +16: [2023-05-08 11:22:57,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +16: [2023-05-08 11:22:57,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +27: [2023-05-08 11:22:57,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 9: [2023-05-08 11:22:57,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:22:57,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:22:57,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +19: [2023-05-08 11:22:57,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +19: [2023-05-08 11:22:57,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +19: [2023-05-08 11:22:57,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 4: [2023-05-08 11:22:57,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:22:57,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +29: [2023-05-08 11:22:57,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +29: [2023-05-08 11:22:57,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +21: [2023-05-08 11:22:57,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +21: [2023-05-08 11:22:57,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:22:57,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +17: [2023-05-08 11:22:57,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +30: [2023-05-08 11:22:57,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +30: [2023-05-08 11:22:57,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +30: [2023-05-08 11:22:57,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +17: [2023-05-08 11:22:57,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +17: [2023-05-08 11:22:57,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +30: [2023-05-08 11:22:57,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 4: [2023-05-08 11:22:57,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... + 9: [2023-05-08 11:22:57,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:22:57,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +21: [2023-05-08 11:22:57,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +29: [2023-05-08 11:22:57,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +21: [2023-05-08 11:22:57,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +21: [2023-05-08 11:22:57,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:22:57,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:22:57,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:22:57,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:22:57,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:22:57,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:22:57,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +19: [2023-05-08 11:22:57,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:22:57,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:22:57,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +19: [2023-05-08 11:22:57,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +19: [2023-05-08 11:22:57,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +21: [2023-05-08 11:22:57,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt... +19: [2023-05-08 11:22:57,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:22:57,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:22:57,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +30: [2023-05-08 11:22:57,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +30: [2023-05-08 11:22:57,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +30: [2023-05-08 11:22:57,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +30: [2023-05-08 11:22:57,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:22:57,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:22:57,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:22:57,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:22:57,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:22:57,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:22:57,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:22:57,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:22:57,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:22:57,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:22:57,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +22: [2023-05-08 11:22:57,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:22:57,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:22:57,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:22:57,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +22: [2023-05-08 11:22:57,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +22: [2023-05-08 11:22:57,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:22:57,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +13: [2023-05-08 11:22:57,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:22:57,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:22:57,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +13: [2023-05-08 11:22:57,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +13: [2023-05-08 11:22:57,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +16: [2023-05-08 11:22:57,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +30: [2023-05-08 11:22:57,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:22:57,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +30: [2023-05-08 11:22:57,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +30: [2023-05-08 11:22:57,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +24: [2023-05-08 11:22:57,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +24: [2023-05-08 11:22:57,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +24: [2023-05-08 11:22:57,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +24: [2023-05-08 11:22:57,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:22:57,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:22:57,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +13: [2023-05-08 11:22:57,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:22:57,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +24: [2023-05-08 11:22:57,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +24: [2023-05-08 11:22:57,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:22:57,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:22:57,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:22:57,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:22:57,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +19: [2023-05-08 11:22:57,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +19: [2023-05-08 11:22:57,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +26: [2023-05-08 11:22:57,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +26: [2023-05-08 11:22:57,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +24: [2023-05-08 11:22:57,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +13: [2023-05-08 11:22:57,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:22:57,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +13: [2023-05-08 11:22:57,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +13: [2023-05-08 11:22:57,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +13: [2023-05-08 11:22:57,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +13: [2023-05-08 11:22:57,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +13: [2023-05-08 11:22:57,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:22:57,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:22:57,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +17: [2023-05-08 11:22:57,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +30: [2023-05-08 11:22:57,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +16: [2023-05-08 11:22:57,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:22:57,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +17: [2023-05-08 11:22:57,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +13: [2023-05-08 11:22:57,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +26: [2023-05-08 11:22:57,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +13: [2023-05-08 11:22:57,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:22:57,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:22:57,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +26: [2023-05-08 11:22:57,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +16: [2023-05-08 11:22:57,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:22:57,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +22: [2023-05-08 11:22:57,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +17: [2023-05-08 11:22:57,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:22:57,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:22:57,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +20: [2023-05-08 11:22:57,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:22:57,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +20: [2023-05-08 11:22:57,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +20: [2023-05-08 11:22:57,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:22:57,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:22:57,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +20: [2023-05-08 11:22:57,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +20: [2023-05-08 11:22:57,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:22:57,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +18: [2023-05-08 11:22:57,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +30: [2023-05-08 11:22:57,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +18: [2023-05-08 11:22:57,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +30: [2023-05-08 11:22:57,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +18: [2023-05-08 11:22:57,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +18: [2023-05-08 11:22:57,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +18: [2023-05-08 11:22:57,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +18: [2023-05-08 11:22:57,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +18: [2023-05-08 11:22:57,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:22:57,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:22:57,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +17: [2023-05-08 11:22:57,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:22:57,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:22:57,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +17: [2023-05-08 11:22:57,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +29: [2023-05-08 11:22:57,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:22:57,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:22:57,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +29: [2023-05-08 11:22:57,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +18: [2023-05-08 11:22:57,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:22:57,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:22:57,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +24: [2023-05-08 11:22:57,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +13: [2023-05-08 11:22:57,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:22:57,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:22:57,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:22:57,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +24: [2023-05-08 11:22:57,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +24: [2023-05-08 11:22:57,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:22:57,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:22:57,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 3: [2023-05-08 11:22:57,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 3: [2023-05-08 11:22:57,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 3: [2023-05-08 11:22:57,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 7: [2023-05-08 11:22:57,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +20: [2023-05-08 11:22:57,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:22:57,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:22:57,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +18: [2023-05-08 11:22:57,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:22:57,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:22:57,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +13: [2023-05-08 11:22:57,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +24: [2023-05-08 11:22:57,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:22:57,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +18: [2023-05-08 11:22:57,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:22:57,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:22:57,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:22:57,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +24: [2023-05-08 11:22:57,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +20: [2023-05-08 11:22:57,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +24: [2023-05-08 11:22:57,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +24: [2023-05-08 11:22:57,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:22:57,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:22:57,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:22:57,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +26: [2023-05-08 11:22:57,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:22:57,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +26: [2023-05-08 11:22:57,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +26: [2023-05-08 11:22:57,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:22:57,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:22:57,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:22:57,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:22:57,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:22:57,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:22:57,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +26: [2023-05-08 11:22:57,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:22:57,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:22:57,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +13: [2023-05-08 11:22:57,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:22:57,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +18: [2023-05-08 11:22:57,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +18: [2023-05-08 11:22:57,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +18: [2023-05-08 11:22:57,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +18: [2023-05-08 11:22:57,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +26: [2023-05-08 11:22:57,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:22:57,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +26: [2023-05-08 11:22:57,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 7: [2023-05-08 11:22:57,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:22:57,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:22:57,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:22:57,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:22:57,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:22:57,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:22:57,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:22:57,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:22:57,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:22:57,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +23: [2023-05-08 11:22:57,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +23: [2023-05-08 11:22:57,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +20: [2023-05-08 11:22:57,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:22:57,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +20: [2023-05-08 11:22:57,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +13: [2023-05-08 11:22:57,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +13: [2023-05-08 11:22:57,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:22:57,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +18: [2023-05-08 11:22:57,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 7: [2023-05-08 11:22:57,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:22:57,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:22:57,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:22:57,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:22:57,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:22:57,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:22:57,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:22:57,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:22:57,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:22:57,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:22:57,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:22:57,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:22:57,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:22:57,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:22:57,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:22:57,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 3: [2023-05-08 11:22:57,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:22:57,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:22:57,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:22:57,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +23: [2023-05-08 11:22:57,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 3: [2023-05-08 11:22:57,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +26: [2023-05-08 11:22:57,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:22:57,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:22:57,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:22:57,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +21: [2023-05-08 11:22:57,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:22:57,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +25: [2023-05-08 11:22:57,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:22:57,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:22:57,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:22:57,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:22:57,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +25: [2023-05-08 11:22:57,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:22:57,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +26: [2023-05-08 11:22:57,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:22:57,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:22:57,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:22:57,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:22:57,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +23: [2023-05-08 11:22:57,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:22:57,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:22:57,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:22:57,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +26: [2023-05-08 11:22:57,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:22:57,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +21: [2023-05-08 11:22:57,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:22:57,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:22:57,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +23: [2023-05-08 11:22:57,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +23: [2023-05-08 11:22:57,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +23: [2023-05-08 11:22:57,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +25: [2023-05-08 11:22:57,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:22:57,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +21: [2023-05-08 11:22:57,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:22:57,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +30: [2023-05-08 11:22:57,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +30: [2023-05-08 11:22:57,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:22:57,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +30: [2023-05-08 11:22:57,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +30: [2023-05-08 11:22:57,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:22:57,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +30: [2023-05-08 11:22:57,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:22:57,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:22:57,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:22:57,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:22:57,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +15: [2023-05-08 11:22:57,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +15: [2023-05-08 11:22:57,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +15: [2023-05-08 11:22:57,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +21: [2023-05-08 11:22:57,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:22:57,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +30: [2023-05-08 11:22:57,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:22:57,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:22:57,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +17: [2023-05-08 11:22:57,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:22:57,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +17: [2023-05-08 11:22:57,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +17: [2023-05-08 11:22:57,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:22:57,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:22:57,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +15: [2023-05-08 11:22:57,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:22:57,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:22:57,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +15: [2023-05-08 11:22:57,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:22:57,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 3: [2023-05-08 11:22:57,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:22:57,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +30: [2023-05-08 11:22:57,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +30: [2023-05-08 11:22:57,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +30: [2023-05-08 11:22:57,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:22:57,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:22:57,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:22:57,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:22:57,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:22:57,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:22:57,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:22:57,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:22:57,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:22:57,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:22:57,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +30: [2023-05-08 11:22:57,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +30: [2023-05-08 11:22:57,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:22:57,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:22:57,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +17: [2023-05-08 11:22:57,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +17: [2023-05-08 11:22:57,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +15: [2023-05-08 11:22:57,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:22:57,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +15: [2023-05-08 11:22:57,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:22:57,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 6: [2023-05-08 11:22:57,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +10: [2023-05-08 11:22:57,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:22:57,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +10: [2023-05-08 11:22:57,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:22:57,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:22:57,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:22:57,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:22:57,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +15: [2023-05-08 11:22:57,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:22:57,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +10: [2023-05-08 11:22:57,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +10: [2023-05-08 11:22:57,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:22:57,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +17: [2023-05-08 11:22:57,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +30: [2023-05-08 11:22:57,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +17: [2023-05-08 11:22:57,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +17: [2023-05-08 11:22:57,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:22:57,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:22:57,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:22:57,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +11: [2023-05-08 11:22:57,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:22:57,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:22:57,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:22:57,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:22:57,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +11: [2023-05-08 11:22:57,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +11: [2023-05-08 11:22:57,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 5: [2023-05-08 11:22:57,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 5: [2023-05-08 11:22:57,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 5: [2023-05-08 11:22:57,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +30: [2023-05-08 11:22:57,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:22:57,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:22:57,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:22:57,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +17: [2023-05-08 11:22:57,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +19: [2023-05-08 11:22:57,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:22:57,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:22:57,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:22:57,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:22:57,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:22:57,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:22:57,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:22:57,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:22:57,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +10: [2023-05-08 11:22:57,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 5: [2023-05-08 11:22:57,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 5: [2023-05-08 11:22:57,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +11: [2023-05-08 11:22:57,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:22:57,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:22:57,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +10: [2023-05-08 11:22:57,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:22:57,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:22:57,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:22:57,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:22:57,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +28: [2023-05-08 11:22:57,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +28: [2023-05-08 11:22:57,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +28: [2023-05-08 11:22:57,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +28: [2023-05-08 11:22:57,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 2: [2023-05-08 11:22:57,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:22:57,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +12: [2023-05-08 11:22:57,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +12: [2023-05-08 11:22:57,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +12: [2023-05-08 11:22:57,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 2: [2023-05-08 11:22:57,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +28: [2023-05-08 11:22:57,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +28: [2023-05-08 11:22:57,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +28: [2023-05-08 11:22:57,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +15: [2023-05-08 11:22:57,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +15: [2023-05-08 11:22:57,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +15: [2023-05-08 11:22:57,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +15: [2023-05-08 11:22:57,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +15: [2023-05-08 11:22:57,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +15: [2023-05-08 11:22:57,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:22:57,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:22:57,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +28: [2023-05-08 11:22:57,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:22:57,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +15: [2023-05-08 11:22:57,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:22:57,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:22:57,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:22:57,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:22:57,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:22:57,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +16: [2023-05-08 11:22:57,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +16: [2023-05-08 11:22:57,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +16: [2023-05-08 11:22:57,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +16: [2023-05-08 11:22:57,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:22:57,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +16: [2023-05-08 11:22:57,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:22:57,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 1: [2023-05-08 11:22:57,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +14: [2023-05-08 11:22:57,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:22:57,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:22:57,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:22:57,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +27: [2023-05-08 11:22:57,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:22:57,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:22:57,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 0: [2023-05-08 11:22:57,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +20: [2023-05-08 11:22:57,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +20: [2023-05-08 11:22:57,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:22:57,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:22:57,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:22:57,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:22:57,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +15: [2023-05-08 11:22:57,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +28: [2023-05-08 11:22:57,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:22:57,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:22:57,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:22:57,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:22:57,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 4: [2023-05-08 11:22:57,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:22:57,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:22:57,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:22:57,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:22:57,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +15: [2023-05-08 11:22:57,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +31: [2023-05-08 11:22:57,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:22:57,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:22:57,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +20: [2023-05-08 11:22:57,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +31: [2023-05-08 11:22:57,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +31: [2023-05-08 11:22:57,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:22:57,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +20: [2023-05-08 11:22:57,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +20: [2023-05-08 11:22:57,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:22:57,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:22:57,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +31: [2023-05-08 11:22:57,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +15: [2023-05-08 11:22:57,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +31: [2023-05-08 11:22:57,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:22:57,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:22:57,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:22:57,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:22:57,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:22:57,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:22:57,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +20: [2023-05-08 11:22:57,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +20: [2023-05-08 11:22:57,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +11: [2023-05-08 11:22:57,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +11: [2023-05-08 11:22:57,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:22:57,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 0: [2023-05-08 11:22:57,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +11: [2023-05-08 11:22:57,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +28: [2023-05-08 11:22:57,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +20: [2023-05-08 11:22:57,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:22:57,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +18: [2023-05-08 11:22:57,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:22:57,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:22:57,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:22:57,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:22:57,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +27: [2023-05-08 11:22:57,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +28: [2023-05-08 11:22:57,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:22:57,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:22:57,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:22:57,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +18: [2023-05-08 11:22:57,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:22:57,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:22:57,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +28: [2023-05-08 11:22:57,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:22:57,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +28: [2023-05-08 11:22:57,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +28: [2023-05-08 11:22:57,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +12: [2023-05-08 11:22:57,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:22:57,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:22:57,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:22:57,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +20: [2023-05-08 11:22:57,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:22:57,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +15: [2023-05-08 11:22:57,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +28: [2023-05-08 11:22:57,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:22:57,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +31: [2023-05-08 11:22:57,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:22:57,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +12: [2023-05-08 11:22:57,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +12: [2023-05-08 11:22:57,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +12: [2023-05-08 11:22:57,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +28: [2023-05-08 11:22:57,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +28: [2023-05-08 11:22:57,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:22:57,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:22:57,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +16: [2023-05-08 11:22:57,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +14: [2023-05-08 11:22:57,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 4: [2023-05-08 11:22:57,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +11: [2023-05-08 11:22:57,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:22:57,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +16: [2023-05-08 11:22:57,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +16: [2023-05-08 11:22:57,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +11: [2023-05-08 11:22:57,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:22:57,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 4: [2023-05-08 11:22:57,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 5: [2023-05-08 11:22:57,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:22:57,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:22:57,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +18: [2023-05-08 11:22:57,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:22:57,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:22:57,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:22:57,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +31: [2023-05-08 11:22:57,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:22:57,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +28: [2023-05-08 11:22:57,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +18: [2023-05-08 11:22:57,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 2: [2023-05-08 11:22:57,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 8: [2023-05-08 11:22:57,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +28: [2023-05-08 11:22:57,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:22:57,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +18: [2023-05-08 11:22:57,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +29: [2023-05-08 11:22:57,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +29: [2023-05-08 11:22:57,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:22:57,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:22:57,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +31: [2023-05-08 11:22:57,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:22:57,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +29: [2023-05-08 11:22:57,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 1: [2023-05-08 11:22:57,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:22:57,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:22:57,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:22:57,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +28: [2023-05-08 11:22:57,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:22:57,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +15: [2023-05-08 11:22:57,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +26: [2023-05-08 11:22:57,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:22:57,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:22:57,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +26: [2023-05-08 11:22:57,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:22:57,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:22:57,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:22:57,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +16: [2023-05-08 11:22:57,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:22:57,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:22:57,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:22:57,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:22:57,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:22:57,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:22:57,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:22:57,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:22:57,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +26: [2023-05-08 11:22:57,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +11: [2023-05-08 11:22:57,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:22:57,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:22:57,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +16: [2023-05-08 11:22:57,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:22:57,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +16: [2023-05-08 11:22:57,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:22:57,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +27: [2023-05-08 11:22:57,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:22:57,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:22:57,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:22:57,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +18: [2023-05-08 11:22:57,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:22:57,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +18: [2023-05-08 11:22:57,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 0: [2023-05-08 11:22:57,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:22:57,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +29: [2023-05-08 11:22:57,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:22:57,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:22:57,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:22:57,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 1: [2023-05-08 11:22:57,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +20: [2023-05-08 11:22:57,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +20: [2023-05-08 11:22:57,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:22:57,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:22:57,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:22:57,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +18: [2023-05-08 11:22:57,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +18: [2023-05-08 11:22:57,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +18: [2023-05-08 11:22:57,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 8: [2023-05-08 11:22:57,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:22:57,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:22:57,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +25: [2023-05-08 11:22:57,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +25: [2023-05-08 11:22:57,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +25: [2023-05-08 11:22:57,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +25: [2023-05-08 11:22:57,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +27: [2023-05-08 11:22:57,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:22:57,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +24: [2023-05-08 11:22:57,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +24: [2023-05-08 11:22:57,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 7: [2023-05-08 11:22:57,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +11: [2023-05-08 11:22:57,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +24: [2023-05-08 11:22:57,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +11: [2023-05-08 11:22:57,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +16: [2023-05-08 11:22:57,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +29: [2023-05-08 11:22:57,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:22:57,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +22: [2023-05-08 11:22:57,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 0: [2023-05-08 11:22:57,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 7: [2023-05-08 11:22:57,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +19: [2023-05-08 11:22:57,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +19: [2023-05-08 11:22:57,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +22: [2023-05-08 11:22:57,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +19: [2023-05-08 11:22:57,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +14: [2023-05-08 11:22:57,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +31: [2023-05-08 11:22:57,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +22: [2023-05-08 11:22:57,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +14: [2023-05-08 11:22:57,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +14: [2023-05-08 11:22:57,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +14: [2023-05-08 11:22:57,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +11: [2023-05-08 11:22:57,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:22:57,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:22:57,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +22: [2023-05-08 11:22:57,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +20: [2023-05-08 11:22:57,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +19: [2023-05-08 11:22:57,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +20: [2023-05-08 11:22:57,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +24: [2023-05-08 11:22:57,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 0: [2023-05-08 11:22:57,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 4: [2023-05-08 11:22:57,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +20: [2023-05-08 11:22:57,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:22:57,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:22:57,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +12: [2023-05-08 11:22:57,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +20: [2023-05-08 11:22:57,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +28: [2023-05-08 11:22:57,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:22:57,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 6: [2023-05-08 11:22:57,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +16: [2023-05-08 11:22:57,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 6: [2023-05-08 11:22:57,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +10: [2023-05-08 11:22:57,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +16: [2023-05-08 11:22:57,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +10: [2023-05-08 11:22:57,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +31: [2023-05-08 11:22:57,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +18: [2023-05-08 11:22:57,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:22:57,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +10: [2023-05-08 11:22:57,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +10: [2023-05-08 11:22:57,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +27: [2023-05-08 11:22:57,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +31: [2023-05-08 11:22:57,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 1: [2023-05-08 11:22:57,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:22:57,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:22:57,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +16: [2023-05-08 11:22:57,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 1: [2023-05-08 11:22:57,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 1: [2023-05-08 11:22:57,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +27: [2023-05-08 11:22:57,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +27: [2023-05-08 11:22:57,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 8: [2023-05-08 11:22:57,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 8: [2023-05-08 11:22:57,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +27: [2023-05-08 11:22:57,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 8: [2023-05-08 11:22:57,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:22:57,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 0: [2023-05-08 11:22:57,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +27: [2023-05-08 11:22:57,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +18: [2023-05-08 11:22:57,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:22:57,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +26: [2023-05-08 11:22:57,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +16: [2023-05-08 11:22:57,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:22:57,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:22:57,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:22:57,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +25: [2023-05-08 11:22:57,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +25: [2023-05-08 11:22:57,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +18: [2023-05-08 11:22:57,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +20: [2023-05-08 11:22:57,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:22:57,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 7: [2023-05-08 11:22:57,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:22:57,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +18: [2023-05-08 11:22:57,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +19: [2023-05-08 11:22:57,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:22:57,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:22:57,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:22:57,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:22:57,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:22:57,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:22:57,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +19: [2023-05-08 11:22:57,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:22:57,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:22:57,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:22:57,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:22:57,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:22:57,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 4: [2023-05-08 11:22:57,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +20: [2023-05-08 11:22:57,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +24: [2023-05-08 11:22:57,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +20: [2023-05-08 11:22:57,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +20: [2023-05-08 11:22:57,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +24: [2023-05-08 11:22:57,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +19: [2023-05-08 11:22:57,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:22:57,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:22:57,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:22:57,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:22:57,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +19: [2023-05-08 11:22:57,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:22:57,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:22:57,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +24: [2023-05-08 11:22:57,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:22:57,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:22:57,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:22:57,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +24: [2023-05-08 11:22:57,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +28: [2023-05-08 11:22:57,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:22:57,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:22:57,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +28: [2023-05-08 11:22:57,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:22:57,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:22:57,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:22:57,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +18: [2023-05-08 11:22:57,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:22:57,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:22:57,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:22:57,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +26: [2023-05-08 11:22:57,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +27: [2023-05-08 11:22:57,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:22:57,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +16: [2023-05-08 11:22:57,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:22:57,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +29: [2023-05-08 11:22:57,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:22:57,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:22:57,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +27: [2023-05-08 11:22:57,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:22:57,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 9: [2023-05-08 11:22:57,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +16: [2023-05-08 11:22:57,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 9: [2023-05-08 11:22:57,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 7: [2023-05-08 11:22:57,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:22:57,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +22: [2023-05-08 11:22:57,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 9: [2023-05-08 11:22:57,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 9: [2023-05-08 11:22:57,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:22:57,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:22:57,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +25: [2023-05-08 11:22:57,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +14: [2023-05-08 11:22:57,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +18: [2023-05-08 11:22:57,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +18: [2023-05-08 11:22:57,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +25: [2023-05-08 11:22:57,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +18: [2023-05-08 11:22:57,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 9: [2023-05-08 11:22:57,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +25: [2023-05-08 11:22:57,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +18: [2023-05-08 11:22:57,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +10: [2023-05-08 11:22:57,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +19: [2023-05-08 11:22:57,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 2: [2023-05-08 11:22:57,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +25: [2023-05-08 11:22:57,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:22:57,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +25: [2023-05-08 11:22:57,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +19: [2023-05-08 11:22:57,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:22:57,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +10: [2023-05-08 11:22:57,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +24: [2023-05-08 11:22:57,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:22:57,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +22: [2023-05-08 11:22:57,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:22:57,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +22: [2023-05-08 11:22:57,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:22:57,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +25: [2023-05-08 11:22:57,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +25: [2023-05-08 11:22:57,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +25: [2023-05-08 11:22:57,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 6: [2023-05-08 11:22:57,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:22:57,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +28: [2023-05-08 11:22:57,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +28: [2023-05-08 11:22:57,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 7: [2023-05-08 11:22:57,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 7: [2023-05-08 11:22:57,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +27: [2023-05-08 11:22:57,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +27: [2023-05-08 11:22:57,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +17: [2023-05-08 11:22:57,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +17: [2023-05-08 11:22:57,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +29: [2023-05-08 11:22:57,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +20: [2023-05-08 11:22:57,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:22:57,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +20: [2023-05-08 11:22:57,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:22:57,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:22:57,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +17: [2023-05-08 11:22:57,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +22: [2023-05-08 11:22:57,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:22:57,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +20: [2023-05-08 11:22:57,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:22:57,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +31: [2023-05-08 11:22:57,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +31: [2023-05-08 11:22:57,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +31: [2023-05-08 11:22:57,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +31: [2023-05-08 11:22:57,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +17: [2023-05-08 11:22:57,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +20: [2023-05-08 11:22:57,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +26: [2023-05-08 11:22:57,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:22:57,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:22:57,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +18: [2023-05-08 11:22:57,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +29: [2023-05-08 11:22:57,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 1: [2023-05-08 11:22:57,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +27: [2023-05-08 11:22:57,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +27: [2023-05-08 11:22:57,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +14: [2023-05-08 11:22:57,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +14: [2023-05-08 11:22:57,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +31: [2023-05-08 11:22:57,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +31: [2023-05-08 11:22:57,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +31: [2023-05-08 11:22:57,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +31: [2023-05-08 11:22:57,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +27: [2023-05-08 11:22:57,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +27: [2023-05-08 11:22:57,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 8: [2023-05-08 11:22:57,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 8: [2023-05-08 11:22:57,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 8: [2023-05-08 11:22:57,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +14: [2023-05-08 11:22:57,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +14: [2023-05-08 11:22:57,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +14: [2023-05-08 11:22:57,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +14: [2023-05-08 11:22:57,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +10: [2023-05-08 11:22:57,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +10: [2023-05-08 11:22:57,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +10: [2023-05-08 11:22:57,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +10: [2023-05-08 11:22:57,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +24: [2023-05-08 11:22:57,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:22:57,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:22:57,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +18: [2023-05-08 11:22:57,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +30: [2023-05-08 11:22:57,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 6: [2023-05-08 11:22:57,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +30: [2023-05-08 11:22:57,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +30: [2023-05-08 11:22:57,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 6: [2023-05-08 11:22:57,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +24: [2023-05-08 11:22:57,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 0: [2023-05-08 11:22:57,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 0: [2023-05-08 11:22:57,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 0: [2023-05-08 11:22:57,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 0: [2023-05-08 11:22:57,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +18: [2023-05-08 11:22:57,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +30: [2023-05-08 11:22:57,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +27: [2023-05-08 11:22:57,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +27: [2023-05-08 11:22:57,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +26: [2023-05-08 11:22:57,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:22:57,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:22:57,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:22:57,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +17: [2023-05-08 11:22:57,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +17: [2023-05-08 11:22:57,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:22:57,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:22:57,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... + 6: [2023-05-08 11:22:57,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +19: [2023-05-08 11:22:57,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:22:57,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:22:57,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +26: [2023-05-08 11:22:57,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +26: [2023-05-08 11:22:57,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +19: [2023-05-08 11:22:57,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +26: [2023-05-08 11:22:57,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +30: [2023-05-08 11:22:57,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:22:57,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +21: [2023-05-08 11:22:57,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +21: [2023-05-08 11:22:57,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +21: [2023-05-08 11:22:57,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +21: [2023-05-08 11:22:57,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt. +30: [2023-05-08 11:22:57,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 3: [2023-05-08 11:22:57,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 3: [2023-05-08 11:22:57,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 3: [2023-05-08 11:22:57,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +30: [2023-05-08 11:22:57,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:22:57,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:22:57,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +17: [2023-05-08 11:22:57,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +26: [2023-05-08 11:22:57,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:22:57,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +21: [2023-05-08 11:22:57,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:22:57,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:22:57,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +21: [2023-05-08 11:22:57,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:22:57,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +26: [2023-05-08 11:22:57,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:22:57,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +17: [2023-05-08 11:22:57,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +17: [2023-05-08 11:22:57,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +17: [2023-05-08 11:22:57,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +17: [2023-05-08 11:22:57,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +17: [2023-05-08 11:22:57,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +30: [2023-05-08 11:22:57,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:22:57,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 3: [2023-05-08 11:22:57,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +13: [2023-05-08 11:22:57,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:22:57,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:22:57,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +13: [2023-05-08 11:22:57,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:22:57,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +13: [2023-05-08 11:22:57,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +13: [2023-05-08 11:22:57,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:22:57,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +26: [2023-05-08 11:22:57,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:22:57,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:22:57,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:22:57,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:22:57,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:22:57,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:22:57,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:22:57,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:22:57,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +25: [2023-05-08 11:22:57,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +26: [2023-05-08 11:22:57,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:22:57,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:22:57,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +26: [2023-05-08 11:22:57,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +30: [2023-05-08 11:22:57,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:22:57,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +21: [2023-05-08 11:22:57,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:22:57,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +30: [2023-05-08 11:22:57,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +21: [2023-05-08 11:22:57,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +21: [2023-05-08 11:22:57,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +30: [2023-05-08 11:22:57,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:22:57,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:22:57,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +21: [2023-05-08 11:22:57,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:22:57,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +21: [2023-05-08 11:22:57,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:22:57,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +21: [2023-05-08 11:22:57,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +21: [2023-05-08 11:22:57,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +30: [2023-05-08 11:22:57,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 3: [2023-05-08 11:22:57,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +30: [2023-05-08 11:22:57,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +21: [2023-05-08 11:22:57,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +21: [2023-05-08 11:22:57,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +21: [2023-05-08 11:22:57,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt. +21: [2023-05-08 11:22:57,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt... +13: [2023-05-08 11:22:57,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:22:57,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:22:57,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:22:57,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:22:57,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +13: [2023-05-08 11:22:57,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:22:57,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:22:57,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:22:57,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:22:57,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:22:57,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:22:57,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:22:57,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:22:57,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:22:57,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:22:57,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:22:57,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:22:57,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:22:57,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:22:57,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +25: [2023-05-08 11:22:57,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:22:57,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:22:57,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:22:57,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:22:57,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:22:57,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:22:57,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:22:57,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:22:57,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:22:57,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:22:57,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:22:57,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:22:57,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:22:57,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +30: [2023-05-08 11:22:57,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:22:57,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +30: [2023-05-08 11:22:57,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:22:57,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:22:57,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:22:57,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +30: [2023-05-08 11:22:57,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +30: [2023-05-08 11:22:57,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:22:57,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:22:57,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +24: [2023-05-08 11:22:57,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +24: [2023-05-08 11:22:57,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:22:57,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:22:57,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:22:57,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:22:57,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:22:57,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +24: [2023-05-08 11:22:57,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:22:57,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:22:57,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:22:57,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:22:57,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:22:57,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:22:57,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:22:57,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:22:57,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:22:57,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:22:57,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:22:57,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:22:57,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:22:57,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:22:57,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:22:57,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:22:57,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:22:57,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:22:57,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:22:57,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +15: [2023-05-08 11:22:57,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +15: [2023-05-08 11:22:57,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +15: [2023-05-08 11:22:57,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:22:57,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +17: [2023-05-08 11:22:57,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +17: [2023-05-08 11:22:57,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +15: [2023-05-08 11:22:57,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +17: [2023-05-08 11:22:57,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +17: [2023-05-08 11:22:57,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:22:57,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:22:57,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:22:57,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +17: [2023-05-08 11:22:57,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:22:57,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +17: [2023-05-08 11:22:57,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +17: [2023-05-08 11:22:57,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +17: [2023-05-08 11:22:57,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:22:57,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:22:57,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +24: [2023-05-08 11:22:57,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:22:57,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +24: [2023-05-08 11:22:57,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:22:57,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:22:57,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:22:57,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:22:57,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:22:57,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +11: [2023-05-08 11:22:57,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:22:57,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:22:57,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:22:57,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +23: [2023-05-08 11:22:57,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +23: [2023-05-08 11:22:57,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +23: [2023-05-08 11:22:57,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:22:57,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:22:57,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:22:57,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +23: [2023-05-08 11:22:57,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:22:57,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:22:57,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:22:57,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:22:57,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +24: [2023-05-08 11:22:57,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +24: [2023-05-08 11:22:57,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +24: [2023-05-08 11:22:57,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:22:57,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:22:57,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +24: [2023-05-08 11:22:57,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:22:57,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:22:57,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:22:57,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:22:57,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:22:57,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +19: [2023-05-08 11:22:57,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:22:57,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:22:57,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:22:57,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +17: [2023-05-08 11:22:57,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:22:57,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +11: [2023-05-08 11:22:57,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +11: [2023-05-08 11:22:57,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:22:57,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:22:57,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +23: [2023-05-08 11:22:57,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +23: [2023-05-08 11:22:57,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +23: [2023-05-08 11:22:57,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:22:57,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:22:57,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +15: [2023-05-08 11:22:57,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +15: [2023-05-08 11:22:57,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:22:57,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:22:57,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:22:57,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:22:57,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +17: [2023-05-08 11:22:57,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +19: [2023-05-08 11:22:57,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:22:57,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +15: [2023-05-08 11:22:57,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:22:57,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:22:57,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:22:57,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:22:57,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:22:57,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:22:57,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +13: [2023-05-08 11:22:57,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +13: [2023-05-08 11:22:57,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +13: [2023-05-08 11:22:57,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +23: [2023-05-08 11:22:57,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +28: [2023-05-08 11:22:57,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +23: [2023-05-08 11:22:57,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +23: [2023-05-08 11:22:57,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +15: [2023-05-08 11:22:57,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +15: [2023-05-08 11:22:57,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +15: [2023-05-08 11:22:57,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +23: [2023-05-08 11:22:57,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +28: [2023-05-08 11:22:57,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +28: [2023-05-08 11:22:57,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 5: [2023-05-08 11:22:57,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 5: [2023-05-08 11:22:57,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +15: [2023-05-08 11:22:57,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 5: [2023-05-08 11:22:57,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +28: [2023-05-08 11:22:57,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 5: [2023-05-08 11:22:57,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 2: [2023-05-08 11:22:57,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +13: [2023-05-08 11:22:57,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +13: [2023-05-08 11:22:57,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +13: [2023-05-08 11:22:57,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +13: [2023-05-08 11:22:57,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:22:57,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +15: [2023-05-08 11:22:57,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +15: [2023-05-08 11:22:57,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +15: [2023-05-08 11:22:57,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:22:57,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:22:57,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:22:57,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:22:57,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:22:57,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:22:57,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:22:57,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +13: [2023-05-08 11:22:57,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +13: [2023-05-08 11:22:57,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 8: [2023-05-08 11:22:57,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:22:57,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +13: [2023-05-08 11:22:57,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +28: [2023-05-08 11:22:57,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +15: [2023-05-08 11:22:57,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +15: [2023-05-08 11:22:57,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 2: [2023-05-08 11:22:57,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +10: [2023-05-08 11:22:57,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +10: [2023-05-08 11:22:57,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:22:57,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +15: [2023-05-08 11:22:57,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +15: [2023-05-08 11:22:57,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:22:57,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:22:57,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +15: [2023-05-08 11:22:57,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 5: [2023-05-08 11:22:57,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 5: [2023-05-08 11:22:57,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 5: [2023-05-08 11:22:57,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +10: [2023-05-08 11:22:57,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +10: [2023-05-08 11:22:57,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:22:57,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:22:57,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:22:57,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:22:57,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +23: [2023-05-08 11:22:57,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:22:57,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 8: [2023-05-08 11:22:57,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +23: [2023-05-08 11:22:57,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:22:57,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 9: [2023-05-08 11:22:57,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:22:57,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +16: [2023-05-08 11:22:57,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +16: [2023-05-08 11:22:57,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 8: [2023-05-08 11:22:57,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +16: [2023-05-08 11:22:57,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 2: [2023-05-08 11:22:57,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:22:57,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +13: [2023-05-08 11:22:57,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:22:57,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:22:57,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +15: [2023-05-08 11:22:57,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +13: [2023-05-08 11:22:57,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +13: [2023-05-08 11:22:57,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +28: [2023-05-08 11:22:57,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +15: [2023-05-08 11:22:57,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 1: [2023-05-08 11:22:57,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 1: [2023-05-08 11:22:57,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 2: [2023-05-08 11:22:57,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 1: [2023-05-08 11:22:57,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 5: [2023-05-08 11:22:57,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +28: [2023-05-08 11:22:57,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +28: [2023-05-08 11:22:57,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +16: [2023-05-08 11:22:57,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +16: [2023-05-08 11:22:57,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:22:57,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:22:57,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:22:57,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 7: [2023-05-08 11:22:57,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 7: [2023-05-08 11:22:57,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +10: [2023-05-08 11:22:57,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +10: [2023-05-08 11:22:57,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +10: [2023-05-08 11:22:57,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +10: [2023-05-08 11:22:57,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +10: [2023-05-08 11:22:57,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +10: [2023-05-08 11:22:57,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +10: [2023-05-08 11:22:57,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +28: [2023-05-08 11:22:57,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:22:57,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:22:57,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +11: [2023-05-08 11:22:57,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +11: [2023-05-08 11:22:57,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +11: [2023-05-08 11:22:57,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 1: [2023-05-08 11:22:57,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:22:57,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +23: [2023-05-08 11:22:57,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +20: [2023-05-08 11:22:57,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +20: [2023-05-08 11:22:57,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +20: [2023-05-08 11:22:57,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +20: [2023-05-08 11:22:57,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 7: [2023-05-08 11:22:57,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:22:57,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +10: [2023-05-08 11:22:57,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +10: [2023-05-08 11:22:57,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:22:57,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +11: [2023-05-08 11:22:57,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +10: [2023-05-08 11:22:57,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +10: [2023-05-08 11:22:57,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +10: [2023-05-08 11:22:57,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +12: [2023-05-08 11:22:57,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +12: [2023-05-08 11:22:57,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +12: [2023-05-08 11:22:57,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 4: [2023-05-08 11:22:57,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +12: [2023-05-08 11:22:57,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 4: [2023-05-08 11:22:57,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:22:57,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +10: [2023-05-08 11:22:57,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +11: [2023-05-08 11:22:57,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +11: [2023-05-08 11:22:57,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +11: [2023-05-08 11:22:57,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +20: [2023-05-08 11:22:57,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:22:57,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:22:57,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +20: [2023-05-08 11:22:57,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:22:57,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +17: [2023-05-08 11:22:57,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +17: [2023-05-08 11:22:57,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 4: [2023-05-08 11:22:57,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 4: [2023-05-08 11:22:57,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 4: [2023-05-08 11:22:57,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:22:57,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:22:57,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +29: [2023-05-08 11:22:57,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +29: [2023-05-08 11:22:57,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +29: [2023-05-08 11:22:57,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +19: [2023-05-08 11:22:57,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +19: [2023-05-08 11:22:57,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:22:57,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:22:57,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 1: [2023-05-08 11:22:57,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 1: [2023-05-08 11:22:57,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +12: [2023-05-08 11:22:57,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:22:57,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:22:57,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:22:57,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 7: [2023-05-08 11:22:57,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +14: [2023-05-08 11:22:57,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +14: [2023-05-08 11:22:57,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:22:57,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +14: [2023-05-08 11:22:57,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +14: [2023-05-08 11:22:57,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:22:57,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +17: [2023-05-08 11:22:57,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 7: [2023-05-08 11:22:57,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +17: [2023-05-08 11:22:57,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:22:57,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:22:57,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +14: [2023-05-08 11:22:57,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:22:57,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:22:57,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:22:57,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:22:57,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +17: [2023-05-08 11:22:57,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:22:57,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:22:57,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 6: [2023-05-08 11:22:57,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +11: [2023-05-08 11:22:57,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +19: [2023-05-08 11:22:57,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 6: [2023-05-08 11:22:57,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +29: [2023-05-08 11:22:57,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:22:57,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:22:57,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:22:57,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:22:57,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 6: [2023-05-08 11:22:57,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +20: [2023-05-08 11:22:57,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +20: [2023-05-08 11:22:57,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 4: [2023-05-08 11:22:57,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +31: [2023-05-08 11:22:57,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +31: [2023-05-08 11:22:57,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +31: [2023-05-08 11:22:57,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +31: [2023-05-08 11:22:57,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 1: [2023-05-08 11:22:57,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:22:57,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:22:57,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:22:57,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 1: [2023-05-08 11:22:57,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 7: [2023-05-08 11:22:57,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +28: [2023-05-08 11:22:57,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:22:57,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:22:57,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +28: [2023-05-08 11:22:57,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:22:57,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +28: [2023-05-08 11:22:57,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:22:57,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +18: [2023-05-08 11:22:57,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +28: [2023-05-08 11:22:57,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +18: [2023-05-08 11:22:57,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:22:57,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:22:57,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:22:57,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +12: [2023-05-08 11:22:57,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:22:57,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +25: [2023-05-08 11:22:57,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +25: [2023-05-08 11:22:57,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +25: [2023-05-08 11:22:57,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 7: [2023-05-08 11:22:57,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +12: [2023-05-08 11:22:57,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +12: [2023-05-08 11:22:57,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:22:57,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +12: [2023-05-08 11:22:57,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:22:57,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:22:57,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +12: [2023-05-08 11:22:57,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +18: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +12: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +18: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +31: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +31: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:22:57,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +12: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 0: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 0: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +26: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +26: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +16: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 0: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +26: [2023-05-08 11:22:57,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +31: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +26: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +31: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +17: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +29: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +29: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +29: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +29: [2023-05-08 11:22:57,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +29: [2023-05-08 11:22:57,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:22:57,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +29: [2023-05-08 11:22:57,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:22:57,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:22:57,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:22:57,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:22:57,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:22:57,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +16: [2023-05-08 11:22:57,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:22:57,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +17: [2023-05-08 11:22:57,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 6: [2023-05-08 11:22:57,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +31: [2023-05-08 11:22:57,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +11: [2023-05-08 11:22:57,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +14: [2023-05-08 11:22:57,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +27: [2023-05-08 11:22:57,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:22:57,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:22:57,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +31: [2023-05-08 11:22:57,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:22:57,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:22:57,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:22:57,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +31: [2023-05-08 11:22:57,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:22:57,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +27: [2023-05-08 11:22:57,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +27: [2023-05-08 11:22:57,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +29: [2023-05-08 11:22:57,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:22:57,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +31: [2023-05-08 11:22:57,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:22:57,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +31: [2023-05-08 11:22:57,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +27: [2023-05-08 11:22:57,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +10: [2023-05-08 11:22:57,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:22:57,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +27: [2023-05-08 11:22:57,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +17: [2023-05-08 11:22:57,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +17: [2023-05-08 11:22:57,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +20: [2023-05-08 11:22:57,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:22:57,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +31: [2023-05-08 11:22:57,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +11: [2023-05-08 11:22:57,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:22:57,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +10: [2023-05-08 11:22:57,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +31: [2023-05-08 11:22:57,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +31: [2023-05-08 11:22:57,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:22:57,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:22:57,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +25: [2023-05-08 11:22:57,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:22:57,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +25: [2023-05-08 11:22:57,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:22:57,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +14: [2023-05-08 11:22:57,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +29: [2023-05-08 11:22:57,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +29: [2023-05-08 11:22:57,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:22:57,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:22:57,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +25: [2023-05-08 11:22:57,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:22:57,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +24: [2023-05-08 11:22:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +24: [2023-05-08 11:22:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +14: [2023-05-08 11:22:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +29: [2023-05-08 11:22:57,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +27: [2023-05-08 11:22:57,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:22:57,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +10: [2023-05-08 11:22:57,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +14: [2023-05-08 11:22:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 6: [2023-05-08 11:22:57,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:22:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:22:57,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +14: [2023-05-08 11:22:57,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +18: [2023-05-08 11:22:57,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +14: [2023-05-08 11:22:57,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:22:57,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:22:57,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +25: [2023-05-08 11:22:57,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +17: [2023-05-08 11:22:57,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +17: [2023-05-08 11:22:57,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +18: [2023-05-08 11:22:57,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +12: [2023-05-08 11:22:57,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +12: [2023-05-08 11:22:57,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:22:57,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:22:57,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:22:57,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:22:57,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:22:57,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +14: [2023-05-08 11:22:57,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:22:57,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:22:57,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 8: [2023-05-08 11:22:57,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 5: [2023-05-08 11:22:57,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:22:57,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 8: [2023-05-08 11:22:57,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 6: [2023-05-08 11:22:57,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:22:57,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 6: [2023-05-08 11:22:57,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 0: [2023-05-08 11:22:57,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:22:57,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:22:57,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:22:57,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +12: [2023-05-08 11:22:57,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:22:57,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +14: [2023-05-08 11:22:57,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:22:57,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +18: [2023-05-08 11:22:57,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +29: [2023-05-08 11:22:57,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +29: [2023-05-08 11:22:57,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:22:57,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:22:57,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +29: [2023-05-08 11:22:57,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +20: [2023-05-08 11:22:57,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 0: [2023-05-08 11:22:57,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 0: [2023-05-08 11:22:57,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:22:57,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:22:57,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +20: [2023-05-08 11:22:57,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +24: [2023-05-08 11:22:57,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:22:57,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:22:57,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:22:57,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +14: [2023-05-08 11:22:57,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:22:57,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +12: [2023-05-08 11:22:57,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:22:57,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +24: [2023-05-08 11:22:57,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +14: [2023-05-08 11:22:57,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +27: [2023-05-08 11:22:57,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +12: [2023-05-08 11:22:57,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +28: [2023-05-08 11:22:57,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:22:57,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:22:57,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +18: [2023-05-08 11:22:57,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:22:57,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:22:57,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:22:57,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +28: [2023-05-08 11:22:57,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +28: [2023-05-08 11:22:57,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:22:57,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:22:57,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +27: [2023-05-08 11:22:57,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:22:57,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:22:57,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:22:57,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +26: [2023-05-08 11:22:57,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:22:57,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:22:57,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +14: [2023-05-08 11:22:57,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:22:57,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +12: [2023-05-08 11:22:57,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +29: [2023-05-08 11:22:57,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +29: [2023-05-08 11:22:57,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +29: [2023-05-08 11:22:57,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +18: [2023-05-08 11:22:57,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +18: [2023-05-08 11:22:57,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +18: [2023-05-08 11:22:57,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +16: [2023-05-08 11:22:57,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:22:57,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:22:57,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:22:57,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:22:57,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 4: [2023-05-08 11:22:57,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:22:57,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +29: [2023-05-08 11:22:57,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:22:57,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:22:57,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +29: [2023-05-08 11:22:57,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 4: [2023-05-08 11:22:57,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 5: [2023-05-08 11:22:57,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:22:57,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:22:57,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +26: [2023-05-08 11:22:57,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +20: [2023-05-08 11:22:57,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:22:57,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +31: [2023-05-08 11:22:57,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 2: [2023-05-08 11:22:57,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +31: [2023-05-08 11:22:57,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:22:57,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +28: [2023-05-08 11:22:57,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:22:57,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +16: [2023-05-08 11:22:57,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +26: [2023-05-08 11:22:57,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +16: [2023-05-08 11:22:57,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +31: [2023-05-08 11:22:57,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 1: [2023-05-08 11:22:57,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:22:57,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:22:57,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +26: [2023-05-08 11:22:57,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:22:57,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +26: [2023-05-08 11:22:57,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +24: [2023-05-08 11:22:57,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +24: [2023-05-08 11:22:57,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +18: [2023-05-08 11:22:57,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:22:57,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +24: [2023-05-08 11:22:57,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 3: [2023-05-08 11:22:57,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +31: [2023-05-08 11:22:57,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:22:57,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +30: [2023-05-08 11:22:57,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 3: [2023-05-08 11:22:57,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 3: [2023-05-08 11:22:57,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 2: [2023-05-08 11:22:57,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 3: [2023-05-08 11:22:57,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 1: [2023-05-08 11:22:57,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:22:57,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +27: [2023-05-08 11:22:57,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:22:57,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:22:57,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +30: [2023-05-08 11:22:57,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +14: [2023-05-08 11:22:57,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +30: [2023-05-08 11:22:57,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +29: [2023-05-08 11:22:57,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +26: [2023-05-08 11:22:57,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +27: [2023-05-08 11:22:57,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +18: [2023-05-08 11:22:57,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:22:57,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +27: [2023-05-08 11:22:57,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:22:57,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:22:57,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:22:57,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:22:57,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +31: [2023-05-08 11:22:57,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 6: [2023-05-08 11:22:57,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +27: [2023-05-08 11:22:57,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:22:57,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +16: [2023-05-08 11:22:57,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:22:57,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +29: [2023-05-08 11:22:57,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:22:57,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:22:57,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +31: [2023-05-08 11:22:57,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:22:57,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +27: [2023-05-08 11:22:57,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +30: [2023-05-08 11:22:57,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +27: [2023-05-08 11:22:57,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:22:57,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +14: [2023-05-08 11:22:57,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:22:57,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +22: [2023-05-08 11:22:57,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +24: [2023-05-08 11:22:57,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 2: [2023-05-08 11:22:57,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:22:57,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +31: [2023-05-08 11:22:57,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:22:57,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +30: [2023-05-08 11:22:57,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +27: [2023-05-08 11:22:57,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 0: [2023-05-08 11:22:57,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +27: [2023-05-08 11:22:57,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +21: [2023-05-08 11:22:57,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +21: [2023-05-08 11:22:57,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +21: [2023-05-08 11:22:57,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. +21: [2023-05-08 11:22:57,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +31: [2023-05-08 11:22:57,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 2: [2023-05-08 11:22:57,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +29: [2023-05-08 11:22:57,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:22:57,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:22:57,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:22:57,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +14: [2023-05-08 11:22:57,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:22:57,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:22:57,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +27: [2023-05-08 11:22:57,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 0: [2023-05-08 11:22:57,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +14: [2023-05-08 11:22:57,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +14: [2023-05-08 11:22:57,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +18: [2023-05-08 11:22:57,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:22:57,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +22: [2023-05-08 11:22:57,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:22:57,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:22:57,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:22:57,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:22:57,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 1: [2023-05-08 11:22:57,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:22:57,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:22:57,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +22: [2023-05-08 11:22:57,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:22:57,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +27: [2023-05-08 11:22:57,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +30: [2023-05-08 11:22:57,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +21: [2023-05-08 11:22:57,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +21: [2023-05-08 11:22:57,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:22:57,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:22:57,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:22:57,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 2: [2023-05-08 11:22:57,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +18: [2023-05-08 11:22:57,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:22:57,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 2: [2023-05-08 11:22:57,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 2: [2023-05-08 11:22:57,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +30: [2023-05-08 11:22:57,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:22:57,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:22:57,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +30: [2023-05-08 11:22:57,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +22: [2023-05-08 11:22:57,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +22: [2023-05-08 11:22:57,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 3: [2023-05-08 11:22:57,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 0: [2023-05-08 11:22:57,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 0: [2023-05-08 11:22:57,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +21: [2023-05-08 11:22:57,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:22:57,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +21: [2023-05-08 11:22:57,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:22:57,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... + 3: [2023-05-08 11:22:57,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 3: [2023-05-08 11:22:57,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +22: [2023-05-08 11:22:57,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +13: [2023-05-08 11:22:57,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +13: [2023-05-08 11:22:57,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +13: [2023-05-08 11:22:57,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +13: [2023-05-08 11:22:57,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +22: [2023-05-08 11:22:57,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +22: [2023-05-08 11:22:57,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:22:57,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:22:57,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +21: [2023-05-08 11:22:57,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:22:57,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +21: [2023-05-08 11:22:57,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:22:57,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +21: [2023-05-08 11:22:57,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt... +13: [2023-05-08 11:22:57,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +13: [2023-05-08 11:22:57,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:22:57,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +15: [2023-05-08 11:22:57,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +15: [2023-05-08 11:22:57,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +15: [2023-05-08 11:22:57,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +13: [2023-05-08 11:22:57,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +13: [2023-05-08 11:22:57,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:22:57,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:22:57,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:22:57,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:22:57,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +13: [2023-05-08 11:22:57,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:22:57,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:22:57,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:22:57,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:22:57,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:22:57,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:22:57,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:22:57,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:22:57,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +13: [2023-05-08 11:22:57,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:22:57,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:22:57,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +13: [2023-05-08 11:22:57,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:22:57,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +13: [2023-05-08 11:22:57,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:22:57,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +28: [2023-05-08 11:22:57,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +28: [2023-05-08 11:22:57,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +28: [2023-05-08 11:22:57,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +13: [2023-05-08 11:22:57,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +15: [2023-05-08 11:22:57,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +15: [2023-05-08 11:22:57,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:22:57,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +26: [2023-05-08 11:22:57,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +26: [2023-05-08 11:22:57,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +26: [2023-05-08 11:22:57,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +11: [2023-05-08 11:22:57,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +11: [2023-05-08 11:22:57,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +11: [2023-05-08 11:22:57,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:22:57,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +11: [2023-05-08 11:22:57,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +11: [2023-05-08 11:22:57,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:22:57,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +11: [2023-05-08 11:22:57,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +11: [2023-05-08 11:22:57,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:22:57,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +11: [2023-05-08 11:22:57,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:22:57,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:22:57,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:22:57,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +15: [2023-05-08 11:22:57,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:22:57,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +28: [2023-05-08 11:22:57,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:22:57,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +25: [2023-05-08 11:22:57,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:22:57,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +26: [2023-05-08 11:22:57,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +26: [2023-05-08 11:22:57,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +28: [2023-05-08 11:22:57,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +26: [2023-05-08 11:22:57,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +26: [2023-05-08 11:22:57,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:22:57,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +11: [2023-05-08 11:22:57,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +11: [2023-05-08 11:22:57,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +11: [2023-05-08 11:22:57,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:22:57,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +25: [2023-05-08 11:22:57,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 1: [2023-05-08 11:22:57,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +26: [2023-05-08 11:22:57,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:22:57,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +11: [2023-05-08 11:22:57,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +11: [2023-05-08 11:22:57,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +26: [2023-05-08 11:22:57,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 1: [2023-05-08 11:22:57,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 1: [2023-05-08 11:22:57,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +11: [2023-05-08 11:22:57,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +11: [2023-05-08 11:22:57,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +26: [2023-05-08 11:22:57,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:22:57,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:22:57,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +26: [2023-05-08 11:22:57,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +26: [2023-05-08 11:22:57,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:22:57,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:22:57,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:22:57,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:22:57,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +28: [2023-05-08 11:22:57,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:22:57,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:22:57,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +23: [2023-05-08 11:22:57,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +23: [2023-05-08 11:22:57,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +23: [2023-05-08 11:22:57,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +25: [2023-05-08 11:22:57,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:22:57,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +11: [2023-05-08 11:22:57,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:22:57,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:22:57,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:22:57,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:22:57,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:22:57,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:22:57,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:22:57,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:22:57,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:22:57,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +11: [2023-05-08 11:22:57,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:22:57,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:22:57,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +11: [2023-05-08 11:22:57,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 1: [2023-05-08 11:22:57,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 1: [2023-05-08 11:22:57,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 1: [2023-05-08 11:22:57,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +23: [2023-05-08 11:22:57,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +23: [2023-05-08 11:22:57,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 1: [2023-05-08 11:22:57,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:22:57,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:22:57,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 1: [2023-05-08 11:22:57,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +23: [2023-05-08 11:22:57,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +23: [2023-05-08 11:22:57,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +11: [2023-05-08 11:22:57,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +11: [2023-05-08 11:22:57,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:22:57,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 5: [2023-05-08 11:22:57,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 5: [2023-05-08 11:22:57,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 5: [2023-05-08 11:22:57,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 5: [2023-05-08 11:22:57,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +26: [2023-05-08 11:22:57,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +22: [2023-05-08 11:22:57,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:22:57,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:22:57,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +23: [2023-05-08 11:22:57,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +10: [2023-05-08 11:22:57,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +22: [2023-05-08 11:22:57,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:22:57,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +28: [2023-05-08 11:22:57,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +28: [2023-05-08 11:22:57,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +28: [2023-05-08 11:22:57,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +10: [2023-05-08 11:22:57,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +22: [2023-05-08 11:22:57,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:22:57,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:22:57,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 7: [2023-05-08 11:22:57,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 7: [2023-05-08 11:22:57,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +19: [2023-05-08 11:22:57,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +19: [2023-05-08 11:22:57,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +19: [2023-05-08 11:22:57,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +19: [2023-05-08 11:22:57,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +19: [2023-05-08 11:22:57,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:22:57,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:22:57,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:22:57,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:22:57,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:22:57,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:22:57,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:22:57,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:22:57,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:22:57,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:22:57,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:22:57,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:22:57,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:22:57,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:22:57,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:22:57,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:22:57,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:22:57,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:22:57,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +13: [2023-05-08 11:22:57,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:22:57,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +19: [2023-05-08 11:22:57,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +19: [2023-05-08 11:22:57,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 5: [2023-05-08 11:22:57,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:22:57,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 5: [2023-05-08 11:22:57,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:22:57,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 5: [2023-05-08 11:22:57,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:22:57,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:22:57,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 6: [2023-05-08 11:22:57,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +10: [2023-05-08 11:22:57,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:22:57,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +20: [2023-05-08 11:22:57,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 7: [2023-05-08 11:22:57,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 6: [2023-05-08 11:22:57,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +22: [2023-05-08 11:22:57,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:22:57,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +21: [2023-05-08 11:22:57,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:22:57,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 7: [2023-05-08 11:22:57,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:22:57,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +13: [2023-05-08 11:22:57,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:22:57,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 4: [2023-05-08 11:22:57,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 4: [2023-05-08 11:22:57,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +24: [2023-05-08 11:22:57,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +24: [2023-05-08 11:22:57,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:22:57,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +24: [2023-05-08 11:22:57,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:22:57,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +24: [2023-05-08 11:22:57,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:22:57,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:22:57,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:22:57,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:22:57,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +10: [2023-05-08 11:22:57,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +17: [2023-05-08 11:22:57,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +17: [2023-05-08 11:22:57,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +17: [2023-05-08 11:22:57,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:22:57,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:22:57,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:22:57,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:22:57,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:22:57,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:22:57,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:22:57,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +17: [2023-05-08 11:22:57,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +17: [2023-05-08 11:22:57,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:22:57,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:22:57,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 5: [2023-05-08 11:22:57,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +22: [2023-05-08 11:22:57,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:22:57,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:22:57,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:22:57,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:22:57,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +19: [2023-05-08 11:22:57,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:22:57,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +19: [2023-05-08 11:22:57,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:22:57,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +21: [2023-05-08 11:22:57,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:22:57,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:22:57,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:22:57,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 4: [2023-05-08 11:22:57,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +24: [2023-05-08 11:22:57,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +24: [2023-05-08 11:22:57,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +10: [2023-05-08 11:22:57,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:22:57,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:22:57,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +21: [2023-05-08 11:22:57,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:22:57,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:22:57,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +10: [2023-05-08 11:22:57,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 4: [2023-05-08 11:22:57,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +21: [2023-05-08 11:22:57,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 7: [2023-05-08 11:22:57,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 5: [2023-05-08 11:22:57,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +24: [2023-05-08 11:22:57,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +19: [2023-05-08 11:22:57,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +19: [2023-05-08 11:22:57,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:22:57,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +17: [2023-05-08 11:22:57,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +19: [2023-05-08 11:22:57,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:22:57,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:22:57,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +19: [2023-05-08 11:22:57,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +10: [2023-05-08 11:22:57,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +10: [2023-05-08 11:22:57,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +10: [2023-05-08 11:22:57,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +10: [2023-05-08 11:22:57,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 2: [2023-05-08 11:22:57,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +24: [2023-05-08 11:22:57,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:22:57,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +17: [2023-05-08 11:22:57,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:22:57,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:22:57,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +24: [2023-05-08 11:22:57,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +20: [2023-05-08 11:22:57,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 6: [2023-05-08 11:22:57,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:22:57,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:22:57,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +20: [2023-05-08 11:22:57,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:22:57,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +24: [2023-05-08 11:22:57,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:22:57,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +10: [2023-05-08 11:22:57,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 7: [2023-05-08 11:22:57,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 7: [2023-05-08 11:22:57,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +24: [2023-05-08 11:22:57,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +24: [2023-05-08 11:22:57,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:22:57,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:22:57,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +20: [2023-05-08 11:22:57,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +17: [2023-05-08 11:22:57,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +20: [2023-05-08 11:22:57,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +17: [2023-05-08 11:22:57,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:22:57,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:22:57,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:22:57,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:22:57,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:22:57,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:22:57,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:22:57,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:22:57,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +24: [2023-05-08 11:22:57,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:22:57,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:22:57,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 4: [2023-05-08 11:22:57,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +17: [2023-05-08 11:22:57,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:22:57,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:22:57,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 6: [2023-05-08 11:22:57,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:22:57,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:22:57,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 2: [2023-05-08 11:22:57,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:22:57,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 6: [2023-05-08 11:22:57,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +24: [2023-05-08 11:22:57,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:22:57,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 2: [2023-05-08 11:22:57,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:22:57,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:22:57,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:22:57,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 6: [2023-05-08 11:22:57,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:22:57,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:22:57,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 3: [2023-05-08 11:22:57,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:22:57,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:22:57,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:22:57,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:22:57,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:22:57,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +12: [2023-05-08 11:22:57,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +12: [2023-05-08 11:22:57,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +12: [2023-05-08 11:22:57,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +31: [2023-05-08 11:22:57,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +31: [2023-05-08 11:22:57,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 2: [2023-05-08 11:22:57,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:22:57,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +31: [2023-05-08 11:22:57,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 3: [2023-05-08 11:22:57,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 3: [2023-05-08 11:22:57,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 0: [2023-05-08 11:22:57,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 0: [2023-05-08 11:22:57,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 3: [2023-05-08 11:22:57,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:22:57,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +22: [2023-05-08 11:22:57,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +18: [2023-05-08 11:22:57,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +18: [2023-05-08 11:22:57,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +18: [2023-05-08 11:22:57,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +22: [2023-05-08 11:22:57,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +23: [2023-05-08 11:22:57,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:22:57,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:22:57,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +12: [2023-05-08 11:22:57,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +31: [2023-05-08 11:22:57,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +21: [2023-05-08 11:22:57,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +21: [2023-05-08 11:22:57,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +21: [2023-05-08 11:22:57,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +18: [2023-05-08 11:22:57,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +21: [2023-05-08 11:22:57,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +31: [2023-05-08 11:22:57,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:22:57,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +31: [2023-05-08 11:22:57,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:22:57,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:22:57,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +25: [2023-05-08 11:22:57,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +31: [2023-05-08 11:22:57,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:22:57,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:22:57,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 0: [2023-05-08 11:22:57,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:22:57,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:22:57,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:22:57,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +21: [2023-05-08 11:22:57,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:22:57,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:22:57,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +21: [2023-05-08 11:22:57,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:22:57,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:22:57,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:22:57,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:22:57,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:22:57,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:22:57,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:22:57,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +14: [2023-05-08 11:22:57,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +14: [2023-05-08 11:22:57,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +14: [2023-05-08 11:22:57,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:22:57,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:22:57,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:22:57,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +29: [2023-05-08 11:22:57,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:22:57,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +29: [2023-05-08 11:22:57,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:22:57,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:22:57,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +29: [2023-05-08 11:22:57,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +12: [2023-05-08 11:22:57,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:22:57,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:22:57,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +12: [2023-05-08 11:22:57,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +14: [2023-05-08 11:22:57,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:22:57,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:22:57,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +12: [2023-05-08 11:22:57,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +12: [2023-05-08 11:22:57,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +29: [2023-05-08 11:22:57,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:22:57,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:22:57,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +27: [2023-05-08 11:22:57,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 8: [2023-05-08 11:22:57,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 8: [2023-05-08 11:22:57,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +27: [2023-05-08 11:22:57,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 8: [2023-05-08 11:22:57,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +31: [2023-05-08 11:22:57,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +31: [2023-05-08 11:22:57,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +31: [2023-05-08 11:22:57,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +29: [2023-05-08 11:22:57,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +27: [2023-05-08 11:22:57,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +29: [2023-05-08 11:22:57,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +27: [2023-05-08 11:22:57,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +29: [2023-05-08 11:22:57,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +29: [2023-05-08 11:22:57,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +29: [2023-05-08 11:22:57,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +14: [2023-05-08 11:22:57,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +19: [2023-05-08 11:22:57,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +19: [2023-05-08 11:22:57,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +19: [2023-05-08 11:22:57,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +14: [2023-05-08 11:22:57,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +12: [2023-05-08 11:22:57,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +14: [2023-05-08 11:22:57,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. + 3: [2023-05-08 11:22:57,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 3: [2023-05-08 11:22:57,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +14: [2023-05-08 11:22:57,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt. +18: [2023-05-08 11:22:57,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:22:57,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +18: [2023-05-08 11:22:57,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 3: [2023-05-08 11:22:57,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 9: [2023-05-08 11:22:57,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 0: [2023-05-08 11:22:57,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +12: [2023-05-08 11:22:57,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 7: [2023-05-08 11:22:57,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:22:57,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +31: [2023-05-08 11:22:57,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 7: [2023-05-08 11:22:57,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:22:57,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +18: [2023-05-08 11:22:57,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +18: [2023-05-08 11:22:57,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +18: [2023-05-08 11:22:57,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +22: [2023-05-08 11:22:57,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:22:57,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +21: [2023-05-08 11:22:57,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:22:57,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:22:57,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:22:57,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:22:57,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 0: [2023-05-08 11:22:57,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:22:57,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 0: [2023-05-08 11:22:57,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +27: [2023-05-08 11:22:57,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:22:57,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 8: [2023-05-08 11:22:57,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +21: [2023-05-08 11:22:57,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 9: [2023-05-08 11:22:57,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:22:57,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:22:57,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:22:57,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +31: [2023-05-08 11:22:57,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 9: [2023-05-08 11:22:57,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +25: [2023-05-08 11:22:57,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +19: [2023-05-08 11:22:57,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +19: [2023-05-08 11:22:57,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +29: [2023-05-08 11:22:57,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +14: [2023-05-08 11:22:57,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:22:57,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +31: [2023-05-08 11:22:57,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +27: [2023-05-08 11:22:57,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +14: [2023-05-08 11:22:57,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:22:57,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:22:57,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +19: [2023-05-08 11:22:57,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +19: [2023-05-08 11:22:57,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:22:57,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:22:57,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:22:57,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +22: [2023-05-08 11:22:57,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +14: [2023-05-08 11:22:57,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +14: [2023-05-08 11:22:57,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:22:57,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:22:57,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 9: [2023-05-08 11:22:57,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +18: [2023-05-08 11:22:57,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 9: [2023-05-08 11:22:57,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +14: [2023-05-08 11:22:57,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt... +29: [2023-05-08 11:22:57,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +29: [2023-05-08 11:22:57,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +29: [2023-05-08 11:22:57,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:22:57,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +29: [2023-05-08 11:22:57,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:22:57,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:22:57,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 7: [2023-05-08 11:22:57,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:22:57,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:22:57,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +14: [2023-05-08 11:22:57,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +25: [2023-05-08 11:22:57,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +27: [2023-05-08 11:22:57,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:22:57,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +29: [2023-05-08 11:22:57,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 9: [2023-05-08 11:22:57,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +14: [2023-05-08 11:22:57,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:22:57,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:22:57,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 9: [2023-05-08 11:22:57,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 9: [2023-05-08 11:22:57,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +27: [2023-05-08 11:22:57,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:22:57,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +19: [2023-05-08 11:22:57,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +19: [2023-05-08 11:22:57,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:22:57,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:22:57,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +19: [2023-05-08 11:22:57,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 9: [2023-05-08 11:22:57,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:22:57,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +19: [2023-05-08 11:22:57,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +27: [2023-05-08 11:22:57,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:22:57,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +14: [2023-05-08 11:22:57,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:22:57,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +14: [2023-05-08 11:22:57,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +17: [2023-05-08 11:22:57,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +17: [2023-05-08 11:22:57,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +17: [2023-05-08 11:22:57,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 8: [2023-05-08 11:22:57,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +17: [2023-05-08 11:22:57,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 8: [2023-05-08 11:22:57,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 8: [2023-05-08 11:22:57,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 7: [2023-05-08 11:22:57,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 8: [2023-05-08 11:22:57,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 8: [2023-05-08 11:22:57,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:22:57,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 3: [2023-05-08 11:22:57,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 3: [2023-05-08 11:22:57,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +29: [2023-05-08 11:22:57,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +10: [2023-05-08 11:22:57,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +10: [2023-05-08 11:22:57,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +14: [2023-05-08 11:22:57,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +10: [2023-05-08 11:22:57,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +10: [2023-05-08 11:22:57,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +10: [2023-05-08 11:22:57,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +10: [2023-05-08 11:22:57,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 3: [2023-05-08 11:22:57,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 3: [2023-05-08 11:22:57,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +10: [2023-05-08 11:22:57,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +19: [2023-05-08 11:22:57,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +14: [2023-05-08 11:22:57,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +10: [2023-05-08 11:22:57,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +14: [2023-05-08 11:22:57,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +19: [2023-05-08 11:22:57,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +17: [2023-05-08 11:22:57,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +14: [2023-05-08 11:22:57,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:22:57,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:22:57,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +30: [2023-05-08 11:22:57,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +17: [2023-05-08 11:22:57,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:22:57,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +30: [2023-05-08 11:22:57,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +30: [2023-05-08 11:22:57,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +30: [2023-05-08 11:22:57,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:22:57,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:22:57,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:22:57,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:22:57,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +14: [2023-05-08 11:22:57,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +14: [2023-05-08 11:22:57,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 8: [2023-05-08 11:22:57,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:22:57,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:22:57,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:22:57,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:22:57,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +29: [2023-05-08 11:22:57,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +29: [2023-05-08 11:22:57,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 7: [2023-05-08 11:22:57,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:22:57,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:22:57,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +10: [2023-05-08 11:22:57,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:22:57,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +14: [2023-05-08 11:22:57,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +14: [2023-05-08 11:22:57,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt. +14: [2023-05-08 11:22:57,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... +14: [2023-05-08 11:22:57,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt... + 8: [2023-05-08 11:22:57,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:22:57,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 8: [2023-05-08 11:22:57,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +30: [2023-05-08 11:22:57,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +18: [2023-05-08 11:22:57,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +18: [2023-05-08 11:22:57,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +18: [2023-05-08 11:22:57,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:22:57,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:22:57,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +18: [2023-05-08 11:22:57,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +17: [2023-05-08 11:22:57,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +17: [2023-05-08 11:22:57,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +18: [2023-05-08 11:22:57,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +10: [2023-05-08 11:22:57,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +18: [2023-05-08 11:22:57,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +10: [2023-05-08 11:22:57,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +10: [2023-05-08 11:22:57,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:22:57,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:22:57,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:22:57,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:22:57,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:22:57,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:22:57,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:22:57,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +17: [2023-05-08 11:22:57,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +17: [2023-05-08 11:22:57,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +17: [2023-05-08 11:22:57,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +17: [2023-05-08 11:22:57,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +30: [2023-05-08 11:22:57,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:22:57,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:22:57,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +30: [2023-05-08 11:22:57,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +30: [2023-05-08 11:22:57,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +17: [2023-05-08 11:22:57,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +17: [2023-05-08 11:22:57,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +10: [2023-05-08 11:22:57,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +10: [2023-05-08 11:22:57,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:22:57,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +18: [2023-05-08 11:22:57,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:22:57,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +30: [2023-05-08 11:22:57,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:22:57,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +30: [2023-05-08 11:22:57,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +20: [2023-05-08 11:22:57,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:22:57,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:22:57,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:22:57,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:22:57,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:22:57,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:22:57,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:22:57,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +30: [2023-05-08 11:22:57,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:22:57,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +20: [2023-05-08 11:22:57,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +10: [2023-05-08 11:22:57,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +18: [2023-05-08 11:22:57,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +18: [2023-05-08 11:22:57,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:22:57,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:22:57,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:22:57,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +27: [2023-05-08 11:22:57,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:22:57,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:22:57,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:22:57,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:22:57,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:22:57,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:22:57,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:22:57,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 2: [2023-05-08 11:22:57,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 2: [2023-05-08 11:22:57,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +28: [2023-05-08 11:22:57,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +28: [2023-05-08 11:22:57,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +28: [2023-05-08 11:22:57,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +28: [2023-05-08 11:22:57,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +28: [2023-05-08 11:22:57,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +28: [2023-05-08 11:22:57,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +28: [2023-05-08 11:22:57,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +28: [2023-05-08 11:22:57,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +10: [2023-05-08 11:22:57,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:22:57,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:22:57,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +20: [2023-05-08 11:22:57,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +26: [2023-05-08 11:22:57,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 7: [2023-05-08 11:22:57,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 7: [2023-05-08 11:22:57,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +26: [2023-05-08 11:22:57,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +26: [2023-05-08 11:22:57,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +28: [2023-05-08 11:22:57,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +26: [2023-05-08 11:22:57,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 7: [2023-05-08 11:22:57,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 7: [2023-05-08 11:22:57,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +28: [2023-05-08 11:22:57,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +28: [2023-05-08 11:22:57,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +18: [2023-05-08 11:22:57,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:22:57,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +20: [2023-05-08 11:22:57,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 4: [2023-05-08 11:22:57,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 4: [2023-05-08 11:22:57,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 4: [2023-05-08 11:22:57,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 6: [2023-05-08 11:22:57,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 6: [2023-05-08 11:22:57,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 6: [2023-05-08 11:22:57,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +20: [2023-05-08 11:22:57,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:22:57,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 1: [2023-05-08 11:22:57,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 1: [2023-05-08 11:22:57,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +27: [2023-05-08 11:22:57,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:22:57,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 1: [2023-05-08 11:22:57,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 1: [2023-05-08 11:22:57,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +15: [2023-05-08 11:22:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +11: [2023-05-08 11:22:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +16: [2023-05-08 11:22:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +11: [2023-05-08 11:22:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +15: [2023-05-08 11:22:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +13: [2023-05-08 11:22:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +13: [2023-05-08 11:22:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +16: [2023-05-08 11:22:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +11: [2023-05-08 11:22:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +16: [2023-05-08 11:22:57,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +11: [2023-05-08 11:22:57,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +13: [2023-05-08 11:22:57,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +30: [2023-05-08 11:22:57,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +30: [2023-05-08 11:22:57,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +30: [2023-05-08 11:22:57,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +20: [2023-05-08 11:22:57,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +30: [2023-05-08 11:22:57,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 2: [2023-05-08 11:22:57,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +26: [2023-05-08 11:22:57,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:22:57,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +13: [2023-05-08 11:22:57,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +28: [2023-05-08 11:22:57,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:22:57,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 7: [2023-05-08 11:22:57,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +26: [2023-05-08 11:22:57,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +26: [2023-05-08 11:22:57,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +26: [2023-05-08 11:22:57,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +28: [2023-05-08 11:22:57,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:22:57,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 4: [2023-05-08 11:22:57,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +28: [2023-05-08 11:22:57,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:22:57,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 1: [2023-05-08 11:22:57,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +26: [2023-05-08 11:22:57,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +26: [2023-05-08 11:22:57,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +26: [2023-05-08 11:22:57,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +26: [2023-05-08 11:22:57,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:22:57,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +26: [2023-05-08 11:22:57,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +26: [2023-05-08 11:22:57,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +26: [2023-05-08 11:22:57,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +26: [2023-05-08 11:22:57,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:22:57,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:22:57,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:22:57,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +27: [2023-05-08 11:22:57,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 0: [2023-05-08 11:22:57,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:22:57,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +31: [2023-05-08 11:22:57,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:22:57,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +31: [2023-05-08 11:22:57,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:22:57,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:22:57,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:22:57,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +16: [2023-05-08 11:22:57,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:22:57,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +31: [2023-05-08 11:22:57,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +11: [2023-05-08 11:22:57,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +11: [2023-05-08 11:22:57,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:22:57,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +11: [2023-05-08 11:22:57,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +16: [2023-05-08 11:22:57,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +11: [2023-05-08 11:22:57,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +11: [2023-05-08 11:22:57,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:22:57,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +24: [2023-05-08 11:22:57,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +13: [2023-05-08 11:22:57,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:22:57,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +24: [2023-05-08 11:22:57,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +24: [2023-05-08 11:22:57,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 1: [2023-05-08 11:22:57,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:22:57,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +28: [2023-05-08 11:22:57,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:22:57,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +28: [2023-05-08 11:22:57,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:22:57,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:22:57,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:22:57,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +11: [2023-05-08 11:22:57,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +11: [2023-05-08 11:22:57,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +24: [2023-05-08 11:22:57,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +11: [2023-05-08 11:22:57,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +30: [2023-05-08 11:22:57,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +27: [2023-05-08 11:22:57,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +16: [2023-05-08 11:22:57,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +15: [2023-05-08 11:22:57,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +11: [2023-05-08 11:22:57,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:22:57,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:22:57,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:22:57,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +11: [2023-05-08 11:22:57,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +11: [2023-05-08 11:22:57,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +11: [2023-05-08 11:22:57,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:22:57,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +26: [2023-05-08 11:22:57,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +26: [2023-05-08 11:22:57,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +28: [2023-05-08 11:22:57,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:22:57,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +28: [2023-05-08 11:22:57,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 2: [2023-05-08 11:22:57,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +28: [2023-05-08 11:22:57,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +28: [2023-05-08 11:22:57,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 7: [2023-05-08 11:22:57,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +28: [2023-05-08 11:22:57,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:22:57,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 5: [2023-05-08 11:22:57,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +20: [2023-05-08 11:22:57,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:22:57,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +27: [2023-05-08 11:22:57,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:22:57,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +16: [2023-05-08 11:22:58,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +16: [2023-05-08 11:22:58,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +16: [2023-05-08 11:22:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:22:58,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +16: [2023-05-08 11:22:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +26: [2023-05-08 11:22:58,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +26: [2023-05-08 11:22:58,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:22:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +26: [2023-05-08 11:22:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +26: [2023-05-08 11:22:58,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 6: [2023-05-08 11:22:58,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:22:58,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +24: [2023-05-08 11:22:58,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +24: [2023-05-08 11:22:58,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:22:58,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +28: [2023-05-08 11:22:58,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:22:58,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 1: [2023-05-08 11:22:58,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:22:58,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +31: [2023-05-08 11:22:58,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:22:58,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:22:58,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:22:58,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +16: [2023-05-08 11:22:58,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:22:58,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 7: [2023-05-08 11:22:58,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 7: [2023-05-08 11:22:58,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 7: [2023-05-08 11:22:58,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 7: [2023-05-08 11:22:58,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +28: [2023-05-08 11:22:58,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +16: [2023-05-08 11:22:58,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +16: [2023-05-08 11:22:58,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:22:58,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +16: [2023-05-08 11:22:58,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:22:58,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +20: [2023-05-08 11:22:58,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +20: [2023-05-08 11:22:58,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +24: [2023-05-08 11:22:58,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:22:58,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +11: [2023-05-08 11:22:58,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:22:58,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +16: [2023-05-08 11:22:58,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:22:58,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +16: [2023-05-08 11:22:58,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +11: [2023-05-08 11:22:58,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +13: [2023-05-08 11:22:58,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:22:58,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +16: [2023-05-08 11:22:58,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +16: [2023-05-08 11:22:58,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 4: [2023-05-08 11:22:58,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:22:58,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:22:58,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:22:58,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +12: [2023-05-08 11:22:58,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 4: [2023-05-08 11:22:58,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:22:58,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:22:58,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +12: [2023-05-08 11:22:58,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:22:58,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +16: [2023-05-08 11:22:58,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 4: [2023-05-08 11:22:58,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +13: [2023-05-08 11:22:58,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:22:58,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:22:58,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +12: [2023-05-08 11:22:58,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +11: [2023-05-08 11:22:58,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +11: [2023-05-08 11:22:58,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +15: [2023-05-08 11:22:58,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:22:58,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +13: [2023-05-08 11:22:58,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:22:58,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +20: [2023-05-08 11:22:58,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:22:58,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:22:58,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:22:58,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:22:58,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +30: [2023-05-08 11:22:58,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:22:58,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:22:58,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 1: [2023-05-08 11:22:58,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 1: [2023-05-08 11:22:58,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 7: [2023-05-08 11:22:58,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:22:58,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 7: [2023-05-08 11:22:58,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +31: [2023-05-08 11:22:58,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +30: [2023-05-08 11:22:58,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +30: [2023-05-08 11:22:58,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +27: [2023-05-08 11:22:58,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:22:58,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +31: [2023-05-08 11:22:58,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:22:58,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:22:58,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:22:58,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +15: [2023-05-08 11:22:58,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 6: [2023-05-08 11:22:58,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:22:58,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 6: [2023-05-08 11:22:58,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +31: [2023-05-08 11:22:58,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:22:58,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +12: [2023-05-08 11:22:58,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:22:58,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +30: [2023-05-08 11:22:58,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +30: [2023-05-08 11:22:58,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 1: [2023-05-08 11:22:58,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:22:58,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 5: [2023-05-08 11:22:58,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:22:58,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +28: [2023-05-08 11:22:58,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:22:58,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:22:58,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +16: [2023-05-08 11:22:58,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +16: [2023-05-08 11:22:58,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +20: [2023-05-08 11:22:58,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 4: [2023-05-08 11:22:58,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:22:58,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:22:58,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +11: [2023-05-08 11:22:58,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:22:58,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +24: [2023-05-08 11:22:58,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:22:58,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +16: [2023-05-08 11:22:58,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:22:58,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:22:58,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:22:58,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +16: [2023-05-08 11:22:58,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +24: [2023-05-08 11:22:58,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +24: [2023-05-08 11:22:58,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 2: [2023-05-08 11:22:58,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +26: [2023-05-08 11:22:58,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +26: [2023-05-08 11:22:58,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +26: [2023-05-08 11:22:58,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +26: [2023-05-08 11:22:58,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +26: [2023-05-08 11:22:58,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +28: [2023-05-08 11:22:58,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +28: [2023-05-08 11:22:58,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +30: [2023-05-08 11:22:58,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:22:58,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +26: [2023-05-08 11:22:58,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +28: [2023-05-08 11:22:58,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 1: [2023-05-08 11:22:58,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 1: [2023-05-08 11:22:58,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +28: [2023-05-08 11:22:58,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 4: [2023-05-08 11:22:58,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:22:58,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:22:58,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:22:58,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:22:58,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:22:58,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:22:58,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:22:58,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +25: [2023-05-08 11:22:58,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +25: [2023-05-08 11:22:58,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:22:58,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:22:58,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:22:58,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +30: [2023-05-08 11:22:58,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +31: [2023-05-08 11:22:58,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 2: [2023-05-08 11:22:58,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +12: [2023-05-08 11:22:58,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:22:58,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +23: [2023-05-08 11:22:58,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +16: [2023-05-08 11:22:58,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +16: [2023-05-08 11:22:58,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:22:58,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:22:58,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +23: [2023-05-08 11:22:58,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 5: [2023-05-08 11:22:58,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:22:58,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 0: [2023-05-08 11:22:58,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:22:58,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:22:58,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:22:58,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +11: [2023-05-08 11:22:58,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +11: [2023-05-08 11:22:58,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +11: [2023-05-08 11:22:58,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +11: [2023-05-08 11:22:58,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 5: [2023-05-08 11:22:58,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:22:58,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 6: [2023-05-08 11:22:58,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:22:58,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:22:58,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 1: [2023-05-08 11:22:58,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +24: [2023-05-08 11:22:58,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:22:58,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 1: [2023-05-08 11:22:58,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:22:58,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:22:58,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +20: [2023-05-08 11:22:58,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +16: [2023-05-08 11:22:58,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:22:58,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:22:58,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:22:58,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 5: [2023-05-08 11:22:58,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:22:58,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +26: [2023-05-08 11:22:58,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +26: [2023-05-08 11:22:58,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +26: [2023-05-08 11:22:58,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +26: [2023-05-08 11:22:58,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:22:58,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 1: [2023-05-08 11:22:58,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:22:58,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:22:58,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:22:58,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +16: [2023-05-08 11:22:58,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +11: [2023-05-08 11:22:58,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +11: [2023-05-08 11:22:58,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:22:58,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +25: [2023-05-08 11:22:58,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:22:58,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:22:58,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +11: [2023-05-08 11:22:58,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:22:58,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:22:58,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:22:58,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:22:58,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:22:58,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 1: [2023-05-08 11:22:58,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +11: [2023-05-08 11:22:58,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:22:58,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +16: [2023-05-08 11:22:58,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 1: [2023-05-08 11:22:58,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:22:58,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +12: [2023-05-08 11:22:58,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +11: [2023-05-08 11:22:58,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +28: [2023-05-08 11:22:58,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:22:58,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:22:58,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 0: [2023-05-08 11:22:58,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:22:58,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +16: [2023-05-08 11:22:58,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:22:58,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 4: [2023-05-08 11:22:58,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +25: [2023-05-08 11:22:58,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:22:58,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:22:58,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:22:58,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +25: [2023-05-08 11:22:58,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:22:58,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:22:58,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:22:58,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +20: [2023-05-08 11:22:58,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:22:58,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +16: [2023-05-08 11:22:58,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:22:58,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:22:58,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 5: [2023-05-08 11:22:58,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +25: [2023-05-08 11:22:58,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +25: [2023-05-08 11:22:58,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +25: [2023-05-08 11:22:58,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:22:58,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:22:58,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:22:58,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +23: [2023-05-08 11:22:58,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +10: [2023-05-08 11:22:58,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +10: [2023-05-08 11:22:58,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +10: [2023-05-08 11:22:58,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +10: [2023-05-08 11:22:58,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +23: [2023-05-08 11:22:58,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:22:58,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +12: [2023-05-08 11:22:58,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:22:58,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:22:58,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +17: [2023-05-08 11:22:58,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:22:58,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:22:58,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:22:58,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +17: [2023-05-08 11:22:58,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:22:58,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +18: [2023-05-08 11:22:58,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 9: [2023-05-08 11:22:58,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 9: [2023-05-08 11:22:58,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 9: [2023-05-08 11:22:58,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +17: [2023-05-08 11:22:58,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:22:58,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:22:58,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:22:58,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +25: [2023-05-08 11:22:58,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +25: [2023-05-08 11:22:58,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +25: [2023-05-08 11:22:58,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +12: [2023-05-08 11:22:58,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +12: [2023-05-08 11:22:58,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +12: [2023-05-08 11:22:58,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +18: [2023-05-08 11:22:58,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +25: [2023-05-08 11:22:58,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 9: [2023-05-08 11:22:58,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +10: [2023-05-08 11:22:58,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:22:58,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +31: [2023-05-08 11:22:58,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +31: [2023-05-08 11:22:58,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +10: [2023-05-08 11:22:58,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +10: [2023-05-08 11:22:58,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +10: [2023-05-08 11:22:58,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:22:58,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +12: [2023-05-08 11:22:58,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +18: [2023-05-08 11:22:58,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:22:58,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 9: [2023-05-08 11:22:58,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 9: [2023-05-08 11:22:58,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:22:58,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:22:58,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:22:58,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:22:58,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 9: [2023-05-08 11:22:58,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:22:58,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:22:58,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:22:58,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:22:58,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:22:58,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:22:58,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:22:58,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:22:58,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:22:58,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +21: [2023-05-08 11:22:58,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +13: [2023-05-08 11:22:58,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:22:58,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:22:58,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:22:58,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 9: [2023-05-08 11:22:58,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:22:58,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:22:58,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:22:58,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:22:58,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +21: [2023-05-08 11:22:58,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +12: [2023-05-08 11:22:58,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:22:58,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:22:58,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:22:58,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +12: [2023-05-08 11:22:58,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:22:58,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +14: [2023-05-08 11:22:58,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +14: [2023-05-08 11:22:58,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +14: [2023-05-08 11:22:58,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +17: [2023-05-08 11:22:58,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +12: [2023-05-08 11:22:58,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:22:58,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +21: [2023-05-08 11:22:58,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:22:58,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:22:58,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:22:58,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 8: [2023-05-08 11:22:58,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +14: [2023-05-08 11:22:58,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +17: [2023-05-08 11:22:58,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:22:58,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +21: [2023-05-08 11:22:58,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:22:58,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:22:58,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +17: [2023-05-08 11:22:58,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +18: [2023-05-08 11:22:58,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +18: [2023-05-08 11:22:58,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:22:58,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +22: [2023-05-08 11:22:58,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 8: [2023-05-08 11:22:58,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +22: [2023-05-08 11:22:58,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:22:58,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +10: [2023-05-08 11:22:58,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +10: [2023-05-08 11:22:58,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +10: [2023-05-08 11:22:58,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +18: [2023-05-08 11:22:58,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 8: [2023-05-08 11:22:58,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +10: [2023-05-08 11:22:58,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +10: [2023-05-08 11:22:58,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +10: [2023-05-08 11:22:58,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +13: [2023-05-08 11:22:58,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:22:58,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:22:58,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 9: [2023-05-08 11:22:58,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:22:58,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +21: [2023-05-08 11:22:58,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:22:58,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:22:58,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:22:58,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +25: [2023-05-08 11:22:58,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:22:58,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +25: [2023-05-08 11:22:58,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +25: [2023-05-08 11:22:58,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +12: [2023-05-08 11:22:58,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +31: [2023-05-08 11:22:58,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:22:58,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +31: [2023-05-08 11:22:58,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +14: [2023-05-08 11:22:58,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +14: [2023-05-08 11:22:58,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +12: [2023-05-08 11:22:58,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:22:58,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +18: [2023-05-08 11:22:58,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +18: [2023-05-08 11:22:58,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 9: [2023-05-08 11:22:58,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:22:58,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +14: [2023-05-08 11:22:58,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +14: [2023-05-08 11:22:58,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:22:58,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +10: [2023-05-08 11:22:58,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:22:58,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:22:58,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +22: [2023-05-08 11:22:58,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 9: [2023-05-08 11:22:58,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:22:58,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +10: [2023-05-08 11:22:58,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 8: [2023-05-08 11:22:58,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:22:58,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +13: [2023-05-08 11:22:58,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 8: [2023-05-08 11:22:58,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:22:58,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:22:58,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:22:58,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +31: [2023-05-08 11:22:58,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +18: [2023-05-08 11:22:58,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:22:58,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:22:58,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 9: [2023-05-08 11:22:58,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:22:58,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:22:58,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:22:58,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +21: [2023-05-08 11:22:58,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +25: [2023-05-08 11:22:58,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:22:58,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +21: [2023-05-08 11:22:58,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:22:58,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +22: [2023-05-08 11:22:58,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +22: [2023-05-08 11:22:58,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:22:58,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +31: [2023-05-08 11:22:58,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:22:58,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +31: [2023-05-08 11:22:58,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +21: [2023-05-08 11:22:58,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +21: [2023-05-08 11:22:58,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +21: [2023-05-08 11:22:58,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 8: [2023-05-08 11:22:58,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:22:58,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +21: [2023-05-08 11:22:58,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:22:58,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +14: [2023-05-08 11:22:58,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 8: [2023-05-08 11:22:58,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +13: [2023-05-08 11:22:58,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:22:58,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +14: [2023-05-08 11:22:58,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +22: [2023-05-08 11:22:58,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +21: [2023-05-08 11:22:58,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +21: [2023-05-08 11:22:58,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:22:58,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +31: [2023-05-08 11:22:58,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 8: [2023-05-08 11:22:58,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 8: [2023-05-08 11:22:58,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +14: [2023-05-08 11:22:58,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +14: [2023-05-08 11:22:58,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +14: [2023-05-08 11:22:58,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +14: [2023-05-08 11:22:58,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +21: [2023-05-08 11:22:58,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:22:58,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 8: [2023-05-08 11:22:58,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +21: [2023-05-08 11:22:58,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:22:58,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:22:58,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +22: [2023-05-08 11:22:58,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +21: [2023-05-08 11:22:58,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +21: [2023-05-08 11:22:58,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:22:58,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:22:58,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +23: [2023-05-08 11:22:58,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +23: [2023-05-08 11:22:58,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +23: [2023-05-08 11:22:58,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:22:58,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:22:58,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:22:58,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +23: [2023-05-08 11:22:58,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 0: [2023-05-08 11:22:58,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 8: [2023-05-08 11:22:58,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +14: [2023-05-08 11:22:58,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 8: [2023-05-08 11:22:58,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +27: [2023-05-08 11:22:58,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +27: [2023-05-08 11:22:58,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +27: [2023-05-08 11:22:58,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +22: [2023-05-08 11:22:58,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 0: [2023-05-08 11:22:58,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +27: [2023-05-08 11:22:58,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +22: [2023-05-08 11:22:58,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +22: [2023-05-08 11:22:58,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. + 0: [2023-05-08 11:22:58,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +22: [2023-05-08 11:22:58,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt. +21: [2023-05-08 11:22:58,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +21: [2023-05-08 11:22:58,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:22:58,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +21: [2023-05-08 11:22:58,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +21: [2023-05-08 11:22:58,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:22:58,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:22:58,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 2: [2023-05-08 11:22:58,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:22:58,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 2: [2023-05-08 11:22:58,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 2: [2023-05-08 11:22:58,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 2: [2023-05-08 11:22:58,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 2: [2023-05-08 11:22:58,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 2: [2023-05-08 11:22:58,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:22:58,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +27: [2023-05-08 11:22:58,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:22:58,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +23: [2023-05-08 11:22:58,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +23: [2023-05-08 11:22:58,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 9: [2023-05-08 11:22:58,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 9: [2023-05-08 11:22:58,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:22:58,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:22:58,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:22:58,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:22:58,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 9: [2023-05-08 11:22:58,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +22: [2023-05-08 11:22:58,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 9: [2023-05-08 11:22:58,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 9: [2023-05-08 11:22:58,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 9: [2023-05-08 11:22:58,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 9: [2023-05-08 11:22:58,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:22:58,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt... + 9: [2023-05-08 11:22:58,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +21: [2023-05-08 11:22:58,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:22:58,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 2: [2023-05-08 11:22:58,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:22:58,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:22:58,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:22:58,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +23: [2023-05-08 11:22:58,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:22:58,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:22:58,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +27: [2023-05-08 11:22:58,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:22:58,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 0: [2023-05-08 11:22:58,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:22:58,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:22:58,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +22: [2023-05-08 11:22:58,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +23: [2023-05-08 11:22:58,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:22:58,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:22:58,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:22:58,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... +22: [2023-05-08 11:22:58,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 9: [2023-05-08 11:22:58,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:22:58,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:22:58,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 2: [2023-05-08 11:22:58,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 2: [2023-05-08 11:22:58,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 2: [2023-05-08 11:22:58,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 2: [2023-05-08 11:22:58,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:22:58,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:22:58,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 9: [2023-05-08 11:22:58,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 9: [2023-05-08 11:22:58,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 9: [2023-05-08 11:22:58,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 2: [2023-05-08 11:22:58,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:22:58,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:22:58,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 9: [2023-05-08 11:22:58,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 2: [2023-05-08 11:22:58,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:22:58,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 0: [2023-05-08 11:22:58,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:22:58,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 9: [2023-05-08 11:22:58,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +22: [2023-05-08 11:22:58,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:22:58,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt... + 9: [2023-05-08 11:22:58,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +24: [2023-05-08 11:22:58,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +24: [2023-05-08 11:22:58,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +24: [2023-05-08 11:22:58,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +24: [2023-05-08 11:22:58,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:22:58,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:22:58,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:22:58,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +24: [2023-05-08 11:22:58,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:22:58,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +24: [2023-05-08 11:22:58,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +24: [2023-05-08 11:22:58,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +24: [2023-05-08 11:22:58,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +24: [2023-05-08 11:22:58,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +24: [2023-05-08 11:22:58,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:22:58,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 6: [2023-05-08 11:22:58,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 6: [2023-05-08 11:22:58,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 6: [2023-05-08 11:22:58,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:22:58,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:22:58,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 6: [2023-05-08 11:22:58,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 6: [2023-05-08 11:22:58,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 6: [2023-05-08 11:22:58,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:22:58,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 6: [2023-05-08 11:22:58,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 6: [2023-05-08 11:22:58,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:22:58,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:22:58,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:22:58,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:22:58,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:22:58,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:22:58,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:22:58,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:22:58,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:22:58,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +14: [2023-05-08 11:22:58,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +14: [2023-05-08 11:22:58,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:22:58,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:22:58,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:22:58,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +14: [2023-05-08 11:22:58,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:22:58,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 6: [2023-05-08 11:22:58,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:22:58,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:22:58,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 7: [2023-05-08 11:22:58,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 7: [2023-05-08 11:22:58,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 7: [2023-05-08 11:22:58,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 7: [2023-05-08 11:22:58,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 7: [2023-05-08 11:22:58,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 7: [2023-05-08 11:22:58,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 7: [2023-05-08 11:22:58,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +14: [2023-05-08 11:22:58,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 7: [2023-05-08 11:22:58,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 6: [2023-05-08 11:22:58,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:22:58,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:22:58,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +30: [2023-05-08 11:22:58,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:22:58,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:22:58,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:22:58,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:22:58,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:22:58,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:22:58,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:22:58,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:22:58,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:22:58,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 6: [2023-05-08 11:22:58,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +14: [2023-05-08 11:22:58,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:22:58,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:22:58,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:22:58,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:22:58,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:22:58,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:22:58,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 7: [2023-05-08 11:22:58,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:22:58,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +14: [2023-05-08 11:22:58,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:22:58,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:22:58,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:22:58,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:22:58,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:22:58,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:22:58,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:22:58,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:22:58,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:22:58,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:22:58,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:22:58,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:22:58,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:22:58,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:22:58,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:22:58,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +20: [2023-05-08 11:22:58,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +20: [2023-05-08 11:22:58,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:22:58,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:22:58,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:22:58,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 7: [2023-05-08 11:22:58,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:22:58,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:22:58,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:22:58,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:22:58,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:22:58,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:22:58,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +30: [2023-05-08 11:22:58,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:22:58,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +30: [2023-05-08 11:22:58,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:22:58,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:22:58,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:22:58,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:22:58,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:22:58,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:22:58,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:22:58,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:22:58,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:22:58,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:22:58,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:22:58,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:22:58,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:22:58,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:22:58,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:22:58,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:22:58,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:22:58,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:22:58,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:22:58,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:22:58,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:22:58,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:22:58,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:22:58,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:22:58,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:22:58,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:22:58,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +20: [2023-05-08 11:22:58,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:22:58,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +18: [2023-05-08 11:22:58,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:22:58,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:22:58,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:22:58,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:22:58,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:22:58,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:22:58,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:22:58,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:22:58,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:22:58,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:22:58,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:22:58,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:22:58,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:22:58,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:22:58,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:22:58,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:22:58,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:22:58,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:22:58,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:22:58,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:22:58,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:22:58,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:22:58,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:22:58,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:22:58,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:22:58,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:22:58,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:22:58,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:22:58,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:22:58,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:22:58,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:22:58,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:22:58,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:22:58,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:22:58,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:22:58,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:22:58,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:22:58,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 4: [2023-05-08 11:22:58,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:22:58,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:22:58,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:22:58,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:22:58,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +17: [2023-05-08 11:22:58,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +17: [2023-05-08 11:22:58,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +17: [2023-05-08 11:22:58,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +29: [2023-05-08 11:22:58,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +29: [2023-05-08 11:22:58,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +29: [2023-05-08 11:22:58,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +29: [2023-05-08 11:22:58,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +31: [2023-05-08 11:22:58,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:22:58,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +15: [2023-05-08 11:22:58,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +15: [2023-05-08 11:22:58,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +15: [2023-05-08 11:22:58,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:22:58,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +15: [2023-05-08 11:22:58,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:22:58,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +15: [2023-05-08 11:22:58,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:22:58,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:22:58,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:22:58,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 4: [2023-05-08 11:22:58,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +18: [2023-05-08 11:22:58,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +18: [2023-05-08 11:22:58,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:22:58,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:22:58,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +10: [2023-05-08 11:22:58,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +10: [2023-05-08 11:22:58,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:22:58,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +10: [2023-05-08 11:22:58,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +10: [2023-05-08 11:22:58,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +10: [2023-05-08 11:22:58,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +10: [2023-05-08 11:22:58,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +10: [2023-05-08 11:22:58,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:22:58,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:22:58,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +17: [2023-05-08 11:22:58,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:22:58,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +18: [2023-05-08 11:22:58,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +10: [2023-05-08 11:22:58,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:22:58,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:22:58,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:22:58,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:22:58,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:22:58,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 1: [2023-05-08 11:22:58,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 1: [2023-05-08 11:22:58,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 1: [2023-05-08 11:22:58,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:22:58,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:22:58,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 1: [2023-05-08 11:22:58,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:22:58,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:22:58,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:22:58,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:22:58,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 1: [2023-05-08 11:22:58,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:22:58,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:22:58,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 8: [2023-05-08 11:22:58,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 8: [2023-05-08 11:22:58,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:22:58,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:22:58,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 8: [2023-05-08 11:22:58,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 8: [2023-05-08 11:22:58,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:22:58,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:22:58,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:22:58,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:22:58,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:22:58,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:22:58,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:22:58,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:22:58,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:22:58,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +16: [2023-05-08 11:22:58,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:22:58,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:22:58,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:22:58,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +15: [2023-05-08 11:22:58,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:22:58,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:22:58,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:22:58,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +10: [2023-05-08 11:22:58,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:22:58,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:22:58,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:22:58,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:22:58,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:22:58,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:22:58,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:22:58,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +15: [2023-05-08 11:22:58,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +17: [2023-05-08 11:22:58,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +12: [2023-05-08 11:22:58,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:22:58,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:22:58,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +12: [2023-05-08 11:22:58,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:22:58,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +10: [2023-05-08 11:22:58,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +10: [2023-05-08 11:22:58,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:22:58,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 8: [2023-05-08 11:22:58,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:22:58,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 8: [2023-05-08 11:22:58,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +15: [2023-05-08 11:22:58,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +15: [2023-05-08 11:22:58,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:22:58,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +28: [2023-05-08 11:22:58,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +28: [2023-05-08 11:22:58,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +28: [2023-05-08 11:22:58,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:22:58,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:22:58,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +10: [2023-05-08 11:22:58,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +15: [2023-05-08 11:22:58,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:22:58,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:22:58,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:22:58,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 1: [2023-05-08 11:22:58,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 1: [2023-05-08 11:22:58,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +28: [2023-05-08 11:22:58,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:22:58,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:22:58,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 8: [2023-05-08 11:22:58,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:22:58,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:22:58,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:22:58,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:22:58,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +15: [2023-05-08 11:22:58,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +10: [2023-05-08 11:22:58,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +10: [2023-05-08 11:22:58,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 8: [2023-05-08 11:22:58,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +12: [2023-05-08 11:22:58,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 8: [2023-05-08 11:22:58,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +17: [2023-05-08 11:22:58,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 1: [2023-05-08 11:22:58,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:22:58,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +29: [2023-05-08 11:22:58,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:22:58,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 8: [2023-05-08 11:22:58,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 1: [2023-05-08 11:22:58,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 1: [2023-05-08 11:22:58,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 8: [2023-05-08 11:22:58,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +10: [2023-05-08 11:22:58,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:22:58,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:22:58,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:22:58,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:22:58,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +26: [2023-05-08 11:22:58,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +26: [2023-05-08 11:22:58,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:22:58,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +26: [2023-05-08 11:22:58,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +16: [2023-05-08 11:22:58,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:22:58,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 5: [2023-05-08 11:22:58,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:22:58,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:22:58,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:22:58,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:22:58,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +28: [2023-05-08 11:22:58,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:22:58,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:22:58,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:22:58,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:22:58,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 1: [2023-05-08 11:22:58,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 5: [2023-05-08 11:22:58,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:22:58,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:22:58,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +28: [2023-05-08 11:22:58,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:22:58,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +10: [2023-05-08 11:22:58,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:22:58,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +16: [2023-05-08 11:22:58,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:22:58,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:22:58,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +28: [2023-05-08 11:22:58,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +28: [2023-05-08 11:22:58,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:22:58,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 1: [2023-05-08 11:22:58,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 5: [2023-05-08 11:22:58,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:22:58,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:22:58,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:22:58,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:22:58,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:22:58,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:22:58,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:22:58,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 5: [2023-05-08 11:22:58,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +28: [2023-05-08 11:22:58,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +25: [2023-05-08 11:22:58,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:22:58,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +25: [2023-05-08 11:22:58,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:22:58,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:22:58,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +25: [2023-05-08 11:22:58,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +25: [2023-05-08 11:22:58,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:22:58,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 5: [2023-05-08 11:22:58,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:22:58,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:22:58,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:22:58,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:22:58,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:22:58,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +11: [2023-05-08 11:22:58,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +11: [2023-05-08 11:22:58,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:22:58,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +11: [2023-05-08 11:22:58,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:22:58,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:22:58,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +11: [2023-05-08 11:22:58,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +17: [2023-05-08 11:22:58,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:22:58,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:22:58,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +17: [2023-05-08 11:22:58,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +17: [2023-05-08 11:22:58,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:22:58,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +17: [2023-05-08 11:22:58,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:22:58,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:22:58,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 5: [2023-05-08 11:22:58,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:22:58,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:22:58,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:22:58,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:22:58,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:22:58,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:22:58,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:22:58,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:22:58,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +25: [2023-05-08 11:22:58,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:22:58,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:22:58,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 5: [2023-05-08 11:22:58,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:22:58,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:22:58,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:22:58,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:22:58,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:22:58,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:22:58,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +11: [2023-05-08 11:22:58,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +11: [2023-05-08 11:22:58,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +11: [2023-05-08 11:22:58,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +11: [2023-05-08 11:22:58,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:22:58,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:22:58,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:22:58,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:22:58,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +17: [2023-05-08 11:22:58,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:22:58,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:22:58,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:22:58,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:22:58,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:22:58,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:22:58,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +25: [2023-05-08 11:22:58,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +11: [2023-05-08 11:22:58,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +11: [2023-05-08 11:22:58,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +11: [2023-05-08 11:22:58,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:22:58,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +11: [2023-05-08 11:22:58,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:22:58,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:22:58,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:22:58,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:22:58,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:22:58,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:22:58,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:22:58,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:22:58,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 9: [2023-05-08 11:22:58,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 9: [2023-05-08 11:22:58,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 9: [2023-05-08 11:22:58,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 9: [2023-05-08 11:22:58,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 9: [2023-05-08 11:22:58,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 9: [2023-05-08 11:22:58,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 9: [2023-05-08 11:22:58,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 9: [2023-05-08 11:22:58,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 9: [2023-05-08 11:22:58,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 9: [2023-05-08 11:22:58,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 9: [2023-05-08 11:22:58,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:22:58,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:22:58,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 9: [2023-05-08 11:22:58,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:22:58,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +22: [2023-05-08 11:22:58,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +22: [2023-05-08 11:22:58,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:22:58,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +22: [2023-05-08 11:22:58,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +22: [2023-05-08 11:22:58,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:22:58,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:22:58,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:22:58,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +22: [2023-05-08 11:22:58,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +17: [2023-05-08 11:22:58,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +17: [2023-05-08 11:22:58,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +29: [2023-05-08 11:22:58,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +17: [2023-05-08 11:22:58,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +29: [2023-05-08 11:22:58,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +29: [2023-05-08 11:22:58,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +29: [2023-05-08 11:22:58,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 2: [2023-05-08 11:22:58,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 2: [2023-05-08 11:22:58,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 2: [2023-05-08 11:22:58,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 2: [2023-05-08 11:22:58,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 2: [2023-05-08 11:22:58,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 2: [2023-05-08 11:22:58,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 2: [2023-05-08 11:22:58,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +24: [2023-05-08 11:22:58,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:22:58,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +21: [2023-05-08 11:22:58,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +21: [2023-05-08 11:22:58,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +24: [2023-05-08 11:22:58,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:22:58,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +24: [2023-05-08 11:22:58,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +24: [2023-05-08 11:22:58,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +21: [2023-05-08 11:22:58,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +21: [2023-05-08 11:22:58,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +21: [2023-05-08 11:22:58,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +21: [2023-05-08 11:22:58,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +21: [2023-05-08 11:22:58,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +21: [2023-05-08 11:22:58,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +24: [2023-05-08 11:22:58,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:22:58,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 2: [2023-05-08 11:22:58,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:22:58,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:22:58,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +17: [2023-05-08 11:22:58,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:22:58,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:22:58,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:22:58,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:22:58,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:22:58,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +17: [2023-05-08 11:22:58,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +17: [2023-05-08 11:22:58,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:22:58,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +22: [2023-05-08 11:22:58,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 2: [2023-05-08 11:22:58,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +21: [2023-05-08 11:22:58,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +21: [2023-05-08 11:22:58,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:22:58,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:22:58,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +21: [2023-05-08 11:22:58,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +21: [2023-05-08 11:22:58,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:22:58,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:22:58,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 2: [2023-05-08 11:22:58,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 2: [2023-05-08 11:22:58,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +22: [2023-05-08 11:22:58,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +22: [2023-05-08 11:22:58,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +22: [2023-05-08 11:22:58,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:22:58,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +22: [2023-05-08 11:22:58,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:22:58,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 2: [2023-05-08 11:22:58,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:22:58,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:22:58,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +24: [2023-05-08 11:22:58,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +24: [2023-05-08 11:22:58,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +21: [2023-05-08 11:22:58,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +21: [2023-05-08 11:22:58,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +21: [2023-05-08 11:22:58,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +24: [2023-05-08 11:22:58,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +21: [2023-05-08 11:22:58,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:22:58,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:22:58,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 2: [2023-05-08 11:22:58,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:22:58,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:22:58,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 2: [2023-05-08 11:22:58,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +24: [2023-05-08 11:22:58,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 2: [2023-05-08 11:22:58,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 2: [2023-05-08 11:22:58,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +13: [2023-05-08 11:22:58,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:22:58,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +13: [2023-05-08 11:22:58,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:22:58,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:22:58,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +13: [2023-05-08 11:22:58,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +13: [2023-05-08 11:22:58,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:22:58,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +13: [2023-05-08 11:22:58,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:22:58,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:22:58,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:22:58,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:22:58,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:22:58,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:22:58,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:22:58,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:22:58,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:22:58,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:22:58,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:22:58,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:22:58,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:22:58,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:22:58,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:22:58,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:22:58,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:22:58,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:22:58,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:22:58,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +13: [2023-05-08 11:22:58,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +13: [2023-05-08 11:22:58,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +13: [2023-05-08 11:22:58,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +13: [2023-05-08 11:22:58,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:22:58,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:22:58,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:22:58,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:22:58,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:22:58,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:22:58,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:22:58,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +30: [2023-05-08 11:22:58,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:22:58,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:22:58,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:22:58,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:22:58,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +30: [2023-05-08 11:22:58,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:22:58,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 6: [2023-05-08 11:22:58,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 6: [2023-05-08 11:22:58,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:22:58,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 6: [2023-05-08 11:22:58,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:22:58,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 6: [2023-05-08 11:22:58,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:22:58,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:22:58,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:22:58,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:22:58,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 0: [2023-05-08 11:22:58,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 0: [2023-05-08 11:22:58,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 0: [2023-05-08 11:22:58,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 0: [2023-05-08 11:22:58,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:22:58,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:22:58,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:22:58,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:22:58,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:22:58,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:22:58,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:22:58,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:22:58,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:22:58,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:22:58,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 6: [2023-05-08 11:22:58,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +18: [2023-05-08 11:22:58,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +18: [2023-05-08 11:22:58,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +18: [2023-05-08 11:22:58,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +18: [2023-05-08 11:22:58,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +18: [2023-05-08 11:22:58,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +14: [2023-05-08 11:22:58,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:22:58,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +18: [2023-05-08 11:22:58,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:22:58,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:22:58,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:22:58,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +14: [2023-05-08 11:22:58,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +18: [2023-05-08 11:22:58,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +14: [2023-05-08 11:22:58,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:22:58,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:22:58,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +14: [2023-05-08 11:22:58,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +18: [2023-05-08 11:22:58,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +15: [2023-05-08 11:22:58,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:22:58,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:22:58,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:22:58,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:22:58,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:22:58,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:22:58,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:22:58,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +19: [2023-05-08 11:22:58,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +19: [2023-05-08 11:22:58,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:22:58,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 6: [2023-05-08 11:22:58,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:22:58,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +19: [2023-05-08 11:22:58,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +19: [2023-05-08 11:22:58,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:22:58,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:22:58,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +19: [2023-05-08 11:22:58,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 0: [2023-05-08 11:22:58,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:22:58,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:22:58,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:22:58,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:22:58,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:22:58,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +18: [2023-05-08 11:22:58,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +14: [2023-05-08 11:22:58,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:22:58,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +18: [2023-05-08 11:22:58,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 6: [2023-05-08 11:22:58,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +14: [2023-05-08 11:22:58,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:22:58,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:22:58,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:22:58,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 0: [2023-05-08 11:22:58,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +18: [2023-05-08 11:22:58,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:22:58,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:22:58,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +27: [2023-05-08 11:22:58,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +27: [2023-05-08 11:22:58,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +27: [2023-05-08 11:22:58,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +27: [2023-05-08 11:22:58,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:22:58,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:22:58,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +27: [2023-05-08 11:22:58,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +15: [2023-05-08 11:22:58,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:22:58,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +18: [2023-05-08 11:22:58,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +14: [2023-05-08 11:22:58,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:22:58,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:22:58,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +18: [2023-05-08 11:22:58,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:22:58,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:22:58,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:22:58,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:22:58,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +20: [2023-05-08 11:22:58,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:22:58,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +20: [2023-05-08 11:22:58,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +20: [2023-05-08 11:22:58,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:22:58,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:22:58,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +20: [2023-05-08 11:22:58,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:22:58,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +19: [2023-05-08 11:22:58,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:22:58,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +14: [2023-05-08 11:22:58,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:22:58,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +18: [2023-05-08 11:22:58,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:22:58,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +18: [2023-05-08 11:22:58,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:22:58,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:22:58,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:22:58,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +10: [2023-05-08 11:22:58,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:22:58,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +27: [2023-05-08 11:22:58,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:22:58,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:22:58,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:22:58,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:22:58,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:22:58,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:22:58,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:22:58,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +18: [2023-05-08 11:22:58,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:22:58,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +20: [2023-05-08 11:22:58,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:22:58,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +19: [2023-05-08 11:22:58,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:22:58,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:22:58,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:22:58,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:22:58,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:22:58,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:22:58,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:22:58,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:22:58,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:22:58,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:22:58,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:22:58,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:22:58,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:22:58,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:22:58,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +29: [2023-05-08 11:22:58,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +29: [2023-05-08 11:22:58,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +29: [2023-05-08 11:22:58,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +20: [2023-05-08 11:22:58,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +19: [2023-05-08 11:22:58,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:22:58,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +20: [2023-05-08 11:22:58,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:22:58,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:22:58,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:22:58,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:22:58,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +27: [2023-05-08 11:22:58,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:22:58,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:22:58,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 3: [2023-05-08 11:22:58,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 3: [2023-05-08 11:22:58,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 3: [2023-05-08 11:22:58,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +10: [2023-05-08 11:22:58,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:22:58,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +20: [2023-05-08 11:22:58,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:22:58,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:22:58,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:22:58,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:22:58,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:22:58,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:22:58,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +20: [2023-05-08 11:22:58,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:22:58,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:22:58,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +19: [2023-05-08 11:22:58,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +19: [2023-05-08 11:22:58,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +19: [2023-05-08 11:22:58,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 3: [2023-05-08 11:22:58,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +10: [2023-05-08 11:22:58,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:22:58,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:22:58,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:22:58,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:22:58,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:22:58,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:22:58,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:22:58,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:22:58,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:22:58,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +19: [2023-05-08 11:22:58,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:22:58,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 3: [2023-05-08 11:22:58,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +19: [2023-05-08 11:22:58,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:22:58,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +19: [2023-05-08 11:22:58,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +19: [2023-05-08 11:22:58,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:22:58,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 3: [2023-05-08 11:22:58,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +19: [2023-05-08 11:22:58,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:22:58,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +17: [2023-05-08 11:22:58,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:22:58,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:22:58,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:22:58,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:22:58,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:22:58,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:22:58,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:22:58,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +25: [2023-05-08 11:22:58,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:22:58,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +25: [2023-05-08 11:22:58,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:22:58,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +25: [2023-05-08 11:22:58,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:22:58,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:22:58,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +25: [2023-05-08 11:22:58,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:22:58,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:22:58,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:22:58,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:22:58,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:22:58,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:22:58,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:22:58,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:22:58,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +17: [2023-05-08 11:22:58,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +17: [2023-05-08 11:22:58,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +17: [2023-05-08 11:22:58,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:22:58,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:22:58,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:22:58,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:22:58,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:22:58,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:22:58,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +11: [2023-05-08 11:22:58,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +11: [2023-05-08 11:22:58,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +11: [2023-05-08 11:22:58,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +23: [2023-05-08 11:22:58,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +11: [2023-05-08 11:22:58,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:22:58,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +23: [2023-05-08 11:22:58,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:22:58,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +11: [2023-05-08 11:22:58,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +23: [2023-05-08 11:22:58,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +11: [2023-05-08 11:22:58,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +11: [2023-05-08 11:22:58,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:22:58,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +11: [2023-05-08 11:22:58,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:22:58,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:22:58,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +17: [2023-05-08 11:22:58,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +17: [2023-05-08 11:22:58,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +17: [2023-05-08 11:22:58,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +17: [2023-05-08 11:22:58,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:22:58,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +11: [2023-05-08 11:22:58,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:22:58,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:22:58,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:22:58,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:22:58,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +11: [2023-05-08 11:22:58,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:22:58,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:22:58,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:22:58,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +11: [2023-05-08 11:22:58,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +11: [2023-05-08 11:22:58,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +23: [2023-05-08 11:22:58,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:22:58,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:22:58,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +23: [2023-05-08 11:22:58,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:22:58,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:22:58,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +11: [2023-05-08 11:22:58,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +17: [2023-05-08 11:22:58,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:22:58,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +23: [2023-05-08 11:22:58,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:22:58,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:22:58,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:22:58,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 3: [2023-05-08 11:22:58,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 3: [2023-05-08 11:22:58,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 3: [2023-05-08 11:22:58,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +17: [2023-05-08 11:22:58,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:22:58,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 3: [2023-05-08 11:22:58,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 8: [2023-05-08 11:22:58,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:22:58,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 8: [2023-05-08 11:22:58,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:22:58,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:22:58,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 8: [2023-05-08 11:22:58,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 8: [2023-05-08 11:22:58,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:22:58,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +16: [2023-05-08 11:22:58,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:22:58,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:22:58,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:22:58,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:22:58,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:22:58,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:22:58,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:22:58,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:22:58,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 8: [2023-05-08 11:22:58,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:22:58,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:22:58,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 3: [2023-05-08 11:22:58,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 8: [2023-05-08 11:22:58,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:22:58,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:22:59,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +19: [2023-05-08 11:22:59,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +19: [2023-05-08 11:22:59,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 8: [2023-05-08 11:22:59,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:22:59,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +19: [2023-05-08 11:22:59,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 8: [2023-05-08 11:22:59,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:22:59,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +12: [2023-05-08 11:22:59,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +12: [2023-05-08 11:22:59,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +12: [2023-05-08 11:22:59,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +12: [2023-05-08 11:22:59,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +12: [2023-05-08 11:22:59,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +12: [2023-05-08 11:22:59,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +12: [2023-05-08 11:22:59,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:22:59,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:22:59,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:22:59,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:22:59,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:22:59,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:22:59,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:22:59,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:22:59,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:22:59,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:22:59,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:22:59,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:22:59,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:22:59,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:22:59,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:22:59,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:22:59,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:22:59,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:22:59,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +26: [2023-05-08 11:22:59,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:22:59,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:22:59,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:22:59,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:22:59,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:22:59,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +12: [2023-05-08 11:22:59,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +12: [2023-05-08 11:22:59,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:22:59,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:22:59,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +12: [2023-05-08 11:22:59,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:22:59,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:22:59,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:22:59,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:22:59,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +19: [2023-05-08 11:22:59,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +19: [2023-05-08 11:22:59,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +19: [2023-05-08 11:22:59,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +19: [2023-05-08 11:22:59,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +12: [2023-05-08 11:22:59,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:22:59,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +19: [2023-05-08 11:22:59,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +19: [2023-05-08 11:22:59,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +19: [2023-05-08 11:22:59,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:22:59,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:22:59,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +12: [2023-05-08 11:22:59,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +26: [2023-05-08 11:22:59,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:22:59,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:22:59,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:22:59,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:22:59,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +28: [2023-05-08 11:22:59,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:22:59,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +26: [2023-05-08 11:22:59,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:22:59,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +28: [2023-05-08 11:22:59,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:22:59,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +26: [2023-05-08 11:22:59,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +26: [2023-05-08 11:22:59,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +26: [2023-05-08 11:22:59,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:22:59,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +29: [2023-05-08 11:22:59,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +29: [2023-05-08 11:22:59,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +29: [2023-05-08 11:22:59,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +29: [2023-05-08 11:22:59,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:22:59,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:22:59,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:22:59,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +25: [2023-05-08 11:22:59,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:22:59,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:22:59,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:22:59,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:22:59,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:22:59,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:22:59,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:22:59,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 5: [2023-05-08 11:22:59,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +31: [2023-05-08 11:22:59,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +31: [2023-05-08 11:22:59,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 1: [2023-05-08 11:22:59,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 1: [2023-05-08 11:22:59,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 1: [2023-05-08 11:22:59,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +31: [2023-05-08 11:22:59,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 1: [2023-05-08 11:22:59,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +31: [2023-05-08 11:22:59,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 5: [2023-05-08 11:22:59,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 5: [2023-05-08 11:22:59,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:22:59,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +13: [2023-05-08 11:22:59,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +13: [2023-05-08 11:22:59,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +13: [2023-05-08 11:22:59,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +25: [2023-05-08 11:22:59,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:22:59,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:22:59,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:22:59,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:22:59,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +11: [2023-05-08 11:22:59,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:22:59,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:22:59,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:22:59,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:22:59,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:22:59,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:22:59,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:22:59,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:22:59,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:22:59,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:22:59,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:22:59,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:22:59,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:22:59,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +23: [2023-05-08 11:22:59,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:22:59,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:22:59,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:22:59,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:22:59,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:22:59,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:22:59,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +23: [2023-05-08 11:22:59,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:22:59,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:22:59,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:22:59,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:22:59,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:22:59,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +17: [2023-05-08 11:22:59,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +17: [2023-05-08 11:22:59,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:22:59,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:22:59,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:22:59,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:22:59,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:22:59,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +31: [2023-05-08 11:22:59,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:22:59,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +31: [2023-05-08 11:22:59,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +31: [2023-05-08 11:22:59,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:22:59,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:22:59,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +31: [2023-05-08 11:22:59,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +11: [2023-05-08 11:22:59,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:22:59,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +13: [2023-05-08 11:22:59,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:22:59,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 5: [2023-05-08 11:22:59,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +13: [2023-05-08 11:22:59,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:22:59,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +13: [2023-05-08 11:22:59,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +23: [2023-05-08 11:22:59,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:22:59,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:22:59,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +31: [2023-05-08 11:22:59,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:22:59,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +17: [2023-05-08 11:22:59,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:22:59,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +17: [2023-05-08 11:22:59,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:22:59,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:22:59,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:22:59,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:22:59,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +19: [2023-05-08 11:22:59,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +19: [2023-05-08 11:22:59,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +23: [2023-05-08 11:22:59,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:22:59,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:22:59,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +13: [2023-05-08 11:22:59,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +11: [2023-05-08 11:22:59,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:22:59,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +11: [2023-05-08 11:22:59,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 3: [2023-05-08 11:22:59,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 3: [2023-05-08 11:22:59,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 3: [2023-05-08 11:22:59,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +23: [2023-05-08 11:22:59,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:22:59,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:22:59,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:22:59,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:22:59,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:22:59,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:22:59,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:22:59,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:22:59,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:22:59,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:22:59,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:22:59,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:22:59,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:22:59,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +17: [2023-05-08 11:22:59,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +17: [2023-05-08 11:22:59,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +17: [2023-05-08 11:22:59,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 3: [2023-05-08 11:22:59,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:22:59,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:22:59,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +17: [2023-05-08 11:22:59,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +17: [2023-05-08 11:22:59,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +17: [2023-05-08 11:22:59,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +27: [2023-05-08 11:22:59,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +20: [2023-05-08 11:22:59,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +27: [2023-05-08 11:22:59,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +27: [2023-05-08 11:22:59,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 8: [2023-05-08 11:22:59,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 8: [2023-05-08 11:22:59,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 8: [2023-05-08 11:22:59,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +20: [2023-05-08 11:22:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +20: [2023-05-08 11:22:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +27: [2023-05-08 11:22:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +15: [2023-05-08 11:22:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +15: [2023-05-08 11:22:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +15: [2023-05-08 11:22:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 8: [2023-05-08 11:22:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +30: [2023-05-08 11:22:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +30: [2023-05-08 11:22:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +20: [2023-05-08 11:22:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +30: [2023-05-08 11:22:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +15: [2023-05-08 11:22:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +10: [2023-05-08 11:22:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +10: [2023-05-08 11:22:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +10: [2023-05-08 11:22:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +30: [2023-05-08 11:22:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 0: [2023-05-08 11:22:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 0: [2023-05-08 11:22:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +28: [2023-05-08 11:22:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 0: [2023-05-08 11:22:59,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +28: [2023-05-08 11:22:59,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +28: [2023-05-08 11:22:59,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +11: [2023-05-08 11:22:59,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +23: [2023-05-08 11:22:59,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +23: [2023-05-08 11:22:59,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +28: [2023-05-08 11:22:59,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +11: [2023-05-08 11:22:59,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +11: [2023-05-08 11:22:59,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +23: [2023-05-08 11:22:59,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +10: [2023-05-08 11:22:59,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +23: [2023-05-08 11:22:59,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +11: [2023-05-08 11:22:59,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 4: [2023-05-08 11:22:59,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +22: [2023-05-08 11:22:59,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +22: [2023-05-08 11:22:59,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +18: [2023-05-08 11:22:59,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +18: [2023-05-08 11:22:59,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +18: [2023-05-08 11:22:59,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 6: [2023-05-08 11:22:59,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 4: [2023-05-08 11:22:59,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 6: [2023-05-08 11:22:59,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:22:59,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 0: [2023-05-08 11:22:59,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 9: [2023-05-08 11:22:59,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 4: [2023-05-08 11:22:59,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 9: [2023-05-08 11:22:59,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 9: [2023-05-08 11:22:59,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 9: [2023-05-08 11:22:59,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 6: [2023-05-08 11:22:59,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:22:59,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +22: [2023-05-08 11:22:59,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +17: [2023-05-08 11:22:59,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:22:59,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +27: [2023-05-08 11:22:59,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:22:59,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +20: [2023-05-08 11:22:59,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +30: [2023-05-08 11:22:59,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +20: [2023-05-08 11:22:59,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +30: [2023-05-08 11:22:59,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +30: [2023-05-08 11:22:59,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:22:59,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:22:59,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +27: [2023-05-08 11:22:59,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +30: [2023-05-08 11:22:59,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +20: [2023-05-08 11:22:59,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +20: [2023-05-08 11:22:59,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +10: [2023-05-08 11:22:59,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:22:59,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +10: [2023-05-08 11:22:59,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:22:59,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:22:59,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +11: [2023-05-08 11:22:59,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +10: [2023-05-08 11:22:59,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:22:59,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +10: [2023-05-08 11:22:59,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:22:59,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:22:59,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:22:59,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +18: [2023-05-08 11:22:59,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:22:59,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:22:59,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:22:59,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:22:59,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +11: [2023-05-08 11:22:59,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:22:59,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:22:59,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:22:59,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:22:59,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:22:59,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:22:59,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:22:59,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:22:59,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +15: [2023-05-08 11:22:59,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:22:59,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +22: [2023-05-08 11:22:59,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:22:59,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:22:59,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:22:59,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:22:59,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:22:59,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:22:59,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:22:59,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:22:59,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +19: [2023-05-08 11:22:59,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +31: [2023-05-08 11:22:59,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +19: [2023-05-08 11:22:59,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +19: [2023-05-08 11:22:59,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +19: [2023-05-08 11:22:59,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +19: [2023-05-08 11:22:59,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +19: [2023-05-08 11:22:59,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +19: [2023-05-08 11:22:59,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +29: [2023-05-08 11:22:59,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +29: [2023-05-08 11:22:59,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +29: [2023-05-08 11:22:59,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +29: [2023-05-08 11:22:59,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +29: [2023-05-08 11:22:59,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +20: [2023-05-08 11:22:59,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:22:59,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:22:59,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +29: [2023-05-08 11:22:59,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +29: [2023-05-08 11:22:59,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:22:59,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:22:59,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +29: [2023-05-08 11:22:59,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +31: [2023-05-08 11:22:59,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:22:59,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +20: [2023-05-08 11:22:59,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +20: [2023-05-08 11:22:59,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +31: [2023-05-08 11:22:59,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:22:59,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +20: [2023-05-08 11:22:59,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:22:59,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:22:59,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +23: [2023-05-08 11:22:59,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:22:59,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:22:59,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:22:59,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:22:59,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +29: [2023-05-08 11:22:59,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:22:59,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:22:59,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +29: [2023-05-08 11:22:59,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +23: [2023-05-08 11:22:59,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +20: [2023-05-08 11:22:59,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +30: [2023-05-08 11:22:59,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +15: [2023-05-08 11:22:59,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +15: [2023-05-08 11:22:59,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +15: [2023-05-08 11:22:59,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:22:59,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +19: [2023-05-08 11:22:59,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +19: [2023-05-08 11:22:59,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:22:59,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +19: [2023-05-08 11:22:59,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:22:59,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:22:59,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +23: [2023-05-08 11:22:59,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:22:59,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:22:59,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:22:59,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:22:59,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +31: [2023-05-08 11:22:59,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:22:59,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:22:59,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:22:59,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +31: [2023-05-08 11:22:59,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:22:59,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:22:59,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +15: [2023-05-08 11:22:59,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +18: [2023-05-08 11:22:59,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:22:59,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +18: [2023-05-08 11:22:59,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:22:59,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:22:59,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:22:59,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:22:59,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +30: [2023-05-08 11:22:59,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:22:59,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 9: [2023-05-08 11:22:59,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +10: [2023-05-08 11:22:59,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:22:59,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:22:59,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:22:59,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +23: [2023-05-08 11:22:59,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +30: [2023-05-08 11:22:59,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:22:59,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +13: [2023-05-08 11:22:59,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:22:59,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:22:59,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 2: [2023-05-08 11:22:59,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:22:59,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 2: [2023-05-08 11:22:59,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +13: [2023-05-08 11:22:59,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +13: [2023-05-08 11:22:59,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:22:59,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:22:59,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:22:59,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 4: [2023-05-08 11:22:59,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +28: [2023-05-08 11:22:59,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 2: [2023-05-08 11:22:59,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:22:59,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +18: [2023-05-08 11:22:59,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:22:59,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +18: [2023-05-08 11:22:59,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:22:59,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +11: [2023-05-08 11:22:59,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +11: [2023-05-08 11:22:59,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +28: [2023-05-08 11:22:59,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +13: [2023-05-08 11:22:59,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +30: [2023-05-08 11:22:59,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +22: [2023-05-08 11:22:59,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +30: [2023-05-08 11:22:59,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +22: [2023-05-08 11:22:59,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +11: [2023-05-08 11:22:59,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +11: [2023-05-08 11:22:59,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +22: [2023-05-08 11:22:59,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:22:59,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:22:59,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:22:59,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +22: [2023-05-08 11:22:59,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +30: [2023-05-08 11:22:59,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +30: [2023-05-08 11:22:59,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:22:59,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +28: [2023-05-08 11:22:59,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +28: [2023-05-08 11:22:59,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +28: [2023-05-08 11:22:59,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +28: [2023-05-08 11:22:59,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +18: [2023-05-08 11:22:59,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:22:59,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:22:59,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:22:59,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:22:59,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:22:59,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:22:59,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:22:59,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:22:59,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:22:59,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 0: [2023-05-08 11:22:59,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:22:59,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:22:59,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 0: [2023-05-08 11:22:59,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +18: [2023-05-08 11:22:59,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:22:59,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:22:59,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +22: [2023-05-08 11:22:59,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:22:59,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +18: [2023-05-08 11:22:59,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 4: [2023-05-08 11:22:59,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +22: [2023-05-08 11:22:59,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:22:59,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +22: [2023-05-08 11:22:59,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +26: [2023-05-08 11:22:59,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +26: [2023-05-08 11:22:59,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +26: [2023-05-08 11:22:59,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +22: [2023-05-08 11:22:59,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:22:59,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:22:59,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:22:59,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:22:59,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:22:59,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +11: [2023-05-08 11:22:59,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:22:59,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:22:59,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:22:59,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 4: [2023-05-08 11:22:59,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +11: [2023-05-08 11:22:59,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:22:59,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +11: [2023-05-08 11:22:59,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +20: [2023-05-08 11:22:59,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 0: [2023-05-08 11:22:59,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +23: [2023-05-08 11:22:59,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +23: [2023-05-08 11:22:59,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +28: [2023-05-08 11:22:59,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +28: [2023-05-08 11:22:59,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +10: [2023-05-08 11:22:59,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:22:59,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 0: [2023-05-08 11:22:59,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +11: [2023-05-08 11:22:59,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +22: [2023-05-08 11:22:59,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:22:59,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +10: [2023-05-08 11:22:59,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 4: [2023-05-08 11:22:59,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +10: [2023-05-08 11:22:59,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 7: [2023-05-08 11:22:59,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:22:59,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +16: [2023-05-08 11:22:59,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +16: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +16: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +16: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +14: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +24: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +14: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +14: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +14: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +21: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +21: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +14: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +21: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:22:59,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 0: [2023-05-08 11:22:59,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:22:59,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +14: [2023-05-08 11:22:59,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 3: [2023-05-08 11:22:59,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 3: [2023-05-08 11:22:59,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +14: [2023-05-08 11:22:59,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +16: [2023-05-08 11:22:59,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +16: [2023-05-08 11:22:59,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:22:59,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:22:59,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +21: [2023-05-08 11:22:59,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 3: [2023-05-08 11:22:59,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 4: [2023-05-08 11:22:59,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:22:59,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 7: [2023-05-08 11:22:59,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +10: [2023-05-08 11:22:59,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +15: [2023-05-08 11:22:59,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +15: [2023-05-08 11:22:59,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +15: [2023-05-08 11:22:59,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:22:59,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 3: [2023-05-08 11:22:59,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +15: [2023-05-08 11:22:59,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +18: [2023-05-08 11:22:59,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:22:59,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:22:59,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:22:59,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:22:59,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +30: [2023-05-08 11:22:59,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:22:59,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +18: [2023-05-08 11:22:59,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:22:59,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:22:59,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +10: [2023-05-08 11:22:59,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:22:59,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +20: [2023-05-08 11:22:59,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:22:59,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:22:59,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:22:59,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +15: [2023-05-08 11:22:59,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:22:59,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +22: [2023-05-08 11:22:59,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +27: [2023-05-08 11:22:59,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:22:59,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:22:59,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +15: [2023-05-08 11:22:59,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:22:59,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +26: [2023-05-08 11:22:59,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +15: [2023-05-08 11:22:59,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:22:59,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:22:59,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +15: [2023-05-08 11:22:59,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 0: [2023-05-08 11:22:59,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:22:59,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +29: [2023-05-08 11:22:59,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:22:59,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:22:59,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:22:59,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +16: [2023-05-08 11:22:59,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:22:59,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +14: [2023-05-08 11:22:59,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:22:59,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +19: [2023-05-08 11:22:59,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:22:59,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:22:59,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +12: [2023-05-08 11:22:59,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +12: [2023-05-08 11:22:59,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:22:59,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:22:59,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:22:59,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:22:59,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:22:59,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:22:59,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:22:59,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +20: [2023-05-08 11:22:59,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +12: [2023-05-08 11:22:59,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:22:59,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +20: [2023-05-08 11:22:59,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:22:59,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:22:59,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:22:59,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:22:59,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +15: [2023-05-08 11:22:59,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +28: [2023-05-08 11:22:59,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +28: [2023-05-08 11:22:59,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +28: [2023-05-08 11:22:59,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +28: [2023-05-08 11:22:59,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +28: [2023-05-08 11:22:59,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +22: [2023-05-08 11:22:59,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:22:59,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +19: [2023-05-08 11:22:59,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +28: [2023-05-08 11:22:59,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:22:59,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:22:59,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:22:59,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:22:59,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +28: [2023-05-08 11:22:59,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +21: [2023-05-08 11:22:59,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:22:59,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +20: [2023-05-08 11:22:59,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +13: [2023-05-08 11:22:59,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +28: [2023-05-08 11:22:59,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:22:59,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +10: [2023-05-08 11:22:59,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +24: [2023-05-08 11:22:59,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:22:59,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:22:59,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:22:59,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +22: [2023-05-08 11:22:59,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +13: [2023-05-08 11:22:59,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +16: [2023-05-08 11:22:59,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:22:59,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:22:59,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:22:59,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +20: [2023-05-08 11:22:59,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:22:59,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +26: [2023-05-08 11:22:59,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:22:59,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:22:59,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +22: [2023-05-08 11:22:59,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +19: [2023-05-08 11:22:59,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:22:59,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +15: [2023-05-08 11:22:59,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:22:59,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:22:59,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:22:59,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +24: [2023-05-08 11:22:59,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +27: [2023-05-08 11:22:59,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:22:59,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:22:59,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:22:59,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +24: [2023-05-08 11:22:59,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:22:59,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +24: [2023-05-08 11:22:59,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:22:59,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +29: [2023-05-08 11:22:59,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:22:59,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +20: [2023-05-08 11:22:59,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:22:59,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:22:59,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:22:59,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +22: [2023-05-08 11:22:59,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:22:59,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:22:59,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:22:59,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +19: [2023-05-08 11:22:59,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:22:59,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:22:59,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:22:59,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:22:59,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 5: [2023-05-08 11:22:59,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +13: [2023-05-08 11:22:59,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:22:59,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +22: [2023-05-08 11:22:59,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:22:59,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:22:59,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:22:59,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +20: [2023-05-08 11:22:59,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +24: [2023-05-08 11:22:59,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:22:59,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +10: [2023-05-08 11:22:59,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +16: [2023-05-08 11:22:59,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +10: [2023-05-08 11:22:59,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:22:59,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +28: [2023-05-08 11:22:59,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +28: [2023-05-08 11:22:59,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +19: [2023-05-08 11:22:59,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +14: [2023-05-08 11:22:59,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:22:59,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +27: [2023-05-08 11:22:59,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:22:59,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:22:59,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:22:59,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +12: [2023-05-08 11:22:59,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:22:59,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +22: [2023-05-08 11:22:59,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +27: [2023-05-08 11:22:59,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:22:59,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:22:59,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:22:59,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:22:59,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:22:59,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:22:59,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:22:59,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:22:59,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +10: [2023-05-08 11:22:59,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:22:59,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +10: [2023-05-08 11:22:59,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:22:59,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +28: [2023-05-08 11:22:59,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +28: [2023-05-08 11:22:59,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:22:59,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +14: [2023-05-08 11:22:59,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:22:59,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:22:59,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:22:59,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +28: [2023-05-08 11:22:59,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +28: [2023-05-08 11:22:59,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +12: [2023-05-08 11:22:59,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +16: [2023-05-08 11:22:59,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:22:59,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +12: [2023-05-08 11:22:59,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +28: [2023-05-08 11:22:59,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +28: [2023-05-08 11:22:59,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:22:59,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:22:59,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:22:59,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +13: [2023-05-08 11:22:59,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +13: [2023-05-08 11:22:59,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 8: [2023-05-08 11:22:59,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:22:59,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:22:59,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 5: [2023-05-08 11:22:59,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +26: [2023-05-08 11:22:59,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:22:59,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 5: [2023-05-08 11:22:59,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 5: [2023-05-08 11:22:59,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 5: [2023-05-08 11:22:59,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 1: [2023-05-08 11:22:59,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:22:59,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:22:59,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:22:59,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +13: [2023-05-08 11:22:59,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +13: [2023-05-08 11:22:59,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +13: [2023-05-08 11:22:59,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:22:59,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +31: [2023-05-08 11:22:59,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 1: [2023-05-08 11:22:59,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 1: [2023-05-08 11:22:59,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 1: [2023-05-08 11:22:59,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +31: [2023-05-08 11:22:59,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +31: [2023-05-08 11:22:59,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 5: [2023-05-08 11:22:59,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 5: [2023-05-08 11:22:59,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +13: [2023-05-08 11:22:59,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:22:59,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +13: [2023-05-08 11:22:59,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:22:59,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +13: [2023-05-08 11:22:59,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:22:59,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 5: [2023-05-08 11:22:59,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 5: [2023-05-08 11:22:59,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +31: [2023-05-08 11:22:59,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:22:59,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +13: [2023-05-08 11:22:59,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:22:59,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:22:59,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:22:59,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +31: [2023-05-08 11:22:59,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:22:59,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:22:59,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +31: [2023-05-08 11:22:59,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +31: [2023-05-08 11:22:59,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:22:59,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 8: [2023-05-08 11:22:59,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +31: [2023-05-08 11:22:59,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:22:59,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +20: [2023-05-08 11:22:59,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +20: [2023-05-08 11:22:59,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +20: [2023-05-08 11:22:59,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +20: [2023-05-08 11:22:59,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 8: [2023-05-08 11:22:59,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 7: [2023-05-08 11:22:59,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +31: [2023-05-08 11:22:59,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +31: [2023-05-08 11:22:59,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:22:59,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +31: [2023-05-08 11:22:59,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +31: [2023-05-08 11:22:59,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +31: [2023-05-08 11:22:59,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:22:59,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:22:59,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:22:59,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +23: [2023-05-08 11:22:59,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +23: [2023-05-08 11:22:59,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +23: [2023-05-08 11:22:59,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 8: [2023-05-08 11:22:59,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +25: [2023-05-08 11:22:59,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:22:59,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:22:59,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:22:59,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:22:59,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:22:59,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +20: [2023-05-08 11:22:59,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:22:59,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:22:59,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +25: [2023-05-08 11:22:59,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:22:59,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:22:59,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:22:59,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:22:59,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:22:59,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:22:59,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:22:59,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +27: [2023-05-08 11:22:59,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +27: [2023-05-08 11:22:59,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +20: [2023-05-08 11:22:59,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +27: [2023-05-08 11:22:59,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +25: [2023-05-08 11:22:59,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +28: [2023-05-08 11:22:59,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +11: [2023-05-08 11:22:59,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +11: [2023-05-08 11:22:59,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +11: [2023-05-08 11:22:59,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +28: [2023-05-08 11:22:59,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +28: [2023-05-08 11:22:59,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +11: [2023-05-08 11:22:59,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +28: [2023-05-08 11:22:59,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +31: [2023-05-08 11:22:59,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:22:59,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +20: [2023-05-08 11:22:59,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +20: [2023-05-08 11:22:59,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +31: [2023-05-08 11:22:59,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:22:59,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +20: [2023-05-08 11:22:59,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +31: [2023-05-08 11:22:59,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +31: [2023-05-08 11:22:59,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +27: [2023-05-08 11:22:59,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +25: [2023-05-08 11:22:59,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +27: [2023-05-08 11:22:59,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:22:59,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:22:59,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +19: [2023-05-08 11:22:59,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +23: [2023-05-08 11:22:59,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:22:59,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +27: [2023-05-08 11:22:59,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:22:59,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:22:59,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +27: [2023-05-08 11:22:59,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +25: [2023-05-08 11:22:59,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:22:59,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:22:59,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +28: [2023-05-08 11:22:59,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:22:59,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +28: [2023-05-08 11:22:59,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:22:59,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:22:59,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +11: [2023-05-08 11:22:59,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:22:59,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:22:59,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +29: [2023-05-08 11:22:59,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:22:59,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:22:59,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:22:59,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:22:59,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:22:59,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:22:59,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:22:59,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 6: [2023-05-08 11:22:59,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:22:59,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:22:59,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +18: [2023-05-08 11:22:59,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:22:59,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +18: [2023-05-08 11:22:59,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +18: [2023-05-08 11:22:59,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:22:59,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:22:59,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +18: [2023-05-08 11:22:59,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +27: [2023-05-08 11:22:59,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:22:59,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:22:59,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:22:59,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:22:59,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:22:59,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +27: [2023-05-08 11:22:59,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +23: [2023-05-08 11:22:59,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:22:59,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +31: [2023-05-08 11:22:59,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:22:59,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +27: [2023-05-08 11:22:59,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +27: [2023-05-08 11:22:59,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +29: [2023-05-08 11:22:59,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +18: [2023-05-08 11:22:59,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:22:59,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:22:59,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:22:59,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +29: [2023-05-08 11:22:59,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:22:59,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:22:59,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:22:59,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +27: [2023-05-08 11:22:59,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +27: [2023-05-08 11:22:59,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +27: [2023-05-08 11:22:59,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +27: [2023-05-08 11:22:59,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +19: [2023-05-08 11:22:59,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +10: [2023-05-08 11:22:59,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +11: [2023-05-08 11:22:59,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:22:59,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +11: [2023-05-08 11:22:59,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 9: [2023-05-08 11:22:59,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +10: [2023-05-08 11:22:59,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +29: [2023-05-08 11:22:59,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:22:59,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +10: [2023-05-08 11:22:59,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 6: [2023-05-08 11:22:59,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 9: [2023-05-08 11:22:59,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 4: [2023-05-08 11:22:59,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 4: [2023-05-08 11:22:59,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 4: [2023-05-08 11:22:59,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +28: [2023-05-08 11:22:59,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +29: [2023-05-08 11:22:59,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:22:59,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +15: [2023-05-08 11:22:59,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +28: [2023-05-08 11:22:59,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +29: [2023-05-08 11:22:59,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:22:59,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:22:59,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +15: [2023-05-08 11:22:59,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +25: [2023-05-08 11:22:59,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +18: [2023-05-08 11:22:59,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +18: [2023-05-08 11:22:59,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:22:59,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:22:59,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:22:59,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +28: [2023-05-08 11:22:59,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +11: [2023-05-08 11:22:59,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +11: [2023-05-08 11:22:59,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +30: [2023-05-08 11:22:59,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +30: [2023-05-08 11:22:59,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +30: [2023-05-08 11:22:59,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +30: [2023-05-08 11:22:59,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +11: [2023-05-08 11:22:59,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:22:59,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:22:59,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +10: [2023-05-08 11:22:59,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:22:59,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +30: [2023-05-08 11:22:59,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:22:59,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:22:59,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:22:59,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:22:59,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:22:59,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +29: [2023-05-08 11:22:59,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:22:59,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:22:59,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +10: [2023-05-08 11:22:59,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +10: [2023-05-08 11:22:59,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:22:59,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:22:59,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:22:59,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:22:59,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:22:59,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +30: [2023-05-08 11:22:59,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:22:59,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +11: [2023-05-08 11:22:59,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 7: [2023-05-08 11:22:59,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:22:59,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:22:59,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +30: [2023-05-08 11:22:59,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +30: [2023-05-08 11:22:59,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:22:59,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +23: [2023-05-08 11:22:59,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +23: [2023-05-08 11:22:59,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:22:59,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +15: [2023-05-08 11:22:59,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:22:59,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:22:59,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +15: [2023-05-08 11:22:59,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:22:59,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:22:59,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +23: [2023-05-08 11:22:59,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:22:59,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:22:59,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +18: [2023-05-08 11:22:59,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:22:59,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:22:59,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:22:59,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:22:59,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +18: [2023-05-08 11:22:59,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +30: [2023-05-08 11:22:59,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +30: [2023-05-08 11:22:59,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:22:59,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +29: [2023-05-08 11:22:59,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +29: [2023-05-08 11:22:59,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:22:59,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 9: [2023-05-08 11:22:59,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +11: [2023-05-08 11:22:59,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 9: [2023-05-08 11:22:59,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +23: [2023-05-08 11:22:59,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +29: [2023-05-08 11:22:59,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 0: [2023-05-08 11:22:59,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 4: [2023-05-08 11:22:59,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 6: [2023-05-08 11:22:59,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +10: [2023-05-08 11:22:59,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:22:59,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +26: [2023-05-08 11:22:59,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 9: [2023-05-08 11:22:59,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +18: [2023-05-08 11:22:59,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +26: [2023-05-08 11:22:59,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +10: [2023-05-08 11:22:59,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +26: [2023-05-08 11:22:59,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +10: [2023-05-08 11:22:59,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:22:59,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:22:59,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +10: [2023-05-08 11:22:59,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 7: [2023-05-08 11:22:59,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +22: [2023-05-08 11:22:59,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +22: [2023-05-08 11:22:59,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +15: [2023-05-08 11:22:59,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:22:59,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:22:59,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +22: [2023-05-08 11:22:59,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 0: [2023-05-08 11:22:59,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +22: [2023-05-08 11:22:59,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +23: [2023-05-08 11:22:59,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +15: [2023-05-08 11:22:59,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:22:59,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +15: [2023-05-08 11:22:59,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +15: [2023-05-08 11:22:59,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 4: [2023-05-08 11:22:59,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:22:59,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:22:59,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 4: [2023-05-08 11:22:59,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 4: [2023-05-08 11:22:59,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +15: [2023-05-08 11:22:59,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +17: [2023-05-08 11:22:59,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +10: [2023-05-08 11:22:59,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:22:59,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +29: [2023-05-08 11:22:59,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +10: [2023-05-08 11:22:59,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +29: [2023-05-08 11:22:59,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +17: [2023-05-08 11:22:59,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +17: [2023-05-08 11:22:59,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +18: [2023-05-08 11:22:59,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +18: [2023-05-08 11:22:59,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +18: [2023-05-08 11:22:59,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +18: [2023-05-08 11:22:59,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +29: [2023-05-08 11:22:59,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:22:59,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:22:59,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:22:59,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +29: [2023-05-08 11:22:59,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +23: [2023-05-08 11:22:59,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +29: [2023-05-08 11:22:59,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:22:59,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:22:59,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:22:59,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:22:59,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:22:59,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:22:59,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:22:59,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:22:59,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:22:59,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:22:59,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:22:59,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +15: [2023-05-08 11:22:59,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:22:59,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:22:59,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +15: [2023-05-08 11:22:59,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:22:59,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:22:59,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:22:59,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +22: [2023-05-08 11:22:59,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +22: [2023-05-08 11:22:59,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +22: [2023-05-08 11:22:59,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:22:59,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:22:59,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:22:59,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:22:59,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:22:59,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +22: [2023-05-08 11:22:59,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +22: [2023-05-08 11:22:59,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:22:59,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +22: [2023-05-08 11:22:59,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 3: [2023-05-08 11:22:59,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +26: [2023-05-08 11:22:59,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +18: [2023-05-08 11:22:59,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +15: [2023-05-08 11:22:59,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:22:59,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +18: [2023-05-08 11:22:59,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:22:59,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +18: [2023-05-08 11:22:59,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:22:59,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:22:59,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:22:59,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:22:59,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:22:59,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:22:59,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +23: [2023-05-08 11:22:59,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:22:59,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:22:59,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:22:59,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +15: [2023-05-08 11:22:59,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +22: [2023-05-08 11:22:59,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +17: [2023-05-08 11:22:59,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:22:59,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:22:59,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:22:59,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:22:59,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:22:59,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:22:59,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:22:59,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +29: [2023-05-08 11:22:59,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:22:59,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:22:59,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +30: [2023-05-08 11:22:59,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:22:59,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:22:59,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:22:59,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:22:59,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +24: [2023-05-08 11:22:59,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +24: [2023-05-08 11:22:59,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +24: [2023-05-08 11:22:59,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +24: [2023-05-08 11:22:59,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:22:59,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:22:59,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +21: [2023-05-08 11:22:59,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:22:59,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +15: [2023-05-08 11:22:59,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +24: [2023-05-08 11:22:59,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +26: [2023-05-08 11:22:59,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +24: [2023-05-08 11:22:59,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +24: [2023-05-08 11:22:59,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +24: [2023-05-08 11:22:59,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +22: [2023-05-08 11:22:59,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:22:59,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:22:59,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +26: [2023-05-08 11:22:59,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:22:59,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +22: [2023-05-08 11:22:59,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +18: [2023-05-08 11:22:59,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:22:59,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:22:59,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +18: [2023-05-08 11:22:59,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 0: [2023-05-08 11:22:59,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:22:59,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 0: [2023-05-08 11:22:59,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +26: [2023-05-08 11:22:59,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +18: [2023-05-08 11:22:59,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:22:59,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +18: [2023-05-08 11:22:59,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:22:59,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +18: [2023-05-08 11:22:59,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +18: [2023-05-08 11:22:59,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +18: [2023-05-08 11:22:59,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 0: [2023-05-08 11:22:59,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +22: [2023-05-08 11:22:59,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +22: [2023-05-08 11:22:59,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +21: [2023-05-08 11:22:59,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:22:59,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:22:59,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:22:59,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:22:59,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +15: [2023-05-08 11:22:59,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +24: [2023-05-08 11:22:59,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +16: [2023-05-08 11:22:59,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +16: [2023-05-08 11:22:59,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +16: [2023-05-08 11:22:59,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +16: [2023-05-08 11:22:59,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:22:59,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:22:59,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:22:59,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:22:59,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +13: [2023-05-08 11:22:59,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +13: [2023-05-08 11:22:59,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +13: [2023-05-08 11:22:59,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +16: [2023-05-08 11:22:59,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +22: [2023-05-08 11:22:59,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +22: [2023-05-08 11:22:59,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +22: [2023-05-08 11:22:59,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:22:59,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +12: [2023-05-08 11:22:59,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +22: [2023-05-08 11:22:59,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +12: [2023-05-08 11:22:59,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +16: [2023-05-08 11:22:59,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +30: [2023-05-08 11:22:59,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +25: [2023-05-08 11:22:59,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +25: [2023-05-08 11:22:59,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 5: [2023-05-08 11:22:59,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +25: [2023-05-08 11:22:59,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +12: [2023-05-08 11:22:59,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +24: [2023-05-08 11:22:59,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +24: [2023-05-08 11:22:59,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +24: [2023-05-08 11:22:59,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +25: [2023-05-08 11:22:59,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +24: [2023-05-08 11:22:59,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +24: [2023-05-08 11:22:59,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +22: [2023-05-08 11:22:59,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +22: [2023-05-08 11:22:59,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +24: [2023-05-08 11:22:59,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +14: [2023-05-08 11:22:59,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +14: [2023-05-08 11:22:59,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +14: [2023-05-08 11:22:59,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +14: [2023-05-08 11:22:59,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 2: [2023-05-08 11:22:59,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 2: [2023-05-08 11:22:59,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 2: [2023-05-08 11:22:59,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 6: [2023-05-08 11:22:59,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 6: [2023-05-08 11:22:59,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 6: [2023-05-08 11:22:59,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +16: [2023-05-08 11:22:59,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +16: [2023-05-08 11:22:59,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +16: [2023-05-08 11:22:59,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 2: [2023-05-08 11:22:59,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. + 6: [2023-05-08 11:22:59,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +16: [2023-05-08 11:22:59,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +17: [2023-05-08 11:22:59,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +17: [2023-05-08 11:22:59,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +17: [2023-05-08 11:22:59,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +17: [2023-05-08 11:22:59,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +17: [2023-05-08 11:22:59,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +17: [2023-05-08 11:22:59,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +17: [2023-05-08 11:22:59,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +17: [2023-05-08 11:22:59,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:22:59,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:22:59,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:22:59,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +17: [2023-05-08 11:22:59,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +17: [2023-05-08 11:22:59,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +17: [2023-05-08 11:22:59,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +17: [2023-05-08 11:22:59,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:22:59,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +24: [2023-05-08 11:22:59,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +24: [2023-05-08 11:22:59,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:22:59,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:22:59,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:22:59,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:22:59,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:22:59,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +22: [2023-05-08 11:22:59,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +22: [2023-05-08 11:22:59,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +13: [2023-05-08 11:22:59,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +25: [2023-05-08 11:22:59,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +25: [2023-05-08 11:22:59,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:22:59,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +25: [2023-05-08 11:22:59,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +25: [2023-05-08 11:22:59,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:22:59,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +14: [2023-05-08 11:22:59,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:22:59,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +14: [2023-05-08 11:22:59,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +14: [2023-05-08 11:22:59,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:22:59,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +24: [2023-05-08 11:22:59,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:22:59,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:22:59,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +16: [2023-05-08 11:22:59,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +16: [2023-05-08 11:22:59,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +16: [2023-05-08 11:22:59,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +14: [2023-05-08 11:22:59,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +24: [2023-05-08 11:22:59,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:22:59,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 1: [2023-05-08 11:22:59,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 1: [2023-05-08 11:22:59,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +22: [2023-05-08 11:22:59,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +24: [2023-05-08 11:22:59,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:22:59,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:22:59,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:22:59,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +16: [2023-05-08 11:22:59,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:22:59,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:22:59,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +17: [2023-05-08 11:22:59,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:22:59,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:22:59,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +17: [2023-05-08 11:22:59,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +12: [2023-05-08 11:22:59,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +16: [2023-05-08 11:22:59,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +16: [2023-05-08 11:22:59,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:22:59,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +13: [2023-05-08 11:22:59,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:22:59,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:22:59,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +13: [2023-05-08 11:22:59,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 2: [2023-05-08 11:22:59,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +24: [2023-05-08 11:22:59,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:22:59,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:22:59,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +12: [2023-05-08 11:22:59,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +24: [2023-05-08 11:22:59,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +25: [2023-05-08 11:22:59,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +25: [2023-05-08 11:22:59,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +17: [2023-05-08 11:22:59,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:22:59,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +24: [2023-05-08 11:22:59,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:22:59,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +25: [2023-05-08 11:22:59,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +17: [2023-05-08 11:22:59,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +24: [2023-05-08 11:22:59,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +16: [2023-05-08 11:22:59,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:22:59,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:22:59,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:22:59,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:22:59,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +13: [2023-05-08 11:22:59,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +13: [2023-05-08 11:22:59,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:22:59,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +17: [2023-05-08 11:22:59,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:22:59,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:22:59,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:22:59,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +14: [2023-05-08 11:22:59,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:22:59,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +16: [2023-05-08 11:22:59,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:22:59,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:22:59,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:22:59,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +13: [2023-05-08 11:22:59,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +24: [2023-05-08 11:22:59,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +24: [2023-05-08 11:22:59,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +16: [2023-05-08 11:22:59,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +24: [2023-05-08 11:22:59,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:22:59,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 6: [2023-05-08 11:22:59,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +16: [2023-05-08 11:22:59,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:22:59,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +24: [2023-05-08 11:22:59,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 2: [2023-05-08 11:22:59,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +16: [2023-05-08 11:22:59,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +16: [2023-05-08 11:22:59,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 2: [2023-05-08 11:22:59,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +14: [2023-05-08 11:22:59,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:22:59,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 5: [2023-05-08 11:22:59,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +14: [2023-05-08 11:22:59,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:22:59,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:22:59,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 2: [2023-05-08 11:22:59,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:22:59,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:22:59,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 2: [2023-05-08 11:22:59,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 6: [2023-05-08 11:22:59,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +21: [2023-05-08 11:22:59,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +21: [2023-05-08 11:22:59,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +21: [2023-05-08 11:22:59,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +21: [2023-05-08 11:22:59,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt. +16: [2023-05-08 11:22:59,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:22:59,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +16: [2023-05-08 11:22:59,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +13: [2023-05-08 11:22:59,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:22:59,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:22:59,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +13: [2023-05-08 11:22:59,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +16: [2023-05-08 11:22:59,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:22:59,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:22:59,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +16: [2023-05-08 11:22:59,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... +16: [2023-05-08 11:22:59,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:22:59,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:22:59,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:22:59,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +21: [2023-05-08 11:22:59,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:22:59,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +21: [2023-05-08 11:22:59,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:22:59,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:22:59,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:22:59,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +21: [2023-05-08 11:22:59,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:22:59,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +31: [2023-05-08 11:22:59,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +31: [2023-05-08 11:22:59,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +31: [2023-05-08 11:22:59,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +31: [2023-05-08 11:22:59,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +31: [2023-05-08 11:22:59,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +31: [2023-05-08 11:22:59,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:22:59,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:22:59,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:22:59,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:22:59,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +31: [2023-05-08 11:22:59,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:22:59,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:22:59,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 8: [2023-05-08 11:22:59,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 8: [2023-05-08 11:22:59,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 8: [2023-05-08 11:22:59,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +13: [2023-05-08 11:22:59,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:22:59,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:22:59,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +21: [2023-05-08 11:22:59,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +21: [2023-05-08 11:22:59,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 2: [2023-05-08 11:22:59,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:22:59,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 2: [2023-05-08 11:22:59,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:22:59,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +21: [2023-05-08 11:22:59,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:22:59,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +21: [2023-05-08 11:22:59,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt. +21: [2023-05-08 11:22:59,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:22:59,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:22:59,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:22:59,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:22:59,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +31: [2023-05-08 11:22:59,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 2: [2023-05-08 11:22:59,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +31: [2023-05-08 11:22:59,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:22:59,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:22:59,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:22:59,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:22:59,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 2: [2023-05-08 11:22:59,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 2: [2023-05-08 11:22:59,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:22:59,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +11: [2023-05-08 11:22:59,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +11: [2023-05-08 11:22:59,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +11: [2023-05-08 11:22:59,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +31: [2023-05-08 11:22:59,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +31: [2023-05-08 11:22:59,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +10: [2023-05-08 11:22:59,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +10: [2023-05-08 11:22:59,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +10: [2023-05-08 11:22:59,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +10: [2023-05-08 11:22:59,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +10: [2023-05-08 11:22:59,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +10: [2023-05-08 11:22:59,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +10: [2023-05-08 11:22:59,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +10: [2023-05-08 11:22:59,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +11: [2023-05-08 11:22:59,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:22:59,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 3: [2023-05-08 11:22:59,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:22:59,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 3: [2023-05-08 11:22:59,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 3: [2023-05-08 11:22:59,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 3: [2023-05-08 11:22:59,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:22:59,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:22:59,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +20: [2023-05-08 11:22:59,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:22:59,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +20: [2023-05-08 11:22:59,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:22:59,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:22:59,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +20: [2023-05-08 11:22:59,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +11: [2023-05-08 11:22:59,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +31: [2023-05-08 11:22:59,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +31: [2023-05-08 11:22:59,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:22:59,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:22:59,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:22:59,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:22:59,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:22:59,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:22:59,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +28: [2023-05-08 11:22:59,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +28: [2023-05-08 11:22:59,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +28: [2023-05-08 11:22:59,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +28: [2023-05-08 11:22:59,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:22:59,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:22:59,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 3: [2023-05-08 11:22:59,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +10: [2023-05-08 11:22:59,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +10: [2023-05-08 11:22:59,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:22:59,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:22:59,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:22:59,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:22:59,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:22:59,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:22:59,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +19: [2023-05-08 11:22:59,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:22:59,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:22:59,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +14: [2023-05-08 11:22:59,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:22:59,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:22:59,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:22:59,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +14: [2023-05-08 11:22:59,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:22:59,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:22:59,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:22:59,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:22:59,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:22:59,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +19: [2023-05-08 11:22:59,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +19: [2023-05-08 11:22:59,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +19: [2023-05-08 11:22:59,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +10: [2023-05-08 11:22:59,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +19: [2023-05-08 11:22:59,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +19: [2023-05-08 11:22:59,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +26: [2023-05-08 11:22:59,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +26: [2023-05-08 11:22:59,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +19: [2023-05-08 11:22:59,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +26: [2023-05-08 11:22:59,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:22:59,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:22:59,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:22:59,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +19: [2023-05-08 11:22:59,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:22:59,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +14: [2023-05-08 11:22:59,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:22:59,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +19: [2023-05-08 11:22:59,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +26: [2023-05-08 11:22:59,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:22:59,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +26: [2023-05-08 11:22:59,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:22:59,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +26: [2023-05-08 11:22:59,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:22:59,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +23: [2023-05-08 11:22:59,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +23: [2023-05-08 11:22:59,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +23: [2023-05-08 11:22:59,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +23: [2023-05-08 11:22:59,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +20: [2023-05-08 11:22:59,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:22:59,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +29: [2023-05-08 11:22:59,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +29: [2023-05-08 11:22:59,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:22:59,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:22:59,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 3: [2023-05-08 11:22:59,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +29: [2023-05-08 11:22:59,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +29: [2023-05-08 11:22:59,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +10: [2023-05-08 11:22:59,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:22:59,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +11: [2023-05-08 11:22:59,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +20: [2023-05-08 11:22:59,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:22:59,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +29: [2023-05-08 11:22:59,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +10: [2023-05-08 11:22:59,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +27: [2023-05-08 11:22:59,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +11: [2023-05-08 11:22:59,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +27: [2023-05-08 11:22:59,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +10: [2023-05-08 11:22:59,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:22:59,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +27: [2023-05-08 11:22:59,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +10: [2023-05-08 11:22:59,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +27: [2023-05-08 11:22:59,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +29: [2023-05-08 11:22:59,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +29: [2023-05-08 11:22:59,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:22:59,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:22:59,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:22:59,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:22:59,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:22:59,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:22:59,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:22:59,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +14: [2023-05-08 11:22:59,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:22:59,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +20: [2023-05-08 11:22:59,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +14: [2023-05-08 11:22:59,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:22:59,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +20: [2023-05-08 11:22:59,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +20: [2023-05-08 11:22:59,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:22:59,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +10: [2023-05-08 11:22:59,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:22:59,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:22:59,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 2: [2023-05-08 11:22:59,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +27: [2023-05-08 11:22:59,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:22:59,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:22:59,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +27: [2023-05-08 11:22:59,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:22:59,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 4: [2023-05-08 11:22:59,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 9: [2023-05-08 11:22:59,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +19: [2023-05-08 11:22:59,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +19: [2023-05-08 11:22:59,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:22:59,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 2: [2023-05-08 11:22:59,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 2: [2023-05-08 11:22:59,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 5: [2023-05-08 11:22:59,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +29: [2023-05-08 11:22:59,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +27: [2023-05-08 11:22:59,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +12: [2023-05-08 11:22:59,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:22:59,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 5: [2023-05-08 11:22:59,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +14: [2023-05-08 11:22:59,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:22:59,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +12: [2023-05-08 11:22:59,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +20: [2023-05-08 11:22:59,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:22:59,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:22:59,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:22:59,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +10: [2023-05-08 11:22:59,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +10: [2023-05-08 11:22:59,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +10: [2023-05-08 11:22:59,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 0: [2023-05-08 11:22:59,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:22:59,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:22:59,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +30: [2023-05-08 11:22:59,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +30: [2023-05-08 11:22:59,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +26: [2023-05-08 11:22:59,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +26: [2023-05-08 11:22:59,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +26: [2023-05-08 11:22:59,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:22:59,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:22:59,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +25: [2023-05-08 11:22:59,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 4: [2023-05-08 11:22:59,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:22:59,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +25: [2023-05-08 11:22:59,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +26: [2023-05-08 11:22:59,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +25: [2023-05-08 11:22:59,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +23: [2023-05-08 11:22:59,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +26: [2023-05-08 11:22:59,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +26: [2023-05-08 11:22:59,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +26: [2023-05-08 11:22:59,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +23: [2023-05-08 11:22:59,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 9: [2023-05-08 11:22:59,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 9: [2023-05-08 11:22:59,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 9: [2023-05-08 11:22:59,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +23: [2023-05-08 11:22:59,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +12: [2023-05-08 11:22:59,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:22:59,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:22:59,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:22:59,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 2: [2023-05-08 11:22:59,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:22:59,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +18: [2023-05-08 11:22:59,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +18: [2023-05-08 11:22:59,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 4: [2023-05-08 11:22:59,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +29: [2023-05-08 11:22:59,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 4: [2023-05-08 11:22:59,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 4: [2023-05-08 11:22:59,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +19: [2023-05-08 11:22:59,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +29: [2023-05-08 11:22:59,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +18: [2023-05-08 11:22:59,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 0: [2023-05-08 11:22:59,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 5: [2023-05-08 11:22:59,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 0: [2023-05-08 11:22:59,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +15: [2023-05-08 11:22:59,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +15: [2023-05-08 11:22:59,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 5: [2023-05-08 11:22:59,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:22:59,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:22:59,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +23: [2023-05-08 11:22:59,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +23: [2023-05-08 11:22:59,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +10: [2023-05-08 11:22:59,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +10: [2023-05-08 11:22:59,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:22:59,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +19: [2023-05-08 11:22:59,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:22:59,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:22:59,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +23: [2023-05-08 11:22:59,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +30: [2023-05-08 11:22:59,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:22:59,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:22:59,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +29: [2023-05-08 11:22:59,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 1: [2023-05-08 11:22:59,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:22:59,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +25: [2023-05-08 11:22:59,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:22:59,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:22:59,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +28: [2023-05-08 11:22:59,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:22:59,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:22:59,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:22:59,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:22:59,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +25: [2023-05-08 11:22:59,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:22:59,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +28: [2023-05-08 11:22:59,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:22:59,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +28: [2023-05-08 11:22:59,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +28: [2023-05-08 11:22:59,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +28: [2023-05-08 11:22:59,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +25: [2023-05-08 11:22:59,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:22:59,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +28: [2023-05-08 11:22:59,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +28: [2023-05-08 11:22:59,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +28: [2023-05-08 11:22:59,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +28: [2023-05-08 11:22:59,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +28: [2023-05-08 11:22:59,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:22:59,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:22:59,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 6: [2023-05-08 11:22:59,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +27: [2023-05-08 11:22:59,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +27: [2023-05-08 11:22:59,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +27: [2023-05-08 11:22:59,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +27: [2023-05-08 11:22:59,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +10: [2023-05-08 11:22:59,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +30: [2023-05-08 11:22:59,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +26: [2023-05-08 11:22:59,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +29: [2023-05-08 11:22:59,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +18: [2023-05-08 11:22:59,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:22:59,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 9: [2023-05-08 11:22:59,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +30: [2023-05-08 11:22:59,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:22:59,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +18: [2023-05-08 11:22:59,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +26: [2023-05-08 11:22:59,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +27: [2023-05-08 11:22:59,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +27: [2023-05-08 11:22:59,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +20: [2023-05-08 11:22:59,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:22:59,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:22:59,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +20: [2023-05-08 11:22:59,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +20: [2023-05-08 11:22:59,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +20: [2023-05-08 11:22:59,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +20: [2023-05-08 11:22:59,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +26: [2023-05-08 11:22:59,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +26: [2023-05-08 11:22:59,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:22:59,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:22:59,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +12: [2023-05-08 11:22:59,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +27: [2023-05-08 11:22:59,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:22:59,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +12: [2023-05-08 11:22:59,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +30: [2023-05-08 11:22:59,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:22:59,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:22:59,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:22:59,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:22:59,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:22:59,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 4: [2023-05-08 11:22:59,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +30: [2023-05-08 11:22:59,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:22:59,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:22:59,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:22:59,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +18: [2023-05-08 11:22:59,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:22:59,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:22:59,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:22:59,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:22:59,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +26: [2023-05-08 11:22:59,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:22:59,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +16: [2023-05-08 11:22:59,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:22:59,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:22:59,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:22:59,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +16: [2023-05-08 11:22:59,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:22:59,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +16: [2023-05-08 11:22:59,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:22:59,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 6: [2023-05-08 11:22:59,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:22:59,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:22:59,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +29: [2023-05-08 11:22:59,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:22:59,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +10: [2023-05-08 11:22:59,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +15: [2023-05-08 11:22:59,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:22:59,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:22:59,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +29: [2023-05-08 11:22:59,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +29: [2023-05-08 11:22:59,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +27: [2023-05-08 11:22:59,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:22:59,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:22:59,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +29: [2023-05-08 11:22:59,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +12: [2023-05-08 11:22:59,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 2: [2023-05-08 11:22:59,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 2: [2023-05-08 11:22:59,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +12: [2023-05-08 11:22:59,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:22:59,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +12: [2023-05-08 11:22:59,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +12: [2023-05-08 11:22:59,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +25: [2023-05-08 11:22:59,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +25: [2023-05-08 11:22:59,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +25: [2023-05-08 11:22:59,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +25: [2023-05-08 11:22:59,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:22:59,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +30: [2023-05-08 11:22:59,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 5: [2023-05-08 11:22:59,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 5: [2023-05-08 11:22:59,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 9: [2023-05-08 11:22:59,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +27: [2023-05-08 11:22:59,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:22:59,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +25: [2023-05-08 11:22:59,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:22:59,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 9: [2023-05-08 11:22:59,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +10: [2023-05-08 11:22:59,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:22:59,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:22:59,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:22:59,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:22:59,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +18: [2023-05-08 11:22:59,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +10: [2023-05-08 11:22:59,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +25: [2023-05-08 11:22:59,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:22:59,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +25: [2023-05-08 11:22:59,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:22:59,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +25: [2023-05-08 11:22:59,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:22:59,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +25: [2023-05-08 11:22:59,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +25: [2023-05-08 11:22:59,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 2: [2023-05-08 11:22:59,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:22:59,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:22:59,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +14: [2023-05-08 11:22:59,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +27: [2023-05-08 11:22:59,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +26: [2023-05-08 11:22:59,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:22:59,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +26: [2023-05-08 11:22:59,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 2: [2023-05-08 11:22:59,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 7: [2023-05-08 11:22:59,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:22:59,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +21: [2023-05-08 11:22:59,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 4: [2023-05-08 11:22:59,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:22:59,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 5: [2023-05-08 11:22:59,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:22:59,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 5: [2023-05-08 11:22:59,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +30: [2023-05-08 11:22:59,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:22:59,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 7: [2023-05-08 11:22:59,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 9: [2023-05-08 11:22:59,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:22:59,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 9: [2023-05-08 11:22:59,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +13: [2023-05-08 11:22:59,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +13: [2023-05-08 11:22:59,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +13: [2023-05-08 11:22:59,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 1: [2023-05-08 11:22:59,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +16: [2023-05-08 11:22:59,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:22:59,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:22:59,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 5: [2023-05-08 11:22:59,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +14: [2023-05-08 11:22:59,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +15: [2023-05-08 11:22:59,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +15: [2023-05-08 11:22:59,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +30: [2023-05-08 11:22:59,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +12: [2023-05-08 11:22:59,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:22:59,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +14: [2023-05-08 11:22:59,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +12: [2023-05-08 11:22:59,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 7: [2023-05-08 11:22:59,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +24: [2023-05-08 11:22:59,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +30: [2023-05-08 11:22:59,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 1: [2023-05-08 11:22:59,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +30: [2023-05-08 11:22:59,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:22:59,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +12: [2023-05-08 11:22:59,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +12: [2023-05-08 11:22:59,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 6: [2023-05-08 11:22:59,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:22:59,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:22:59,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +16: [2023-05-08 11:22:59,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +27: [2023-05-08 11:22:59,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:22:59,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +24: [2023-05-08 11:22:59,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 9: [2023-05-08 11:22:59,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +15: [2023-05-08 11:22:59,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:22:59,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +15: [2023-05-08 11:22:59,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +18: [2023-05-08 11:22:59,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +12: [2023-05-08 11:22:59,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +12: [2023-05-08 11:22:59,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:22:59,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 0: [2023-05-08 11:22:59,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 0: [2023-05-08 11:22:59,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 5: [2023-05-08 11:22:59,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:22:59,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +12: [2023-05-08 11:22:59,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +14: [2023-05-08 11:22:59,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +24: [2023-05-08 11:22:59,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +27: [2023-05-08 11:22:59,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +16: [2023-05-08 11:22:59,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:22:59,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 6: [2023-05-08 11:22:59,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +22: [2023-05-08 11:22:59,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +22: [2023-05-08 11:22:59,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 1: [2023-05-08 11:22:59,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:22:59,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +27: [2023-05-08 11:22:59,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +10: [2023-05-08 11:22:59,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:22:59,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:22:59,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:22:59,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +16: [2023-05-08 11:22:59,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:22:59,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +21: [2023-05-08 11:22:59,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +21: [2023-05-08 11:22:59,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:22:59,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. +26: [2023-05-08 11:22:59,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +31: [2023-05-08 11:22:59,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +31: [2023-05-08 11:22:59,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +21: [2023-05-08 11:22:59,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +21: [2023-05-08 11:22:59,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +10: [2023-05-08 11:22:59,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +13: [2023-05-08 11:22:59,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:22:59,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +31: [2023-05-08 11:22:59,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 6: [2023-05-08 11:22:59,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:22:59,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 6: [2023-05-08 11:22:59,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 6: [2023-05-08 11:22:59,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... + 7: [2023-05-08 11:22:59,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +13: [2023-05-08 11:22:59,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +13: [2023-05-08 11:22:59,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:22:59,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +29: [2023-05-08 11:22:59,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +29: [2023-05-08 11:22:59,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +14: [2023-05-08 11:22:59,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:22:59,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:22:59,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:22:59,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +29: [2023-05-08 11:22:59,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +29: [2023-05-08 11:22:59,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +29: [2023-05-08 11:22:59,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +15: [2023-05-08 11:22:59,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 4: [2023-05-08 11:22:59,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +18: [2023-05-08 11:22:59,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +16: [2023-05-08 11:22:59,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:22:59,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt. + 1: [2023-05-08 11:22:59,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +25: [2023-05-08 11:22:59,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +25: [2023-05-08 11:22:59,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +14: [2023-05-08 11:22:59,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +25: [2023-05-08 11:22:59,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +15: [2023-05-08 11:22:59,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 5: [2023-05-08 11:22:59,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:22:59,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:22:59,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:22:59,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +24: [2023-05-08 11:22:59,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +24: [2023-05-08 11:22:59,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +24: [2023-05-08 11:22:59,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:22:59,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:22:59,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:22:59,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +30: [2023-05-08 11:22:59,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:22:59,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:22:59,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:22:59,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +27: [2023-05-08 11:22:59,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +16: [2023-05-08 11:22:59,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:22:59,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:22:59,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +30: [2023-05-08 11:22:59,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +28: [2023-05-08 11:22:59,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +28: [2023-05-08 11:22:59,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +28: [2023-05-08 11:22:59,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +28: [2023-05-08 11:22:59,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +28: [2023-05-08 11:22:59,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +28: [2023-05-08 11:22:59,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +28: [2023-05-08 11:22:59,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +28: [2023-05-08 11:22:59,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +30: [2023-05-08 11:22:59,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:22:59,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +22: [2023-05-08 11:22:59,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:22:59,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:22:59,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +13: [2023-05-08 11:22:59,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +16: [2023-05-08 11:22:59,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt... +25: [2023-05-08 11:22:59,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +31: [2023-05-08 11:22:59,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:22:59,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:22:59,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +13: [2023-05-08 11:22:59,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +14: [2023-05-08 11:22:59,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:22:59,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:22:59,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +22: [2023-05-08 11:22:59,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +16: [2023-05-08 11:22:59,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:22:59,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +16: [2023-05-08 11:22:59,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +25: [2023-05-08 11:22:59,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:22:59,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:22:59,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +21: [2023-05-08 11:22:59,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:22:59,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:22:59,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:22:59,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +21: [2023-05-08 11:22:59,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +21: [2023-05-08 11:22:59,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +14: [2023-05-08 11:22:59,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:22:59,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:22:59,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +14: [2023-05-08 11:22:59,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +23: [2023-05-08 11:22:59,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:22:59,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:22:59,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:22:59,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +23: [2023-05-08 11:22:59,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +23: [2023-05-08 11:22:59,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +21: [2023-05-08 11:22:59,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:22:59,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:22:59,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +23: [2023-05-08 11:22:59,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 8: [2023-05-08 11:22:59,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 8: [2023-05-08 11:22:59,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +13: [2023-05-08 11:22:59,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:22:59,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +24: [2023-05-08 11:22:59,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:22:59,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:22:59,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:22:59,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +24: [2023-05-08 11:22:59,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +24: [2023-05-08 11:22:59,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +22: [2023-05-08 11:22:59,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:22:59,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +13: [2023-05-08 11:22:59,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +14: [2023-05-08 11:22:59,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:22:59,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +13: [2023-05-08 11:22:59,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +25: [2023-05-08 11:22:59,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +30: [2023-05-08 11:22:59,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:22:59,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:22:59,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +16: [2023-05-08 11:22:59,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:22:59,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +31: [2023-05-08 11:22:59,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +31: [2023-05-08 11:22:59,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +16: [2023-05-08 11:22:59,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:22:59,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +24: [2023-05-08 11:22:59,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +24: [2023-05-08 11:22:59,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +31: [2023-05-08 11:22:59,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +31: [2023-05-08 11:22:59,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +31: [2023-05-08 11:22:59,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +31: [2023-05-08 11:22:59,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +16: [2023-05-08 11:22:59,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +23: [2023-05-08 11:22:59,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:22:59,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:22:59,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:22:59,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +16: [2023-05-08 11:22:59,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt... +31: [2023-05-08 11:22:59,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +31: [2023-05-08 11:22:59,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +22: [2023-05-08 11:22:59,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:22:59,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:22:59,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:22:59,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +11: [2023-05-08 11:22:59,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +22: [2023-05-08 11:22:59,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +22: [2023-05-08 11:22:59,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +11: [2023-05-08 11:22:59,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 8: [2023-05-08 11:22:59,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +11: [2023-05-08 11:22:59,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 8: [2023-05-08 11:22:59,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +11: [2023-05-08 11:22:59,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +23: [2023-05-08 11:22:59,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +23: [2023-05-08 11:22:59,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:22:59,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +21: [2023-05-08 11:22:59,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:22:59,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +21: [2023-05-08 11:22:59,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +21: [2023-05-08 11:22:59,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +21: [2023-05-08 11:22:59,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:22:59,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:22:59,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:22:59,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:22:59,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:22:59,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +22: [2023-05-08 11:22:59,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:22:59,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:22:59,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +22: [2023-05-08 11:22:59,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:22:59,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:22:59,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:22:59,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:22:59,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:22:59,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:22:59,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:22:59,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:22:59,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:22:59,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:22:59,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:22:59,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:22:59,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:22:59,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:22:59,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:22:59,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:22:59,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +24: [2023-05-08 11:22:59,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:22:59,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:22:59,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:22:59,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +24: [2023-05-08 11:22:59,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +24: [2023-05-08 11:22:59,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +24: [2023-05-08 11:22:59,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:22:59,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:22:59,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +24: [2023-05-08 11:22:59,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +22: [2023-05-08 11:22:59,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:22:59,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +22: [2023-05-08 11:22:59,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +24: [2023-05-08 11:22:59,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:22:59,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:22:59,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:22:59,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:22:59,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:22:59,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:22:59,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:22:59,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:22:59,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:22:59,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +21: [2023-05-08 11:22:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +21: [2023-05-08 11:22:59,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +23: [2023-05-08 11:22:59,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +23: [2023-05-08 11:22:59,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +23: [2023-05-08 11:22:59,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +23: [2023-05-08 11:22:59,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +17: [2023-05-08 11:22:59,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +17: [2023-05-08 11:22:59,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +17: [2023-05-08 11:22:59,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +17: [2023-05-08 11:22:59,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 8: [2023-05-08 11:22:59,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +21: [2023-05-08 11:22:59,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:22:59,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:22:59,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:22:59,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:22:59,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:22:59,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +11: [2023-05-08 11:22:59,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:22:59,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:22:59,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:22:59,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +18: [2023-05-08 11:22:59,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:22:59,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:22:59,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +21: [2023-05-08 11:22:59,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:22:59,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:22:59,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:22:59,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +11: [2023-05-08 11:22:59,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +24: [2023-05-08 11:22:59,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:22:59,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:22:59,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +25: [2023-05-08 11:22:59,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +25: [2023-05-08 11:22:59,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +25: [2023-05-08 11:22:59,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +11: [2023-05-08 11:22:59,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +11: [2023-05-08 11:22:59,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +11: [2023-05-08 11:22:59,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:22:59,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:22:59,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +23: [2023-05-08 11:22:59,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:22:59,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:22:59,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +23: [2023-05-08 11:22:59,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:22:59,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:22:59,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:22:59,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:22:59,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +21: [2023-05-08 11:22:59,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:22:59,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:22:59,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:22:59,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:22:59,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +18: [2023-05-08 11:22:59,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:22:59,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:22:59,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +22: [2023-05-08 11:22:59,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +18: [2023-05-08 11:22:59,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 2: [2023-05-08 11:22:59,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +17: [2023-05-08 11:22:59,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:22:59,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:22:59,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 2: [2023-05-08 11:22:59,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +24: [2023-05-08 11:22:59,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +19: [2023-05-08 11:22:59,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +19: [2023-05-08 11:22:59,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +17: [2023-05-08 11:22:59,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +19: [2023-05-08 11:22:59,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 2: [2023-05-08 11:22:59,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +19: [2023-05-08 11:22:59,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 2: [2023-05-08 11:22:59,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +22: [2023-05-08 11:22:59,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +17: [2023-05-08 11:22:59,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:22:59,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:22:59,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +18: [2023-05-08 11:22:59,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +25: [2023-05-08 11:22:59,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +11: [2023-05-08 11:22:59,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:22:59,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:22:59,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:22:59,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:22:59,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:22:59,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:22:59,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:22:59,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:22:59,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:22:59,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +24: [2023-05-08 11:22:59,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +25: [2023-05-08 11:22:59,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +25: [2023-05-08 11:22:59,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +11: [2023-05-08 11:22:59,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +11: [2023-05-08 11:22:59,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +11: [2023-05-08 11:22:59,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:22:59,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +25: [2023-05-08 11:22:59,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +11: [2023-05-08 11:22:59,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +20: [2023-05-08 11:22:59,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +20: [2023-05-08 11:22:59,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +28: [2023-05-08 11:22:59,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +28: [2023-05-08 11:22:59,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +20: [2023-05-08 11:22:59,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +20: [2023-05-08 11:22:59,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +28: [2023-05-08 11:22:59,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +24: [2023-05-08 11:22:59,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +27: [2023-05-08 11:22:59,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +27: [2023-05-08 11:22:59,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +27: [2023-05-08 11:22:59,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +23: [2023-05-08 11:22:59,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +27: [2023-05-08 11:22:59,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 2: [2023-05-08 11:22:59,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:22:59,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:22:59,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +19: [2023-05-08 11:22:59,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:22:59,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:22:59,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:22:59,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:22:59,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:22:59,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 3: [2023-05-08 11:22:59,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 3: [2023-05-08 11:22:59,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 3: [2023-05-08 11:22:59,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +23: [2023-05-08 11:22:59,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:22:59,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +17: [2023-05-08 11:22:59,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:22:59,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:22:59,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +23: [2023-05-08 11:22:59,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:22:59,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +21: [2023-05-08 11:22:59,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +23: [2023-05-08 11:22:59,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +11: [2023-05-08 11:22:59,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:22:59,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:22:59,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +22: [2023-05-08 11:22:59,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:22:59,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:22:59,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:22:59,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +20: [2023-05-08 11:22:59,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:22:59,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:22:59,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +24: [2023-05-08 11:22:59,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:22:59,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +20: [2023-05-08 11:22:59,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +20: [2023-05-08 11:22:59,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:22:59,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:22:59,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +28: [2023-05-08 11:22:59,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +17: [2023-05-08 11:22:59,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +17: [2023-05-08 11:22:59,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +25: [2023-05-08 11:22:59,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:22:59,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +31: [2023-05-08 11:22:59,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +27: [2023-05-08 11:22:59,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:22:59,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:22:59,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:22:59,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +27: [2023-05-08 11:22:59,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +27: [2023-05-08 11:22:59,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +17: [2023-05-08 11:22:59,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 7: [2023-05-08 11:22:59,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:22:59,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 3: [2023-05-08 11:22:59,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:22:59,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:22:59,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 3: [2023-05-08 11:22:59,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:22:59,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:22:59,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 7: [2023-05-08 11:22:59,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:22:59,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:22:59,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:22:59,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:22:59,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +29: [2023-05-08 11:22:59,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:22:59,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:22:59,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +29: [2023-05-08 11:22:59,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:22:59,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +29: [2023-05-08 11:22:59,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:22:59,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:22:59,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:22:59,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:22:59,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +25: [2023-05-08 11:22:59,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 2: [2023-05-08 11:22:59,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 5: [2023-05-08 11:22:59,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 5: [2023-05-08 11:22:59,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +28: [2023-05-08 11:22:59,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +28: [2023-05-08 11:22:59,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +31: [2023-05-08 11:22:59,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:22:59,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:22:59,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 2: [2023-05-08 11:22:59,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:22:59,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 6: [2023-05-08 11:22:59,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +27: [2023-05-08 11:22:59,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:22:59,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:22:59,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +20: [2023-05-08 11:22:59,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:22:59,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +20: [2023-05-08 11:22:59,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +27: [2023-05-08 11:22:59,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 7: [2023-05-08 11:22:59,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +31: [2023-05-08 11:22:59,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:22:59,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +20: [2023-05-08 11:22:59,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 6: [2023-05-08 11:22:59,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 6: [2023-05-08 11:22:59,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 6: [2023-05-08 11:22:59,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:22:59,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +13: [2023-05-08 11:22:59,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +13: [2023-05-08 11:22:59,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +13: [2023-05-08 11:22:59,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +13: [2023-05-08 11:22:59,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +13: [2023-05-08 11:22:59,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:22:59,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:22:59,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:22:59,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +20: [2023-05-08 11:22:59,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +13: [2023-05-08 11:22:59,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +13: [2023-05-08 11:22:59,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +27: [2023-05-08 11:22:59,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +27: [2023-05-08 11:22:59,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 5: [2023-05-08 11:22:59,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +10: [2023-05-08 11:22:59,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 0: [2023-05-08 11:22:59,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +13: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +10: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +10: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 2: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +20: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +20: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +27: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +27: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:22:59,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +27: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +27: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +25: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +28: [2023-05-08 11:22:59,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:22:59,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +29: [2023-05-08 11:22:59,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:22:59,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:22:59,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +10: [2023-05-08 11:22:59,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:22:59,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:22:59,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:22:59,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +10: [2023-05-08 11:22:59,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:22:59,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 3: [2023-05-08 11:22:59,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 2: [2023-05-08 11:22:59,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 7: [2023-05-08 11:22:59,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:22:59,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:22:59,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +28: [2023-05-08 11:22:59,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +10: [2023-05-08 11:22:59,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:22:59,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:22:59,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +12: [2023-05-08 11:22:59,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +12: [2023-05-08 11:22:59,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +12: [2023-05-08 11:22:59,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +10: [2023-05-08 11:22:59,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +12: [2023-05-08 11:22:59,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +28: [2023-05-08 11:22:59,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:22:59,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 7: [2023-05-08 11:22:59,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +10: [2023-05-08 11:22:59,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:22:59,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +29: [2023-05-08 11:22:59,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 3: [2023-05-08 11:22:59,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 2: [2023-05-08 11:22:59,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:22:59,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +10: [2023-05-08 11:22:59,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +13: [2023-05-08 11:22:59,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +10: [2023-05-08 11:22:59,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:22:59,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 2: [2023-05-08 11:22:59,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +13: [2023-05-08 11:22:59,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +10: [2023-05-08 11:22:59,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:22:59,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +14: [2023-05-08 11:22:59,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +14: [2023-05-08 11:22:59,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +20: [2023-05-08 11:22:59,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 0: [2023-05-08 11:22:59,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +29: [2023-05-08 11:22:59,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +12: [2023-05-08 11:22:59,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +12: [2023-05-08 11:22:59,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:22:59,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +12: [2023-05-08 11:22:59,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +29: [2023-05-08 11:22:59,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +12: [2023-05-08 11:22:59,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:22:59,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +12: [2023-05-08 11:22:59,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:22:59,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +10: [2023-05-08 11:22:59,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:22:59,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:22:59,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 0: [2023-05-08 11:22:59,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:22:59,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:22:59,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +28: [2023-05-08 11:22:59,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +12: [2023-05-08 11:22:59,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:22:59,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:22:59,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +28: [2023-05-08 11:22:59,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +28: [2023-05-08 11:22:59,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +28: [2023-05-08 11:22:59,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +28: [2023-05-08 11:22:59,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +28: [2023-05-08 11:22:59,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:22:59,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +12: [2023-05-08 11:22:59,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:22:59,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +20: [2023-05-08 11:22:59,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:22:59,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 9: [2023-05-08 11:22:59,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +21: [2023-05-08 11:22:59,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +21: [2023-05-08 11:22:59,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +21: [2023-05-08 11:22:59,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +21: [2023-05-08 11:22:59,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 9: [2023-05-08 11:22:59,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +10: [2023-05-08 11:22:59,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +20: [2023-05-08 11:22:59,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +10: [2023-05-08 11:22:59,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +29: [2023-05-08 11:22:59,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +16: [2023-05-08 11:22:59,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +16: [2023-05-08 11:22:59,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:22:59,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:22:59,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:22:59,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +24: [2023-05-08 11:22:59,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +24: [2023-05-08 11:22:59,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +24: [2023-05-08 11:22:59,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +24: [2023-05-08 11:22:59,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +14: [2023-05-08 11:22:59,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +16: [2023-05-08 11:22:59,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:22:59,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +16: [2023-05-08 11:22:59,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:22:59,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +13: [2023-05-08 11:22:59,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +12: [2023-05-08 11:22:59,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:22:59,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:22:59,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +16: [2023-05-08 11:22:59,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:22:59,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +20: [2023-05-08 11:22:59,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +13: [2023-05-08 11:22:59,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +13: [2023-05-08 11:22:59,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 9: [2023-05-08 11:22:59,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:22:59,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:22:59,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +21: [2023-05-08 11:22:59,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +30: [2023-05-08 11:22:59,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:22:59,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:22:59,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:22:59,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +13: [2023-05-08 11:22:59,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +21: [2023-05-08 11:22:59,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:22:59,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +20: [2023-05-08 11:22:59,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:22:59,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +10: [2023-05-08 11:22:59,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:22:59,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +21: [2023-05-08 11:22:59,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +30: [2023-05-08 11:22:59,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +13: [2023-05-08 11:22:59,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +13: [2023-05-08 11:22:59,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +20: [2023-05-08 11:22:59,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +13: [2023-05-08 11:22:59,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:22:59,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +13: [2023-05-08 11:22:59,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +12: [2023-05-08 11:22:59,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +12: [2023-05-08 11:22:59,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:22:59,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:22:59,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +24: [2023-05-08 11:22:59,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +24: [2023-05-08 11:22:59,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:22:59,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +10: [2023-05-08 11:22:59,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 7: [2023-05-08 11:22:59,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +10: [2023-05-08 11:22:59,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 7: [2023-05-08 11:22:59,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +13: [2023-05-08 11:22:59,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +24: [2023-05-08 11:22:59,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +24: [2023-05-08 11:22:59,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:22:59,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:22:59,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:22:59,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +10: [2023-05-08 11:22:59,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +26: [2023-05-08 11:22:59,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +10: [2023-05-08 11:22:59,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +12: [2023-05-08 11:22:59,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +13: [2023-05-08 11:22:59,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:22:59,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +12: [2023-05-08 11:22:59,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 3: [2023-05-08 11:22:59,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:22:59,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +12: [2023-05-08 11:22:59,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +14: [2023-05-08 11:22:59,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +14: [2023-05-08 11:22:59,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +14: [2023-05-08 11:22:59,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:22:59,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +18: [2023-05-08 11:22:59,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +18: [2023-05-08 11:22:59,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +14: [2023-05-08 11:22:59,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +14: [2023-05-08 11:22:59,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +10: [2023-05-08 11:22:59,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +10: [2023-05-08 11:22:59,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +18: [2023-05-08 11:22:59,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +26: [2023-05-08 11:22:59,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 6: [2023-05-08 11:22:59,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +28: [2023-05-08 11:22:59,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +14: [2023-05-08 11:22:59,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:22:59,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:22:59,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 9: [2023-05-08 11:22:59,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 9: [2023-05-08 11:22:59,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 9: [2023-05-08 11:22:59,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:22:59,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:22:59,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +28: [2023-05-08 11:22:59,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +12: [2023-05-08 11:22:59,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:22:59,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +20: [2023-05-08 11:22:59,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:22:59,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:22:59,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +12: [2023-05-08 11:22:59,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:22:59,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:22:59,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +26: [2023-05-08 11:22:59,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:22:59,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:22:59,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +27: [2023-05-08 11:22:59,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:22:59,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +30: [2023-05-08 11:22:59,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +27: [2023-05-08 11:22:59,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +27: [2023-05-08 11:22:59,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +27: [2023-05-08 11:22:59,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +27: [2023-05-08 11:22:59,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +27: [2023-05-08 11:22:59,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:22:59,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +16: [2023-05-08 11:22:59,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:22:59,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +22: [2023-05-08 11:22:59,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +21: [2023-05-08 11:22:59,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +26: [2023-05-08 11:22:59,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +19: [2023-05-08 11:22:59,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +19: [2023-05-08 11:22:59,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:22:59,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +27: [2023-05-08 11:22:59,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:22:59,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:22:59,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +19: [2023-05-08 11:22:59,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +21: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:22:59,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +21: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +19: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +19: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 5: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +28: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +19: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:22:59,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +24: [2023-05-08 11:22:59,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +24: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 5: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 5: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 5: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 5: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +28: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +19: [2023-05-08 11:22:59,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +12: [2023-05-08 11:22:59,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:22:59,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 6: [2023-05-08 11:22:59,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:22:59,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +19: [2023-05-08 11:22:59,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +12: [2023-05-08 11:22:59,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:22:59,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:22:59,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:22:59,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:22:59,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:22:59,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:22:59,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +19: [2023-05-08 11:22:59,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:22:59,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +14: [2023-05-08 11:22:59,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:22:59,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:22:59,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +19: [2023-05-08 11:22:59,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:22:59,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:22:59,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +18: [2023-05-08 11:22:59,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +12: [2023-05-08 11:22:59,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +28: [2023-05-08 11:22:59,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +28: [2023-05-08 11:22:59,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:22:59,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:22:59,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +24: [2023-05-08 11:22:59,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:22:59,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +30: [2023-05-08 11:22:59,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:22:59,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +16: [2023-05-08 11:22:59,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:22:59,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:22:59,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:22:59,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:22:59,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +31: [2023-05-08 11:22:59,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +31: [2023-05-08 11:22:59,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +31: [2023-05-08 11:22:59,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +22: [2023-05-08 11:22:59,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:22:59,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:22:59,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:22:59,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:22:59,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:22:59,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:22:59,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +27: [2023-05-08 11:22:59,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +21: [2023-05-08 11:22:59,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:22:59,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +16: [2023-05-08 11:22:59,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +16: [2023-05-08 11:22:59,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +14: [2023-05-08 11:22:59,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +16: [2023-05-08 11:22:59,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt. +22: [2023-05-08 11:22:59,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:22:59,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +22: [2023-05-08 11:22:59,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 0: [2023-05-08 11:22:59,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +24: [2023-05-08 11:22:59,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:22:59,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +24: [2023-05-08 11:22:59,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:22:59,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:22:59,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +14: [2023-05-08 11:22:59,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:22:59,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +14: [2023-05-08 11:22:59,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +14: [2023-05-08 11:22:59,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:22:59,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +19: [2023-05-08 11:22:59,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:22:59,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +15: [2023-05-08 11:22:59,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +14: [2023-05-08 11:22:59,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:22:59,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +21: [2023-05-08 11:22:59,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +25: [2023-05-08 11:22:59,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +25: [2023-05-08 11:22:59,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +25: [2023-05-08 11:22:59,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +16: [2023-05-08 11:22:59,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +31: [2023-05-08 11:22:59,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +15: [2023-05-08 11:22:59,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +30: [2023-05-08 11:22:59,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:22:59,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:22:59,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +29: [2023-05-08 11:22:59,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +29: [2023-05-08 11:22:59,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +26: [2023-05-08 11:22:59,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +16: [2023-05-08 11:22:59,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +27: [2023-05-08 11:22:59,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:22:59,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +18: [2023-05-08 11:22:59,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +18: [2023-05-08 11:22:59,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +26: [2023-05-08 11:22:59,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +25: [2023-05-08 11:22:59,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +30: [2023-05-08 11:22:59,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +31: [2023-05-08 11:22:59,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:22:59,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +26: [2023-05-08 11:22:59,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 4: [2023-05-08 11:22:59,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 9: [2023-05-08 11:22:59,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +29: [2023-05-08 11:22:59,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +30: [2023-05-08 11:22:59,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 4: [2023-05-08 11:22:59,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 4: [2023-05-08 11:22:59,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 1: [2023-05-08 11:22:59,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:22:59,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:22:59,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +16: [2023-05-08 11:22:59,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 1: [2023-05-08 11:22:59,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +14: [2023-05-08 11:22:59,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:22:59,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:22:59,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +26: [2023-05-08 11:22:59,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +14: [2023-05-08 11:22:59,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:22:59,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +18: [2023-05-08 11:22:59,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +18: [2023-05-08 11:22:59,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +14: [2023-05-08 11:22:59,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +16: [2023-05-08 11:22:59,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... +16: [2023-05-08 11:22:59,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +14: [2023-05-08 11:22:59,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +27: [2023-05-08 11:22:59,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:22:59,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:22:59,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:22:59,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +25: [2023-05-08 11:22:59,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:22:59,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:22:59,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:22:59,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:22:59,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +15: [2023-05-08 11:22:59,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:22:59,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +26: [2023-05-08 11:22:59,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:22:59,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:22:59,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +26: [2023-05-08 11:22:59,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +19: [2023-05-08 11:22:59,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:22:59,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +29: [2023-05-08 11:22:59,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:22:59,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +25: [2023-05-08 11:22:59,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:22:59,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:22:59,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 8: [2023-05-08 11:22:59,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 8: [2023-05-08 11:22:59,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +30: [2023-05-08 11:22:59,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +27: [2023-05-08 11:22:59,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 1: [2023-05-08 11:22:59,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:22:59,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +29: [2023-05-08 11:22:59,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 9: [2023-05-08 11:22:59,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +14: [2023-05-08 11:22:59,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:22:59,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:22:59,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +11: [2023-05-08 11:22:59,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +11: [2023-05-08 11:22:59,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +11: [2023-05-08 11:22:59,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 9: [2023-05-08 11:22:59,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:22:59,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:22:59,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +22: [2023-05-08 11:22:59,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 9: [2023-05-08 11:22:59,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +31: [2023-05-08 11:22:59,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +31: [2023-05-08 11:22:59,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:22:59,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +27: [2023-05-08 11:22:59,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:22:59,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +16: [2023-05-08 11:22:59,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +15: [2023-05-08 11:22:59,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +14: [2023-05-08 11:22:59,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:22:59,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +22: [2023-05-08 11:22:59,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +30: [2023-05-08 11:22:59,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +16: [2023-05-08 11:22:59,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:22:59,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:22:59,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +19: [2023-05-08 11:22:59,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +19: [2023-05-08 11:22:59,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:22:59,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +30: [2023-05-08 11:22:59,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +27: [2023-05-08 11:22:59,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:22:59,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 5: [2023-05-08 11:22:59,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:22:59,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +22: [2023-05-08 11:22:59,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:22:59,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:22:59,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +26: [2023-05-08 11:22:59,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:22:59,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +26: [2023-05-08 11:22:59,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +25: [2023-05-08 11:22:59,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +25: [2023-05-08 11:22:59,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +29: [2023-05-08 11:22:59,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +11: [2023-05-08 11:22:59,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:22:59,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:22:59,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +11: [2023-05-08 11:22:59,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:22:59,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +11: [2023-05-08 11:22:59,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:22:59,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +22: [2023-05-08 11:22:59,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +14: [2023-05-08 11:22:59,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:22:59,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:22:59,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +19: [2023-05-08 11:22:59,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +25: [2023-05-08 11:22:59,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +23: [2023-05-08 11:22:59,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +23: [2023-05-08 11:22:59,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +23: [2023-05-08 11:22:59,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +31: [2023-05-08 11:22:59,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +23: [2023-05-08 11:22:59,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +15: [2023-05-08 11:22:59,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +15: [2023-05-08 11:22:59,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +15: [2023-05-08 11:22:59,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +15: [2023-05-08 11:22:59,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 0: [2023-05-08 11:22:59,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +26: [2023-05-08 11:22:59,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:22:59,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +31: [2023-05-08 11:22:59,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 4: [2023-05-08 11:22:59,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +19: [2023-05-08 11:22:59,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +19: [2023-05-08 11:22:59,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +25: [2023-05-08 11:22:59,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:22:59,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 0: [2023-05-08 11:22:59,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +15: [2023-05-08 11:22:59,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +15: [2023-05-08 11:22:59,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +15: [2023-05-08 11:22:59,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +15: [2023-05-08 11:22:59,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +15: [2023-05-08 11:22:59,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:22:59,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:22:59,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 0: [2023-05-08 11:22:59,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +29: [2023-05-08 11:22:59,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:22:59,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +16: [2023-05-08 11:22:59,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +16: [2023-05-08 11:22:59,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 0: [2023-05-08 11:22:59,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +16: [2023-05-08 11:22:59,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt. +16: [2023-05-08 11:22:59,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +16: [2023-05-08 11:22:59,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... +11: [2023-05-08 11:22:59,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +11: [2023-05-08 11:22:59,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:22:59,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:22:59,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt... + 4: [2023-05-08 11:22:59,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 1: [2023-05-08 11:22:59,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 4: [2023-05-08 11:22:59,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +29: [2023-05-08 11:22:59,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:22:59,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:22:59,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:22:59,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +23: [2023-05-08 11:22:59,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:22:59,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:22:59,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:22:59,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +23: [2023-05-08 11:22:59,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:22:59,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:22:59,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:22:59,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +15: [2023-05-08 11:22:59,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +29: [2023-05-08 11:22:59,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:22:59,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:22:59,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +11: [2023-05-08 11:22:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:22:59,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +18: [2023-05-08 11:22:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:22:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:22:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:22:59,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +21: [2023-05-08 11:22:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:22:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:22:59,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +24: [2023-05-08 11:22:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:22:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:22:59,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +24: [2023-05-08 11:22:59,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +18: [2023-05-08 11:22:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +24: [2023-05-08 11:22:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +24: [2023-05-08 11:22:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:22:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:22:59,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +21: [2023-05-08 11:22:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:22:59,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:22:59,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +24: [2023-05-08 11:22:59,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +24: [2023-05-08 11:22:59,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:22:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:22:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:22:59,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +24: [2023-05-08 11:22:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:22:59,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +24: [2023-05-08 11:22:59,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:22:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:22:59,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:22:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +18: [2023-05-08 11:22:59,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +21: [2023-05-08 11:22:59,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:22:59,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +28: [2023-05-08 11:22:59,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +21: [2023-05-08 11:22:59,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:22:59,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +15: [2023-05-08 11:22:59,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +23: [2023-05-08 11:22:59,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +23: [2023-05-08 11:22:59,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +15: [2023-05-08 11:22:59,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +11: [2023-05-08 11:22:59,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +23: [2023-05-08 11:22:59,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +23: [2023-05-08 11:22:59,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +28: [2023-05-08 11:22:59,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +11: [2023-05-08 11:22:59,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +28: [2023-05-08 11:22:59,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +28: [2023-05-08 11:22:59,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +22: [2023-05-08 11:22:59,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:22:59,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:22:59,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:22:59,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:22:59,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +15: [2023-05-08 11:22:59,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:22:59,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:22:59,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +24: [2023-05-08 11:22:59,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:22:59,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +28: [2023-05-08 11:22:59,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:22:59,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +23: [2023-05-08 11:22:59,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +22: [2023-05-08 11:22:59,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:22:59,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:22:59,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +24: [2023-05-08 11:22:59,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +24: [2023-05-08 11:22:59,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:22:59,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:22:59,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:22:59,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +20: [2023-05-08 11:22:59,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +20: [2023-05-08 11:22:59,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +20: [2023-05-08 11:22:59,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +20: [2023-05-08 11:22:59,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 2: [2023-05-08 11:22:59,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +18: [2023-05-08 11:22:59,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:22:59,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +28: [2023-05-08 11:22:59,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:22:59,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:22:59,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 7: [2023-05-08 11:22:59,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +22: [2023-05-08 11:22:59,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +18: [2023-05-08 11:22:59,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +24: [2023-05-08 11:22:59,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:22:59,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 8: [2023-05-08 11:22:59,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:22:59,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:22:59,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:22:59,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:22:59,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:22:59,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +24: [2023-05-08 11:22:59,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:22:59,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:22:59,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +18: [2023-05-08 11:22:59,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +24: [2023-05-08 11:22:59,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 3: [2023-05-08 11:22:59,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 3: [2023-05-08 11:22:59,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 3: [2023-05-08 11:22:59,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +15: [2023-05-08 11:22:59,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:22:59,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +28: [2023-05-08 11:22:59,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:22:59,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +28: [2023-05-08 11:22:59,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 0: [2023-05-08 11:22:59,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +20: [2023-05-08 11:22:59,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +20: [2023-05-08 11:22:59,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:22:59,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:22:59,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:22:59,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +13: [2023-05-08 11:22:59,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +21: [2023-05-08 11:22:59,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:22:59,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +13: [2023-05-08 11:22:59,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +15: [2023-05-08 11:22:59,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:22:59,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +10: [2023-05-08 11:22:59,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:22:59,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:22:59,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +10: [2023-05-08 11:22:59,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +10: [2023-05-08 11:22:59,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:22:59,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +10: [2023-05-08 11:22:59,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +10: [2023-05-08 11:22:59,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:22:59,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:22:59,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +10: [2023-05-08 11:22:59,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:22:59,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +10: [2023-05-08 11:22:59,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:22:59,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:22:59,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 8: [2023-05-08 11:22:59,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +18: [2023-05-08 11:22:59,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:22:59,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +10: [2023-05-08 11:22:59,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:22:59,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:22:59,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:22:59,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:22:59,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:22:59,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +13: [2023-05-08 11:22:59,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:22:59,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +20: [2023-05-08 11:22:59,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +13: [2023-05-08 11:22:59,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:22:59,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:22:59,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +13: [2023-05-08 11:22:59,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +20: [2023-05-08 11:22:59,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:22:59,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:22:59,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:22:59,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +13: [2023-05-08 11:22:59,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:22:59,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +13: [2023-05-08 11:22:59,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:22:59,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +13: [2023-05-08 11:22:59,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:22:59,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +13: [2023-05-08 11:22:59,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +13: [2023-05-08 11:22:59,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 0: [2023-05-08 11:22:59,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +27: [2023-05-08 11:22:59,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +27: [2023-05-08 11:22:59,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +27: [2023-05-08 11:22:59,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +27: [2023-05-08 11:22:59,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 0: [2023-05-08 11:22:59,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 2: [2023-05-08 11:22:59,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:22:59,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:22:59,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 5: [2023-05-08 11:22:59,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 5: [2023-05-08 11:22:59,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 5: [2023-05-08 11:22:59,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +20: [2023-05-08 11:22:59,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +20: [2023-05-08 11:22:59,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 3: [2023-05-08 11:22:59,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:22:59,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 7: [2023-05-08 11:22:59,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 0: [2023-05-08 11:22:59,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:22:59,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +27: [2023-05-08 11:22:59,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +27: [2023-05-08 11:22:59,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:22:59,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +20: [2023-05-08 11:22:59,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +10: [2023-05-08 11:22:59,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +10: [2023-05-08 11:22:59,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +10: [2023-05-08 11:22:59,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +10: [2023-05-08 11:22:59,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +10: [2023-05-08 11:22:59,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +10: [2023-05-08 11:22:59,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +10: [2023-05-08 11:22:59,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +27: [2023-05-08 11:22:59,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 3: [2023-05-08 11:22:59,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +20: [2023-05-08 11:22:59,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +13: [2023-05-08 11:22:59,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:22:59,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +13: [2023-05-08 11:22:59,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:22:59,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +13: [2023-05-08 11:22:59,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +19: [2023-05-08 11:22:59,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 5: [2023-05-08 11:22:59,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:22:59,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:22:59,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 5: [2023-05-08 11:22:59,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 7: [2023-05-08 11:22:59,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +19: [2023-05-08 11:22:59,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 3: [2023-05-08 11:22:59,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +31: [2023-05-08 11:22:59,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +31: [2023-05-08 11:22:59,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +31: [2023-05-08 11:22:59,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +31: [2023-05-08 11:22:59,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +13: [2023-05-08 11:22:59,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:22:59,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +19: [2023-05-08 11:22:59,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +13: [2023-05-08 11:22:59,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:22:59,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +12: [2023-05-08 11:22:59,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:22:59,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 7: [2023-05-08 11:22:59,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 6: [2023-05-08 11:22:59,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 6: [2023-05-08 11:22:59,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +19: [2023-05-08 11:22:59,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 2: [2023-05-08 11:22:59,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 2: [2023-05-08 11:22:59,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 2: [2023-05-08 11:22:59,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 3: [2023-05-08 11:22:59,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 3: [2023-05-08 11:22:59,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +10: [2023-05-08 11:22:59,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:22:59,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +20: [2023-05-08 11:22:59,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:22:59,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:22:59,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:22:59,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:22:59,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:22:59,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:22:59,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +10: [2023-05-08 11:22:59,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:22:59,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +10: [2023-05-08 11:22:59,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +10: [2023-05-08 11:22:59,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +10: [2023-05-08 11:22:59,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:22:59,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:22:59,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +27: [2023-05-08 11:22:59,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +13: [2023-05-08 11:22:59,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:22:59,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:22:59,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +19: [2023-05-08 11:22:59,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +13: [2023-05-08 11:22:59,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:22:59,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:22:59,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +13: [2023-05-08 11:22:59,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +19: [2023-05-08 11:22:59,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:22:59,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +19: [2023-05-08 11:22:59,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:22:59,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:22:59,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +10: [2023-05-08 11:22:59,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +10: [2023-05-08 11:22:59,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +13: [2023-05-08 11:22:59,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:22:59,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:22:59,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:22:59,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +20: [2023-05-08 11:22:59,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:22:59,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +27: [2023-05-08 11:22:59,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +27: [2023-05-08 11:22:59,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +16: [2023-05-08 11:22:59,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 9: [2023-05-08 11:22:59,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +10: [2023-05-08 11:22:59,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:22:59,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +21: [2023-05-08 11:22:59,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +16: [2023-05-08 11:22:59,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +27: [2023-05-08 11:22:59,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:22:59,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +27: [2023-05-08 11:22:59,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +16: [2023-05-08 11:22:59,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +21: [2023-05-08 11:22:59,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +21: [2023-05-08 11:22:59,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 9: [2023-05-08 11:22:59,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +16: [2023-05-08 11:22:59,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +27: [2023-05-08 11:22:59,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:22:59,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:22:59,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +27: [2023-05-08 11:22:59,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +12: [2023-05-08 11:22:59,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +12: [2023-05-08 11:22:59,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +23: [2023-05-08 11:22:59,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +23: [2023-05-08 11:22:59,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +23: [2023-05-08 11:22:59,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:22:59,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +23: [2023-05-08 11:22:59,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:22:59,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:22:59,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:22:59,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:22:59,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +23: [2023-05-08 11:22:59,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:22:59,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +21: [2023-05-08 11:22:59,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +26: [2023-05-08 11:22:59,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +26: [2023-05-08 11:22:59,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +26: [2023-05-08 11:22:59,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +26: [2023-05-08 11:22:59,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +31: [2023-05-08 11:22:59,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:22:59,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +10: [2023-05-08 11:22:59,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:22:59,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +31: [2023-05-08 11:22:59,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:22:59,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:22:59,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +31: [2023-05-08 11:22:59,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 9: [2023-05-08 11:22:59,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 9: [2023-05-08 11:22:59,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +21: [2023-05-08 11:22:59,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +14: [2023-05-08 11:22:59,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 9: [2023-05-08 11:22:59,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +21: [2023-05-08 11:22:59,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:22:59,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:22:59,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:22:59,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +18: [2023-05-08 11:22:59,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +18: [2023-05-08 11:22:59,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +18: [2023-05-08 11:22:59,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 4: [2023-05-08 11:22:59,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 4: [2023-05-08 11:22:59,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 4: [2023-05-08 11:22:59,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 2: [2023-05-08 11:22:59,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 2: [2023-05-08 11:22:59,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +31: [2023-05-08 11:22:59,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:22:59,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 4: [2023-05-08 11:22:59,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 6: [2023-05-08 11:22:59,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +14: [2023-05-08 11:22:59,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +16: [2023-05-08 11:22:59,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:22:59,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:22:59,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 6: [2023-05-08 11:22:59,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 6: [2023-05-08 11:22:59,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +14: [2023-05-08 11:22:59,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +10: [2023-05-08 11:22:59,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 2: [2023-05-08 11:22:59,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +12: [2023-05-08 11:22:59,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +24: [2023-05-08 11:22:59,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +24: [2023-05-08 11:22:59,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +10: [2023-05-08 11:22:59,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +15: [2023-05-08 11:22:59,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +15: [2023-05-08 11:22:59,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +15: [2023-05-08 11:22:59,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +15: [2023-05-08 11:22:59,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +12: [2023-05-08 11:22:59,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:22:59,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:22:59,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 9: [2023-05-08 11:22:59,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:22:59,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 6: [2023-05-08 11:22:59,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +24: [2023-05-08 11:22:59,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. + 2: [2023-05-08 11:22:59,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +24: [2023-05-08 11:22:59,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt. +26: [2023-05-08 11:22:59,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:22:59,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:22:59,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 2: [2023-05-08 11:22:59,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +26: [2023-05-08 11:22:59,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:22:59,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:22:59,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 9: [2023-05-08 11:22:59,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:22:59,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:22:59,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 9: [2023-05-08 11:22:59,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:22:59,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +18: [2023-05-08 11:22:59,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:22:59,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +30: [2023-05-08 11:22:59,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +30: [2023-05-08 11:22:59,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 4: [2023-05-08 11:22:59,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:22:59,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:22:59,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 4: [2023-05-08 11:22:59,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:22:59,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 9: [2023-05-08 11:23:00,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 0: [2023-05-08 11:23:00,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 0: [2023-05-08 11:23:00,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +21: [2023-05-08 11:23:00,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:23:00,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +14: [2023-05-08 11:23:00,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:23:00,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:23:00,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +24: [2023-05-08 11:23:00,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +10: [2023-05-08 11:23:00,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:23:00,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +25: [2023-05-08 11:23:00,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +25: [2023-05-08 11:23:00,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +25: [2023-05-08 11:23:00,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +16: [2023-05-08 11:23:00,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +16: [2023-05-08 11:23:00,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +12: [2023-05-08 11:23:00,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:23:00,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +10: [2023-05-08 11:23:00,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +15: [2023-05-08 11:23:00,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:23:00,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:23:00,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:23:00,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +16: [2023-05-08 11:23:00,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +24: [2023-05-08 11:23:00,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:23:00,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +24: [2023-05-08 11:23:00,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:23:00,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:23:00,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:23:00,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:23:00,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:23:00,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +12: [2023-05-08 11:23:00,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +15: [2023-05-08 11:23:00,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:23:00,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +21: [2023-05-08 11:23:00,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +26: [2023-05-08 11:23:00,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +21: [2023-05-08 11:23:00,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +16: [2023-05-08 11:23:00,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +16: [2023-05-08 11:23:00,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +21: [2023-05-08 11:23:00,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:23:00,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:23:00,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +25: [2023-05-08 11:23:00,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:23:00,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:23:00,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:23:00,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +23: [2023-05-08 11:23:00,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +30: [2023-05-08 11:23:00,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:23:00,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +30: [2023-05-08 11:23:00,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:23:00,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:23:00,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:23:00,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +12: [2023-05-08 11:23:00,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:23:00,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +25: [2023-05-08 11:23:00,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +16: [2023-05-08 11:23:00,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 9: [2023-05-08 11:23:00,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:23:00,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +18: [2023-05-08 11:23:00,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 9: [2023-05-08 11:23:00,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 9: [2023-05-08 11:23:00,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +11: [2023-05-08 11:23:00,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +11: [2023-05-08 11:23:00,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +11: [2023-05-08 11:23:00,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 0: [2023-05-08 11:23:00,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:23:00,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +14: [2023-05-08 11:23:00,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 0: [2023-05-08 11:23:00,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +24: [2023-05-08 11:23:00,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +24: [2023-05-08 11:23:00,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +23: [2023-05-08 11:23:00,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +14: [2023-05-08 11:23:00,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +22: [2023-05-08 11:23:00,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +22: [2023-05-08 11:23:00,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +22: [2023-05-08 11:23:00,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +22: [2023-05-08 11:23:00,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +24: [2023-05-08 11:23:00,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +18: [2023-05-08 11:23:00,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:23:00,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +18: [2023-05-08 11:23:00,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +15: [2023-05-08 11:23:00,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:23:00,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:23:00,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +15: [2023-05-08 11:23:00,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +23: [2023-05-08 11:23:00,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:23:00,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +18: [2023-05-08 11:23:00,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:23:00,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +24: [2023-05-08 11:23:00,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 4: [2023-05-08 11:23:00,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:23:00,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:23:00,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:23:00,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:23:00,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +11: [2023-05-08 11:23:00,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:23:00,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:23:00,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +26: [2023-05-08 11:23:00,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 4: [2023-05-08 11:23:00,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +30: [2023-05-08 11:23:00,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:23:00,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +24: [2023-05-08 11:23:00,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +24: [2023-05-08 11:23:00,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +23: [2023-05-08 11:23:00,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:23:00,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +11: [2023-05-08 11:23:00,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:23:00,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +25: [2023-05-08 11:23:00,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 1: [2023-05-08 11:23:00,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 1: [2023-05-08 11:23:00,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 1: [2023-05-08 11:23:00,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +25: [2023-05-08 11:23:00,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +25: [2023-05-08 11:23:00,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:23:00,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 0: [2023-05-08 11:23:00,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +17: [2023-05-08 11:23:00,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +17: [2023-05-08 11:23:00,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +17: [2023-05-08 11:23:00,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +17: [2023-05-08 11:23:00,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 0: [2023-05-08 11:23:00,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +22: [2023-05-08 11:23:00,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 8: [2023-05-08 11:23:00,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 8: [2023-05-08 11:23:00,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 8: [2023-05-08 11:23:00,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +30: [2023-05-08 11:23:00,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:23:00,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +30: [2023-05-08 11:23:00,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:23:00,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +22: [2023-05-08 11:23:00,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:23:00,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +22: [2023-05-08 11:23:00,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:23:00,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +22: [2023-05-08 11:23:00,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:23:00,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +14: [2023-05-08 11:23:00,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... + 4: [2023-05-08 11:23:00,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 0: [2023-05-08 11:23:00,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +14: [2023-05-08 11:23:00,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt. +14: [2023-05-08 11:23:00,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt... +15: [2023-05-08 11:23:00,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:23:00,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 4: [2023-05-08 11:23:00,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +25: [2023-05-08 11:23:00,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:23:00,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +15: [2023-05-08 11:23:00,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +25: [2023-05-08 11:23:00,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 4: [2023-05-08 11:23:00,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 1: [2023-05-08 11:23:00,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:23:00,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:23:00,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +28: [2023-05-08 11:23:00,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:23:00,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:23:00,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:23:00,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:23:00,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:23:00,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:23:00,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +25: [2023-05-08 11:23:00,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +28: [2023-05-08 11:23:00,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:23:00,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:23:00,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:23:00,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +17: [2023-05-08 11:23:00,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +17: [2023-05-08 11:23:00,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:23:00,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +11: [2023-05-08 11:23:00,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +11: [2023-05-08 11:23:00,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +11: [2023-05-08 11:23:00,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +11: [2023-05-08 11:23:00,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:23:00,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +14: [2023-05-08 11:23:00,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:23:00,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:23:00,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +14: [2023-05-08 11:23:00,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:23:00,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:23:00,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:23:00,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:23:00,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:23:00,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:23:00,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +22: [2023-05-08 11:23:00,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:23:00,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 0: [2023-05-08 11:23:00,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +28: [2023-05-08 11:23:00,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:23:00,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:23:00,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +11: [2023-05-08 11:23:00,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +11: [2023-05-08 11:23:00,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 8: [2023-05-08 11:23:00,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +22: [2023-05-08 11:23:00,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:23:00,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +14: [2023-05-08 11:23:00,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:23:00,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:23:00,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +28: [2023-05-08 11:23:00,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:23:00,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:23:00,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 7: [2023-05-08 11:23:00,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +28: [2023-05-08 11:23:00,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 8: [2023-05-08 11:23:00,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:23:00,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 8: [2023-05-08 11:23:00,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +23: [2023-05-08 11:23:00,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 4: [2023-05-08 11:23:00,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:23:00,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +23: [2023-05-08 11:23:00,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 7: [2023-05-08 11:23:00,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +23: [2023-05-08 11:23:00,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 7: [2023-05-08 11:23:00,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +14: [2023-05-08 11:23:00,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:23:00,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +20: [2023-05-08 11:23:00,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +20: [2023-05-08 11:23:00,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +20: [2023-05-08 11:23:00,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +17: [2023-05-08 11:23:00,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:23:00,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:23:00,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:23:00,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:23:00,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:23:00,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:23:00,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +17: [2023-05-08 11:23:00,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:23:00,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +17: [2023-05-08 11:23:00,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:23:00,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +17: [2023-05-08 11:23:00,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +17: [2023-05-08 11:23:00,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +14: [2023-05-08 11:23:00,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +17: [2023-05-08 11:23:00,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +28: [2023-05-08 11:23:00,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:23:00,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +17: [2023-05-08 11:23:00,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +17: [2023-05-08 11:23:00,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +28: [2023-05-08 11:23:00,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +27: [2023-05-08 11:23:00,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +27: [2023-05-08 11:23:00,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +27: [2023-05-08 11:23:00,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:23:00,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:23:00,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +28: [2023-05-08 11:23:00,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +28: [2023-05-08 11:23:00,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +30: [2023-05-08 11:23:00,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +30: [2023-05-08 11:23:00,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +30: [2023-05-08 11:23:00,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:23:00,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:23:00,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +30: [2023-05-08 11:23:00,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:23:00,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:23:00,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +19: [2023-05-08 11:23:00,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +19: [2023-05-08 11:23:00,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +19: [2023-05-08 11:23:00,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +19: [2023-05-08 11:23:00,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +19: [2023-05-08 11:23:00,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +19: [2023-05-08 11:23:00,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +19: [2023-05-08 11:23:00,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +19: [2023-05-08 11:23:00,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +19: [2023-05-08 11:23:00,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +19: [2023-05-08 11:23:00,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +27: [2023-05-08 11:23:00,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +27: [2023-05-08 11:23:00,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +27: [2023-05-08 11:23:00,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +19: [2023-05-08 11:23:00,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +27: [2023-05-08 11:23:00,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:23:00,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +26: [2023-05-08 11:23:00,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +19: [2023-05-08 11:23:00,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +26: [2023-05-08 11:23:00,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:23:00,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:23:00,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:23:00,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +23: [2023-05-08 11:23:00,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:23:00,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +26: [2023-05-08 11:23:00,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:23:00,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:23:00,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +23: [2023-05-08 11:23:00,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +20: [2023-05-08 11:23:00,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:23:00,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:23:00,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +19: [2023-05-08 11:23:00,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:23:00,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +20: [2023-05-08 11:23:00,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 2: [2023-05-08 11:23:00,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 2: [2023-05-08 11:23:00,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +26: [2023-05-08 11:23:00,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +23: [2023-05-08 11:23:00,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +20: [2023-05-08 11:23:00,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:23:00,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:23:00,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:23:00,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:23:00,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:23:00,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:23:00,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:23:00,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:23:00,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +30: [2023-05-08 11:23:00,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:23:00,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:23:00,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +27: [2023-05-08 11:23:00,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:23:00,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +27: [2023-05-08 11:23:00,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:23:00,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 7: [2023-05-08 11:23:00,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +19: [2023-05-08 11:23:00,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:23:00,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:23:00,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +20: [2023-05-08 11:23:00,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 4: [2023-05-08 11:23:00,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 5: [2023-05-08 11:23:00,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:23:00,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +30: [2023-05-08 11:23:00,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 5: [2023-05-08 11:23:00,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 5: [2023-05-08 11:23:00,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:23:00,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +27: [2023-05-08 11:23:00,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +17: [2023-05-08 11:23:00,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 9: [2023-05-08 11:23:00,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 9: [2023-05-08 11:23:00,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 5: [2023-05-08 11:23:00,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +28: [2023-05-08 11:23:00,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +27: [2023-05-08 11:23:00,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +28: [2023-05-08 11:23:00,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +23: [2023-05-08 11:23:00,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:23:00,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:23:00,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +23: [2023-05-08 11:23:00,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 7: [2023-05-08 11:23:00,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +17: [2023-05-08 11:23:00,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:23:00,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +10: [2023-05-08 11:23:00,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +10: [2023-05-08 11:23:00,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +10: [2023-05-08 11:23:00,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 1: [2023-05-08 11:23:00,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +19: [2023-05-08 11:23:00,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +19: [2023-05-08 11:23:00,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 1: [2023-05-08 11:23:00,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 1: [2023-05-08 11:23:00,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 1: [2023-05-08 11:23:00,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 1: [2023-05-08 11:23:00,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:23:00,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:23:00,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 1: [2023-05-08 11:23:00,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +30: [2023-05-08 11:23:00,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +27: [2023-05-08 11:23:00,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:23:00,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:23:00,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +27: [2023-05-08 11:23:00,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:23:00,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +20: [2023-05-08 11:23:00,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:23:00,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +27: [2023-05-08 11:23:00,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:23:00,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +20: [2023-05-08 11:23:00,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 2: [2023-05-08 11:23:00,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +19: [2023-05-08 11:23:00,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:23:00,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:23:00,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:23:00,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:23:00,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +26: [2023-05-08 11:23:00,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +26: [2023-05-08 11:23:00,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:23:00,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:23:00,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:23:00,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:23:00,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +28: [2023-05-08 11:23:00,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +11: [2023-05-08 11:23:00,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:23:00,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +26: [2023-05-08 11:23:00,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +26: [2023-05-08 11:23:00,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +27: [2023-05-08 11:23:00,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:23:00,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:23:00,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:23:00,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +11: [2023-05-08 11:23:00,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +11: [2023-05-08 11:23:00,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +10: [2023-05-08 11:23:00,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:23:00,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:23:00,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:23:00,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +19: [2023-05-08 11:23:00,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +27: [2023-05-08 11:23:00,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:23:00,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:23:00,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:23:00,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +17: [2023-05-08 11:23:00,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +15: [2023-05-08 11:23:00,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +15: [2023-05-08 11:23:00,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +10: [2023-05-08 11:23:00,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +10: [2023-05-08 11:23:00,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +19: [2023-05-08 11:23:00,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +26: [2023-05-08 11:23:00,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:23:00,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:23:00,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +18: [2023-05-08 11:23:00,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +18: [2023-05-08 11:23:00,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +16: [2023-05-08 11:23:00,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:23:00,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:23:00,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:23:00,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +16: [2023-05-08 11:23:00,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +18: [2023-05-08 11:23:00,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +26: [2023-05-08 11:23:00,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +18: [2023-05-08 11:23:00,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +25: [2023-05-08 11:23:00,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +25: [2023-05-08 11:23:00,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +25: [2023-05-08 11:23:00,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +30: [2023-05-08 11:23:00,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +25: [2023-05-08 11:23:00,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +26: [2023-05-08 11:23:00,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:23:00,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +26: [2023-05-08 11:23:00,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:23:00,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:23:00,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +25: [2023-05-08 11:23:00,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +25: [2023-05-08 11:23:00,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:23:00,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:23:00,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +25: [2023-05-08 11:23:00,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +25: [2023-05-08 11:23:00,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +25: [2023-05-08 11:23:00,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:23:00,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +25: [2023-05-08 11:23:00,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:23:00,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +15: [2023-05-08 11:23:00,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +15: [2023-05-08 11:23:00,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 5: [2023-05-08 11:23:00,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +15: [2023-05-08 11:23:00,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 1: [2023-05-08 11:23:00,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +15: [2023-05-08 11:23:00,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 9: [2023-05-08 11:23:00,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +18: [2023-05-08 11:23:00,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +18: [2023-05-08 11:23:00,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +26: [2023-05-08 11:23:00,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:23:00,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +18: [2023-05-08 11:23:00,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:23:00,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:23:00,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +27: [2023-05-08 11:23:00,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +27: [2023-05-08 11:23:00,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +25: [2023-05-08 11:23:00,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:23:00,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:23:00,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +25: [2023-05-08 11:23:00,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:23:00,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +10: [2023-05-08 11:23:00,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +10: [2023-05-08 11:23:00,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +27: [2023-05-08 11:23:00,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +27: [2023-05-08 11:23:00,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +10: [2023-05-08 11:23:00,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +15: [2023-05-08 11:23:00,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:23:00,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 9: [2023-05-08 11:23:00,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 0: [2023-05-08 11:23:00,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 0: [2023-05-08 11:23:00,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 4: [2023-05-08 11:23:00,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:23:00,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:23:00,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +27: [2023-05-08 11:23:00,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +10: [2023-05-08 11:23:00,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 4: [2023-05-08 11:23:00,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +19: [2023-05-08 11:23:00,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +10: [2023-05-08 11:23:00,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +10: [2023-05-08 11:23:00,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 4: [2023-05-08 11:23:00,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +25: [2023-05-08 11:23:00,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +10: [2023-05-08 11:23:00,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +10: [2023-05-08 11:23:00,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 1: [2023-05-08 11:23:00,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:23:00,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +11: [2023-05-08 11:23:00,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:23:00,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:23:00,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:23:00,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +26: [2023-05-08 11:23:00,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +26: [2023-05-08 11:23:00,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:23:00,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +30: [2023-05-08 11:23:00,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +30: [2023-05-08 11:23:00,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +26: [2023-05-08 11:23:00,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +26: [2023-05-08 11:23:00,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +26: [2023-05-08 11:23:00,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +30: [2023-05-08 11:23:00,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 6: [2023-05-08 11:23:00,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 6: [2023-05-08 11:23:00,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +18: [2023-05-08 11:23:00,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +21: [2023-05-08 11:23:00,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +21: [2023-05-08 11:23:00,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +21: [2023-05-08 11:23:00,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +18: [2023-05-08 11:23:00,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +21: [2023-05-08 11:23:00,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 6: [2023-05-08 11:23:00,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +18: [2023-05-08 11:23:00,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:23:00,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +12: [2023-05-08 11:23:00,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +12: [2023-05-08 11:23:00,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +25: [2023-05-08 11:23:00,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:23:00,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +12: [2023-05-08 11:23:00,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +24: [2023-05-08 11:23:00,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +24: [2023-05-08 11:23:00,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +24: [2023-05-08 11:23:00,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +29: [2023-05-08 11:23:00,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +29: [2023-05-08 11:23:00,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +29: [2023-05-08 11:23:00,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +29: [2023-05-08 11:23:00,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 1: [2023-05-08 11:23:00,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:23:00,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 9: [2023-05-08 11:23:00,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:23:00,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:23:00,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:23:00,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +31: [2023-05-08 11:23:00,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +31: [2023-05-08 11:23:00,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +31: [2023-05-08 11:23:00,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +31: [2023-05-08 11:23:00,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 3: [2023-05-08 11:23:00,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 3: [2023-05-08 11:23:00,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +18: [2023-05-08 11:23:00,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:23:00,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +18: [2023-05-08 11:23:00,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:23:00,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 9: [2023-05-08 11:23:00,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +31: [2023-05-08 11:23:00,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:23:00,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +15: [2023-05-08 11:23:00,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +15: [2023-05-08 11:23:00,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +15: [2023-05-08 11:23:00,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +11: [2023-05-08 11:23:00,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +18: [2023-05-08 11:23:00,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:23:00,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +15: [2023-05-08 11:23:00,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +15: [2023-05-08 11:23:00,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +18: [2023-05-08 11:23:00,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:23:00,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:23:00,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +18: [2023-05-08 11:23:00,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +18: [2023-05-08 11:23:00,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:23:00,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +18: [2023-05-08 11:23:00,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:23:00,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:23:00,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:23:00,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:23:00,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:23:00,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:23:00,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:23:00,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +21: [2023-05-08 11:23:00,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:23:00,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +26: [2023-05-08 11:23:00,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +26: [2023-05-08 11:23:00,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +24: [2023-05-08 11:23:00,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:23:00,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:23:00,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:23:00,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +24: [2023-05-08 11:23:00,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +12: [2023-05-08 11:23:00,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:23:00,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +29: [2023-05-08 11:23:00,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +30: [2023-05-08 11:23:00,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:23:00,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +24: [2023-05-08 11:23:00,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +29: [2023-05-08 11:23:00,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:23:00,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +29: [2023-05-08 11:23:00,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +29: [2023-05-08 11:23:00,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:23:00,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +24: [2023-05-08 11:23:00,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:23:00,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +25: [2023-05-08 11:23:00,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +25: [2023-05-08 11:23:00,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 3: [2023-05-08 11:23:00,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +22: [2023-05-08 11:23:00,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +31: [2023-05-08 11:23:00,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +22: [2023-05-08 11:23:00,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 3: [2023-05-08 11:23:00,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +25: [2023-05-08 11:23:00,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +25: [2023-05-08 11:23:00,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +25: [2023-05-08 11:23:00,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +25: [2023-05-08 11:23:00,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 0: [2023-05-08 11:23:00,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +22: [2023-05-08 11:23:00,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +18: [2023-05-08 11:23:00,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +25: [2023-05-08 11:23:00,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:23:00,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +18: [2023-05-08 11:23:00,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:23:00,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:23:00,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:23:00,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +16: [2023-05-08 11:23:00,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:23:00,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +16: [2023-05-08 11:23:00,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +16: [2023-05-08 11:23:00,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +16: [2023-05-08 11:23:00,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +16: [2023-05-08 11:23:00,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +16: [2023-05-08 11:23:00,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:23:00,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +16: [2023-05-08 11:23:00,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +16: [2023-05-08 11:23:00,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +16: [2023-05-08 11:23:00,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +16: [2023-05-08 11:23:00,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:23:00,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:23:00,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:23:00,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +16: [2023-05-08 11:23:00,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +16: [2023-05-08 11:23:00,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +16: [2023-05-08 11:23:00,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:23:00,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +16: [2023-05-08 11:23:00,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +31: [2023-05-08 11:23:00,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +30: [2023-05-08 11:23:00,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:23:00,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +14: [2023-05-08 11:23:00,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +14: [2023-05-08 11:23:00,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +14: [2023-05-08 11:23:00,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. +14: [2023-05-08 11:23:00,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt. + 6: [2023-05-08 11:23:00,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +24: [2023-05-08 11:23:00,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:23:00,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +24: [2023-05-08 11:23:00,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 4: [2023-05-08 11:23:00,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +24: [2023-05-08 11:23:00,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:23:00,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +25: [2023-05-08 11:23:00,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:23:00,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:23:00,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:23:00,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +30: [2023-05-08 11:23:00,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:23:00,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +21: [2023-05-08 11:23:00,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +21: [2023-05-08 11:23:00,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +21: [2023-05-08 11:23:00,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +12: [2023-05-08 11:23:00,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +12: [2023-05-08 11:23:00,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:23:00,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:23:00,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +12: [2023-05-08 11:23:00,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +25: [2023-05-08 11:23:00,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +25: [2023-05-08 11:23:00,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +22: [2023-05-08 11:23:00,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +18: [2023-05-08 11:23:00,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +18: [2023-05-08 11:23:00,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:23:00,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:23:00,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +21: [2023-05-08 11:23:00,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:23:00,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:23:00,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +18: [2023-05-08 11:23:00,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:23:00,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:23:00,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:23:00,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 0: [2023-05-08 11:23:00,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +14: [2023-05-08 11:23:00,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:23:00,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:23:00,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:23:00,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +14: [2023-05-08 11:23:00,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt... +12: [2023-05-08 11:23:00,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:23:00,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:23:00,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:23:00,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:23:00,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +30: [2023-05-08 11:23:00,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 6: [2023-05-08 11:23:00,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:23:00,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 6: [2023-05-08 11:23:00,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:23:00,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +13: [2023-05-08 11:23:00,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +13: [2023-05-08 11:23:00,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +13: [2023-05-08 11:23:00,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +13: [2023-05-08 11:23:00,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +22: [2023-05-08 11:23:00,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:23:00,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 0: [2023-05-08 11:23:00,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +21: [2023-05-08 11:23:00,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +24: [2023-05-08 11:23:00,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:23:00,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:23:00,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 7: [2023-05-08 11:23:00,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:23:00,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:23:00,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +15: [2023-05-08 11:23:00,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +15: [2023-05-08 11:23:00,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:23:00,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:23:00,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:23:00,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:23:00,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:23:00,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +29: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +29: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +29: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +29: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +31: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +29: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +29: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +29: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +29: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +29: [2023-05-08 11:23:00,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 3: [2023-05-08 11:23:00,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 3: [2023-05-08 11:23:00,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 7: [2023-05-08 11:23:00,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +29: [2023-05-08 11:23:00,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +29: [2023-05-08 11:23:00,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +29: [2023-05-08 11:23:00,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +29: [2023-05-08 11:23:00,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +29: [2023-05-08 11:23:00,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +29: [2023-05-08 11:23:00,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 6: [2023-05-08 11:23:00,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 6: [2023-05-08 11:23:00,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +29: [2023-05-08 11:23:00,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:23:00,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:23:00,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 6: [2023-05-08 11:23:00,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +13: [2023-05-08 11:23:00,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +13: [2023-05-08 11:23:00,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:23:00,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:23:00,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:23:00,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +13: [2023-05-08 11:23:00,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +31: [2023-05-08 11:23:00,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +15: [2023-05-08 11:23:00,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +14: [2023-05-08 11:23:00,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +14: [2023-05-08 11:23:00,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +14: [2023-05-08 11:23:00,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +14: [2023-05-08 11:23:00,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +14: [2023-05-08 11:23:00,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +14: [2023-05-08 11:23:00,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +15: [2023-05-08 11:23:00,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:23:00,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:23:00,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:23:00,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:23:00,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:23:00,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:23:00,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:23:00,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +29: [2023-05-08 11:23:00,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:23:00,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:23:00,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +15: [2023-05-08 11:23:00,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +29: [2023-05-08 11:23:00,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +14: [2023-05-08 11:23:00,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +29: [2023-05-08 11:23:00,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:23:00,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:23:00,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +16: [2023-05-08 11:23:00,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:23:00,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +16: [2023-05-08 11:23:00,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:23:00,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... +14: [2023-05-08 11:23:00,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt... + 3: [2023-05-08 11:23:00,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +15: [2023-05-08 11:23:00,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +15: [2023-05-08 11:23:00,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:23:00,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +31: [2023-05-08 11:23:00,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +31: [2023-05-08 11:23:00,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +31: [2023-05-08 11:23:00,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +29: [2023-05-08 11:23:00,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:23:00,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:23:00,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +15: [2023-05-08 11:23:00,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +15: [2023-05-08 11:23:00,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +29: [2023-05-08 11:23:00,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:23:00,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 2: [2023-05-08 11:23:00,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +29: [2023-05-08 11:23:00,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 2: [2023-05-08 11:23:00,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 6: [2023-05-08 11:23:00,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:23:00,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:23:00,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:23:00,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +29: [2023-05-08 11:23:00,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:23:00,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:23:00,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +31: [2023-05-08 11:23:00,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:23:00,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +29: [2023-05-08 11:23:00,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:23:00,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:23:00,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:23:00,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 2: [2023-05-08 11:23:00,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +31: [2023-05-08 11:23:00,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +31: [2023-05-08 11:23:00,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +24: [2023-05-08 11:23:00,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:23:00,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +19: [2023-05-08 11:23:00,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +19: [2023-05-08 11:23:00,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +19: [2023-05-08 11:23:00,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +31: [2023-05-08 11:23:00,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:23:00,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 2: [2023-05-08 11:23:00,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 2: [2023-05-08 11:23:00,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:23:00,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +11: [2023-05-08 11:23:00,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +11: [2023-05-08 11:23:00,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +11: [2023-05-08 11:23:00,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +31: [2023-05-08 11:23:00,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +11: [2023-05-08 11:23:00,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +19: [2023-05-08 11:23:00,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:23:00,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:23:00,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +22: [2023-05-08 11:23:00,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +22: [2023-05-08 11:23:00,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +22: [2023-05-08 11:23:00,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +22: [2023-05-08 11:23:00,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:23:00,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 8: [2023-05-08 11:23:00,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 8: [2023-05-08 11:23:00,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 8: [2023-05-08 11:23:00,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +19: [2023-05-08 11:23:00,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:23:00,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:23:00,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:23:00,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:23:00,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:23:00,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:23:00,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:23:00,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 6: [2023-05-08 11:23:00,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 8: [2023-05-08 11:23:00,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:23:00,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +20: [2023-05-08 11:23:00,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +20: [2023-05-08 11:23:00,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +20: [2023-05-08 11:23:00,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +22: [2023-05-08 11:23:00,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 6: [2023-05-08 11:23:00,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 8: [2023-05-08 11:23:00,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +16: [2023-05-08 11:23:00,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +16: [2023-05-08 11:23:00,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +16: [2023-05-08 11:23:00,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +16: [2023-05-08 11:23:00,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +24: [2023-05-08 11:23:00,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +24: [2023-05-08 11:23:00,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +24: [2023-05-08 11:23:00,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +24: [2023-05-08 11:23:00,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +11: [2023-05-08 11:23:00,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:23:00,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +11: [2023-05-08 11:23:00,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:23:00,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 6: [2023-05-08 11:23:00,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:23:00,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:23:00,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +20: [2023-05-08 11:23:00,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:23:00,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:23:00,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:23:00,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +20: [2023-05-08 11:23:00,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +16: [2023-05-08 11:23:00,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:23:00,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +24: [2023-05-08 11:23:00,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +24: [2023-05-08 11:23:00,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +24: [2023-05-08 11:23:00,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:23:00,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:23:00,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 8: [2023-05-08 11:23:00,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:23:00,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +20: [2023-05-08 11:23:00,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +20: [2023-05-08 11:23:00,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:23:00,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +22: [2023-05-08 11:23:00,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +20: [2023-05-08 11:23:00,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +20: [2023-05-08 11:23:00,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +20: [2023-05-08 11:23:00,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:23:00,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:23:00,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +28: [2023-05-08 11:23:00,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +28: [2023-05-08 11:23:00,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 8: [2023-05-08 11:23:00,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:23:00,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +20: [2023-05-08 11:23:00,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:23:00,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:23:00,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:23:00,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:23:00,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:23:00,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +19: [2023-05-08 11:23:00,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:23:00,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:23:00,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:23:00,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:23:00,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:23:00,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:23:00,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 9: [2023-05-08 11:23:00,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:23:00,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:23:00,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +19: [2023-05-08 11:23:00,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 9: [2023-05-08 11:23:00,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +20: [2023-05-08 11:23:00,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:23:00,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:23:00,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 9: [2023-05-08 11:23:00,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:23:00,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:23:00,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:23:00,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:23:00,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +16: [2023-05-08 11:23:00,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:23:00,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:23:00,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +16: [2023-05-08 11:23:00,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +22: [2023-05-08 11:23:00,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:23:00,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:23:00,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:23:00,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:23:00,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:23:00,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:23:00,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:23:00,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:23:00,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +22: [2023-05-08 11:23:00,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:23:00,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 8: [2023-05-08 11:23:00,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:23:00,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +16: [2023-05-08 11:23:00,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +16: [2023-05-08 11:23:00,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +28: [2023-05-08 11:23:00,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:23:00,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 6: [2023-05-08 11:23:00,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 6: [2023-05-08 11:23:00,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 6: [2023-05-08 11:23:00,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 8: [2023-05-08 11:23:00,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +23: [2023-05-08 11:23:00,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:23:00,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:23:00,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:23:00,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +23: [2023-05-08 11:23:00,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:23:00,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:23:00,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +23: [2023-05-08 11:23:00,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:23:00,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:23:00,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +22: [2023-05-08 11:23:00,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +25: [2023-05-08 11:23:00,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +25: [2023-05-08 11:23:00,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +25: [2023-05-08 11:23:00,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +25: [2023-05-08 11:23:00,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 7: [2023-05-08 11:23:00,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +25: [2023-05-08 11:23:00,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +25: [2023-05-08 11:23:00,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:23:00,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 1: [2023-05-08 11:23:00,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +25: [2023-05-08 11:23:00,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:23:00,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +23: [2023-05-08 11:23:00,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 1: [2023-05-08 11:23:00,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 1: [2023-05-08 11:23:00,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +23: [2023-05-08 11:23:00,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 6: [2023-05-08 11:23:00,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:23:00,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +24: [2023-05-08 11:23:00,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:23:00,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:23:00,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +24: [2023-05-08 11:23:00,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +10: [2023-05-08 11:23:00,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +10: [2023-05-08 11:23:00,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +10: [2023-05-08 11:23:00,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +10: [2023-05-08 11:23:00,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +10: [2023-05-08 11:23:00,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +10: [2023-05-08 11:23:00,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +10: [2023-05-08 11:23:00,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +20: [2023-05-08 11:23:00,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 6: [2023-05-08 11:23:00,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +19: [2023-05-08 11:23:00,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +20: [2023-05-08 11:23:00,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +20: [2023-05-08 11:23:00,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +20: [2023-05-08 11:23:00,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +10: [2023-05-08 11:23:00,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:23:00,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:23:00,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:23:00,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +16: [2023-05-08 11:23:00,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +30: [2023-05-08 11:23:00,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +20: [2023-05-08 11:23:00,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +20: [2023-05-08 11:23:00,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +30: [2023-05-08 11:23:00,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:23:00,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +30: [2023-05-08 11:23:00,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +30: [2023-05-08 11:23:00,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +30: [2023-05-08 11:23:00,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:23:00,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +20: [2023-05-08 11:23:00,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +20: [2023-05-08 11:23:00,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +30: [2023-05-08 11:23:00,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:23:00,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:23:00,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:23:00,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:23:00,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:23:00,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:23:00,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:23:00,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:23:00,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +17: [2023-05-08 11:23:00,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:23:00,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +17: [2023-05-08 11:23:00,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +19: [2023-05-08 11:23:00,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:23:00,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:23:00,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:23:00,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +11: [2023-05-08 11:23:00,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +11: [2023-05-08 11:23:00,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +11: [2023-05-08 11:23:00,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +11: [2023-05-08 11:23:00,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +11: [2023-05-08 11:23:00,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +11: [2023-05-08 11:23:00,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +11: [2023-05-08 11:23:00,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:23:00,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:23:00,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:23:00,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 4: [2023-05-08 11:23:00,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:23:00,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:23:00,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:23:00,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:23:00,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 4: [2023-05-08 11:23:00,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:23:00,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:23:00,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:23:00,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:23:00,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:23:00,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +28: [2023-05-08 11:23:00,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:23:00,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +13: [2023-05-08 11:23:00,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +13: [2023-05-08 11:23:00,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +13: [2023-05-08 11:23:00,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +13: [2023-05-08 11:23:00,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +13: [2023-05-08 11:23:00,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +13: [2023-05-08 11:23:00,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +13: [2023-05-08 11:23:00,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +13: [2023-05-08 11:23:00,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +13: [2023-05-08 11:23:00,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +13: [2023-05-08 11:23:00,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:23:00,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +13: [2023-05-08 11:23:00,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +13: [2023-05-08 11:23:00,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +28: [2023-05-08 11:23:00,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:23:00,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +13: [2023-05-08 11:23:00,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 0: [2023-05-08 11:23:00,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +10: [2023-05-08 11:23:00,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +13: [2023-05-08 11:23:00,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +12: [2023-05-08 11:23:00,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:23:00,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:23:00,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:23:00,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:23:00,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:23:00,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:23:00,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:23:00,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +13: [2023-05-08 11:23:00,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +13: [2023-05-08 11:23:00,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +19: [2023-05-08 11:23:00,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:23:00,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:23:00,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:23:00,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:23:00,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +25: [2023-05-08 11:23:00,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +25: [2023-05-08 11:23:00,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +25: [2023-05-08 11:23:00,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +20: [2023-05-08 11:23:00,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:23:00,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +14: [2023-05-08 11:23:00,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 9: [2023-05-08 11:23:00,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +14: [2023-05-08 11:23:00,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. + 9: [2023-05-08 11:23:00,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 9: [2023-05-08 11:23:00,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 9: [2023-05-08 11:23:00,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 8: [2023-05-08 11:23:00,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:23:00,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:23:00,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +30: [2023-05-08 11:23:00,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:23:00,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:23:00,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +20: [2023-05-08 11:23:00,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:23:00,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:23:00,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +10: [2023-05-08 11:23:00,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:23:00,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +16: [2023-05-08 11:23:00,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +16: [2023-05-08 11:23:00,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 5: [2023-05-08 11:23:00,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +16: [2023-05-08 11:23:00,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:23:00,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +21: [2023-05-08 11:23:00,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +19: [2023-05-08 11:23:00,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:23:00,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +17: [2023-05-08 11:23:00,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +16: [2023-05-08 11:23:00,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +16: [2023-05-08 11:23:00,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:23:00,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +10: [2023-05-08 11:23:00,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +23: [2023-05-08 11:23:00,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:23:00,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:23:00,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +12: [2023-05-08 11:23:00,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +16: [2023-05-08 11:23:00,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +16: [2023-05-08 11:23:00,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:23:00,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:23:00,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:23:00,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +17: [2023-05-08 11:23:00,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:23:00,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +13: [2023-05-08 11:23:00,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +23: [2023-05-08 11:23:00,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:23:00,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +13: [2023-05-08 11:23:00,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +30: [2023-05-08 11:23:00,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:23:00,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +11: [2023-05-08 11:23:00,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:23:00,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +11: [2023-05-08 11:23:00,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:23:00,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +14: [2023-05-08 11:23:00,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +27: [2023-05-08 11:23:00,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:23:00,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +27: [2023-05-08 11:23:00,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:23:00,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +23: [2023-05-08 11:23:00,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:23:00,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt. +27: [2023-05-08 11:23:00,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +27: [2023-05-08 11:23:00,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:23:00,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:23:00,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:23:00,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +14: [2023-05-08 11:23:00,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +27: [2023-05-08 11:23:00,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +10: [2023-05-08 11:23:00,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:23:00,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +25: [2023-05-08 11:23:00,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:23:00,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +25: [2023-05-08 11:23:00,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +25: [2023-05-08 11:23:00,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +14: [2023-05-08 11:23:00,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:23:00,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +27: [2023-05-08 11:23:00,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:23:00,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +10: [2023-05-08 11:23:00,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:23:00,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:23:00,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:23:00,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +14: [2023-05-08 11:23:00,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +13: [2023-05-08 11:23:00,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:23:00,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +14: [2023-05-08 11:23:00,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:23:00,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:23:00,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:23:00,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:23:00,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:23:00,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:23:00,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +26: [2023-05-08 11:23:00,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +26: [2023-05-08 11:23:00,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +26: [2023-05-08 11:23:00,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:23:00,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:23:00,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:23:00,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:23:00,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:23:00,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:23:00,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:23:00,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:23:00,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:23:00,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:23:00,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:23:00,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:23:00,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:23:00,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:23:00,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +17: [2023-05-08 11:23:00,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +13: [2023-05-08 11:23:00,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +11: [2023-05-08 11:23:00,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:23:00,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:23:00,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:23:00,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:23:00,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +13: [2023-05-08 11:23:00,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:23:00,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:23:00,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:23:00,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +11: [2023-05-08 11:23:00,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +12: [2023-05-08 11:23:00,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:23:00,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:23:00,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +31: [2023-05-08 11:23:00,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:23:00,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:23:00,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +11: [2023-05-08 11:23:00,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:23:00,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +25: [2023-05-08 11:23:00,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:23:00,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +23: [2023-05-08 11:23:00,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +23: [2023-05-08 11:23:00,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +30: [2023-05-08 11:23:00,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:23:00,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +30: [2023-05-08 11:23:00,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:23:00,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:23:00,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:23:00,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:23:00,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:23:00,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +31: [2023-05-08 11:23:00,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:23:00,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:23:00,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 0: [2023-05-08 11:23:00,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 0: [2023-05-08 11:23:00,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +12: [2023-05-08 11:23:00,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +13: [2023-05-08 11:23:00,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +27: [2023-05-08 11:23:00,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:23:00,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 7: [2023-05-08 11:23:00,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +12: [2023-05-08 11:23:00,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:23:00,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:23:00,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:23:00,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:23:00,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:23:00,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:23:00,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:23:00,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +12: [2023-05-08 11:23:00,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +28: [2023-05-08 11:23:00,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 9: [2023-05-08 11:23:00,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +28: [2023-05-08 11:23:00,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +11: [2023-05-08 11:23:00,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:23:00,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:23:00,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +30: [2023-05-08 11:23:00,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +30: [2023-05-08 11:23:00,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 5: [2023-05-08 11:23:00,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +21: [2023-05-08 11:23:00,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:23:00,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +21: [2023-05-08 11:23:00,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:23:00,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:23:00,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +12: [2023-05-08 11:23:00,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:23:00,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:23:00,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 5: [2023-05-08 11:23:00,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +14: [2023-05-08 11:23:00,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:23:00,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:23:00,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +27: [2023-05-08 11:23:00,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:23:00,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +27: [2023-05-08 11:23:00,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:23:00,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +27: [2023-05-08 11:23:00,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +16: [2023-05-08 11:23:00,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:23:00,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:23:00,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:23:00,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 0: [2023-05-08 11:23:00,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:23:00,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +31: [2023-05-08 11:23:00,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:23:00,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:23:00,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:23:00,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +21: [2023-05-08 11:23:00,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 5: [2023-05-08 11:23:00,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 5: [2023-05-08 11:23:00,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 1: [2023-05-08 11:23:00,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 1: [2023-05-08 11:23:00,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 1: [2023-05-08 11:23:00,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 1: [2023-05-08 11:23:00,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +27: [2023-05-08 11:23:00,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:23:00,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +31: [2023-05-08 11:23:00,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +26: [2023-05-08 11:23:00,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +26: [2023-05-08 11:23:00,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +26: [2023-05-08 11:23:00,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +26: [2023-05-08 11:23:00,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:23:00,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +13: [2023-05-08 11:23:00,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 5: [2023-05-08 11:23:00,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:23:00,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +31: [2023-05-08 11:23:00,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +14: [2023-05-08 11:23:00,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:23:00,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:23:00,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 5: [2023-05-08 11:23:00,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +31: [2023-05-08 11:23:00,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:23:00,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +27: [2023-05-08 11:23:00,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:23:00,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:23:00,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:23:00,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:23:00,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +26: [2023-05-08 11:23:00,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:23:00,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:23:00,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:23:00,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:23:00,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:23:00,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... +14: [2023-05-08 11:23:00,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 0: [2023-05-08 11:23:00,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +14: [2023-05-08 11:23:00,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt... + 5: [2023-05-08 11:23:00,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 0: [2023-05-08 11:23:00,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 0: [2023-05-08 11:23:00,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +31: [2023-05-08 11:23:00,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +31: [2023-05-08 11:23:00,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:23:00,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +26: [2023-05-08 11:23:00,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +26: [2023-05-08 11:23:00,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 1: [2023-05-08 11:23:00,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +26: [2023-05-08 11:23:00,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 1: [2023-05-08 11:23:00,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +14: [2023-05-08 11:23:00,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:23:00,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +26: [2023-05-08 11:23:00,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:23:00,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:23:00,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:23:00,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:23:00,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 1: [2023-05-08 11:23:00,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:23:00,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +26: [2023-05-08 11:23:00,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +26: [2023-05-08 11:23:00,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +26: [2023-05-08 11:23:00,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +10: [2023-05-08 11:23:00,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +10: [2023-05-08 11:23:00,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +10: [2023-05-08 11:23:00,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +10: [2023-05-08 11:23:00,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +18: [2023-05-08 11:23:00,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +18: [2023-05-08 11:23:00,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +18: [2023-05-08 11:23:00,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +18: [2023-05-08 11:23:00,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 1: [2023-05-08 11:23:00,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +15: [2023-05-08 11:23:00,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +15: [2023-05-08 11:23:00,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +15: [2023-05-08 11:23:00,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +26: [2023-05-08 11:23:00,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +26: [2023-05-08 11:23:00,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +25: [2023-05-08 11:23:00,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +25: [2023-05-08 11:23:00,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +25: [2023-05-08 11:23:00,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +25: [2023-05-08 11:23:00,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +22: [2023-05-08 11:23:00,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +30: [2023-05-08 11:23:00,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +30: [2023-05-08 11:23:00,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +22: [2023-05-08 11:23:00,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +22: [2023-05-08 11:23:00,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +22: [2023-05-08 11:23:00,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +30: [2023-05-08 11:23:00,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +30: [2023-05-08 11:23:00,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +10: [2023-05-08 11:23:00,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +10: [2023-05-08 11:23:00,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 4: [2023-05-08 11:23:00,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +15: [2023-05-08 11:23:00,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +10: [2023-05-08 11:23:00,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +10: [2023-05-08 11:23:00,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +27: [2023-05-08 11:23:00,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 2: [2023-05-08 11:23:00,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +27: [2023-05-08 11:23:00,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +27: [2023-05-08 11:23:00,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +18: [2023-05-08 11:23:00,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +18: [2023-05-08 11:23:00,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:23:00,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +18: [2023-05-08 11:23:00,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +18: [2023-05-08 11:23:00,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +18: [2023-05-08 11:23:00,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +18: [2023-05-08 11:23:00,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +18: [2023-05-08 11:23:00,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +18: [2023-05-08 11:23:00,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +18: [2023-05-08 11:23:00,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +18: [2023-05-08 11:23:00,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +18: [2023-05-08 11:23:00,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +18: [2023-05-08 11:23:00,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:23:00,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +26: [2023-05-08 11:23:00,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +15: [2023-05-08 11:23:00,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:23:00,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:23:00,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:23:00,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:23:00,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:23:00,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:23:00,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +24: [2023-05-08 11:23:00,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +24: [2023-05-08 11:23:00,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +24: [2023-05-08 11:23:00,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +15: [2023-05-08 11:23:00,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:23:00,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 6: [2023-05-08 11:23:00,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 6: [2023-05-08 11:23:00,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +22: [2023-05-08 11:23:00,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:23:00,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:23:00,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:23:00,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:23:00,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:23:00,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:23:00,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +27: [2023-05-08 11:23:00,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 2: [2023-05-08 11:23:00,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 3: [2023-05-08 11:23:00,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +17: [2023-05-08 11:23:00,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +17: [2023-05-08 11:23:00,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +24: [2023-05-08 11:23:00,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 4: [2023-05-08 11:23:00,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +27: [2023-05-08 11:23:00,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +27: [2023-05-08 11:23:00,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 3: [2023-05-08 11:23:00,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 4: [2023-05-08 11:23:00,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +16: [2023-05-08 11:23:00,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +16: [2023-05-08 11:23:00,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +16: [2023-05-08 11:23:00,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +27: [2023-05-08 11:23:00,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +16: [2023-05-08 11:23:00,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 2: [2023-05-08 11:23:00,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:23:00,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +10: [2023-05-08 11:23:00,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +10: [2023-05-08 11:23:00,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +24: [2023-05-08 11:23:00,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:23:00,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +10: [2023-05-08 11:23:00,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +10: [2023-05-08 11:23:00,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +10: [2023-05-08 11:23:00,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +10: [2023-05-08 11:23:00,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +18: [2023-05-08 11:23:00,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:23:00,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +17: [2023-05-08 11:23:00,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +18: [2023-05-08 11:23:00,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:23:00,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +15: [2023-05-08 11:23:00,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:23:00,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:23:00,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:23:00,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +22: [2023-05-08 11:23:00,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +15: [2023-05-08 11:23:00,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +17: [2023-05-08 11:23:00,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +27: [2023-05-08 11:23:00,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +30: [2023-05-08 11:23:00,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:23:00,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:23:00,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:23:00,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +30: [2023-05-08 11:23:00,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +15: [2023-05-08 11:23:00,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +15: [2023-05-08 11:23:00,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +16: [2023-05-08 11:23:00,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +16: [2023-05-08 11:23:00,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +16: [2023-05-08 11:23:00,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:23:00,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:23:00,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:23:00,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +27: [2023-05-08 11:23:00,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:23:00,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +25: [2023-05-08 11:23:00,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +25: [2023-05-08 11:23:00,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +25: [2023-05-08 11:23:00,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:23:00,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:23:00,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:23:00,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +29: [2023-05-08 11:23:00,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +29: [2023-05-08 11:23:00,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +18: [2023-05-08 11:23:00,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +25: [2023-05-08 11:23:00,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +25: [2023-05-08 11:23:00,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 2: [2023-05-08 11:23:00,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:23:00,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +18: [2023-05-08 11:23:00,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +18: [2023-05-08 11:23:00,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +29: [2023-05-08 11:23:00,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +30: [2023-05-08 11:23:00,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:23:00,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:23:00,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +22: [2023-05-08 11:23:00,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +30: [2023-05-08 11:23:00,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +12: [2023-05-08 11:23:00,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 4: [2023-05-08 11:23:00,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:23:00,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +12: [2023-05-08 11:23:00,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. + 4: [2023-05-08 11:23:00,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +24: [2023-05-08 11:23:00,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +22: [2023-05-08 11:23:00,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +17: [2023-05-08 11:23:00,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:23:00,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +17: [2023-05-08 11:23:00,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:23:00,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 2: [2023-05-08 11:23:00,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 4: [2023-05-08 11:23:00,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +15: [2023-05-08 11:23:00,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:23:00,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +15: [2023-05-08 11:23:00,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +10: [2023-05-08 11:23:00,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +30: [2023-05-08 11:23:00,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:23:00,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:23:00,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:23:00,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:23:00,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +24: [2023-05-08 11:23:00,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +15: [2023-05-08 11:23:00,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +30: [2023-05-08 11:23:00,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +24: [2023-05-08 11:23:00,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +15: [2023-05-08 11:23:00,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +18: [2023-05-08 11:23:00,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +18: [2023-05-08 11:23:00,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:23:00,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 6: [2023-05-08 11:23:00,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:23:00,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +25: [2023-05-08 11:23:00,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:23:00,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +25: [2023-05-08 11:23:00,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +22: [2023-05-08 11:23:00,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +29: [2023-05-08 11:23:00,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +29: [2023-05-08 11:23:00,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +29: [2023-05-08 11:23:00,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:23:00,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +29: [2023-05-08 11:23:00,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:23:00,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:23:00,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:23:00,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +22: [2023-05-08 11:23:00,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:23:00,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:23:00,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:23:00,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:23:00,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +29: [2023-05-08 11:23:00,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:23:00,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:23:00,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:23:00,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +29: [2023-05-08 11:23:00,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +16: [2023-05-08 11:23:00,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +29: [2023-05-08 11:23:00,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +16: [2023-05-08 11:23:00,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +16: [2023-05-08 11:23:00,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +12: [2023-05-08 11:23:00,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +27: [2023-05-08 11:23:00,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +29: [2023-05-08 11:23:00,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:23:00,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +17: [2023-05-08 11:23:00,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +22: [2023-05-08 11:23:00,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +17: [2023-05-08 11:23:00,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +18: [2023-05-08 11:23:00,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:23:00,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:23:00,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +17: [2023-05-08 11:23:00,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:23:00,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +16: [2023-05-08 11:23:00,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:23:00,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:23:00,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +16: [2023-05-08 11:23:00,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 2: [2023-05-08 11:23:00,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 6: [2023-05-08 11:23:00,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 6: [2023-05-08 11:23:00,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +12: [2023-05-08 11:23:00,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:23:00,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +12: [2023-05-08 11:23:00,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:23:00,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +31: [2023-05-08 11:23:00,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +31: [2023-05-08 11:23:00,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +31: [2023-05-08 11:23:00,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +12: [2023-05-08 11:23:00,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +29: [2023-05-08 11:23:00,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:23:00,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:23:00,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:23:00,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +12: [2023-05-08 11:23:00,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:23:00,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 7: [2023-05-08 11:23:00,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:23:00,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:23:00,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +31: [2023-05-08 11:23:00,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:23:00,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:23:00,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:23:00,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +29: [2023-05-08 11:23:00,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:23:00,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:23:00,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:23:00,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +29: [2023-05-08 11:23:00,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:23:00,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +29: [2023-05-08 11:23:00,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:23:00,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:23:00,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +21: [2023-05-08 11:23:00,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +21: [2023-05-08 11:23:00,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +21: [2023-05-08 11:23:00,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +29: [2023-05-08 11:23:00,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:23:00,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +14: [2023-05-08 11:23:00,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +14: [2023-05-08 11:23:00,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +14: [2023-05-08 11:23:00,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt. +29: [2023-05-08 11:23:00,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +31: [2023-05-08 11:23:00,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +31: [2023-05-08 11:23:00,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +29: [2023-05-08 11:23:00,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:23:00,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +29: [2023-05-08 11:23:00,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +31: [2023-05-08 11:23:00,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +31: [2023-05-08 11:23:00,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +31: [2023-05-08 11:23:00,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +31: [2023-05-08 11:23:00,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +14: [2023-05-08 11:23:00,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:23:00,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:23:00,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:23:00,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:23:00,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:23:00,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:23:00,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:23:00,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +22: [2023-05-08 11:23:00,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +22: [2023-05-08 11:23:00,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:23:00,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:23:00,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +22: [2023-05-08 11:23:00,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:23:00,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +31: [2023-05-08 11:23:00,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 8: [2023-05-08 11:23:00,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:23:00,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +22: [2023-05-08 11:23:00,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 8: [2023-05-08 11:23:00,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 8: [2023-05-08 11:23:00,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 8: [2023-05-08 11:23:00,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +21: [2023-05-08 11:23:00,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:23:00,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:23:00,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:23:00,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:23:00,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:23:00,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:23:00,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:23:00,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:23:00,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +21: [2023-05-08 11:23:00,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +13: [2023-05-08 11:23:00,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +13: [2023-05-08 11:23:00,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +21: [2023-05-08 11:23:00,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:23:00,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +13: [2023-05-08 11:23:00,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +21: [2023-05-08 11:23:00,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:23:00,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 7: [2023-05-08 11:23:00,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:23:00,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 7: [2023-05-08 11:23:00,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:23:00,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:23:00,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:23:00,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 3: [2023-05-08 11:23:00,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 3: [2023-05-08 11:23:00,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +14: [2023-05-08 11:23:00,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 3: [2023-05-08 11:23:00,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 3: [2023-05-08 11:23:00,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:23:00,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +22: [2023-05-08 11:23:00,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +11: [2023-05-08 11:23:00,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +11: [2023-05-08 11:23:00,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +11: [2023-05-08 11:23:00,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +11: [2023-05-08 11:23:00,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +13: [2023-05-08 11:23:00,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +28: [2023-05-08 11:23:00,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +28: [2023-05-08 11:23:00,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +28: [2023-05-08 11:23:00,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +28: [2023-05-08 11:23:00,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +22: [2023-05-08 11:23:00,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:23:00,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:23:00,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +14: [2023-05-08 11:23:00,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +14: [2023-05-08 11:23:00,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +14: [2023-05-08 11:23:00,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +21: [2023-05-08 11:23:00,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:23:00,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +13: [2023-05-08 11:23:00,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:23:00,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:23:00,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +22: [2023-05-08 11:23:00,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:23:00,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:23:00,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +14: [2023-05-08 11:23:00,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... + 3: [2023-05-08 11:23:00,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +11: [2023-05-08 11:23:00,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +11: [2023-05-08 11:23:00,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:23:00,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:23:00,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +11: [2023-05-08 11:23:00,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:23:00,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:23:00,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +28: [2023-05-08 11:23:00,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +28: [2023-05-08 11:23:00,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:23:00,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +21: [2023-05-08 11:23:00,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:23:00,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt... +21: [2023-05-08 11:23:00,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:23:00,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:23:00,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 8: [2023-05-08 11:23:00,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:23:00,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +11: [2023-05-08 11:23:00,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 8: [2023-05-08 11:23:00,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +20: [2023-05-08 11:23:00,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +21: [2023-05-08 11:23:00,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +20: [2023-05-08 11:23:00,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +20: [2023-05-08 11:23:00,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +24: [2023-05-08 11:23:00,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +24: [2023-05-08 11:23:00,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +24: [2023-05-08 11:23:00,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +24: [2023-05-08 11:23:00,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:23:00,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:23:00,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:23:00,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +24: [2023-05-08 11:23:00,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +24: [2023-05-08 11:23:00,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +11: [2023-05-08 11:23:00,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +11: [2023-05-08 11:23:00,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 2: [2023-05-08 11:23:00,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +23: [2023-05-08 11:23:00,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +23: [2023-05-08 11:23:00,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +23: [2023-05-08 11:23:00,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +23: [2023-05-08 11:23:00,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:23:00,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:23:00,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:23:00,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:23:00,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:23:00,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:23:00,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:23:00,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:23:00,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:23:00,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:23:00,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:23:00,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:23:00,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +20: [2023-05-08 11:23:00,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +11: [2023-05-08 11:23:00,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +20: [2023-05-08 11:23:00,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +11: [2023-05-08 11:23:00,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +20: [2023-05-08 11:23:00,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +11: [2023-05-08 11:23:00,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +11: [2023-05-08 11:23:00,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:23:00,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +24: [2023-05-08 11:23:00,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +30: [2023-05-08 11:23:00,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +30: [2023-05-08 11:23:00,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +30: [2023-05-08 11:23:00,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +30: [2023-05-08 11:23:00,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:23:00,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +30: [2023-05-08 11:23:00,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +30: [2023-05-08 11:23:00,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +30: [2023-05-08 11:23:00,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +24: [2023-05-08 11:23:00,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:23:00,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +30: [2023-05-08 11:23:00,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:23:00,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +23: [2023-05-08 11:23:00,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:23:00,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:23:00,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:23:00,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:23:00,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:23:00,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:23:00,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:23:00,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:23:00,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:23:00,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:23:00,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:23:00,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:23:00,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +20: [2023-05-08 11:23:00,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +20: [2023-05-08 11:23:00,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +20: [2023-05-08 11:23:00,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +20: [2023-05-08 11:23:00,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +20: [2023-05-08 11:23:00,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +20: [2023-05-08 11:23:00,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:23:00,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:23:00,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +20: [2023-05-08 11:23:00,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +20: [2023-05-08 11:23:00,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +20: [2023-05-08 11:23:00,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:23:00,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +20: [2023-05-08 11:23:00,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +20: [2023-05-08 11:23:00,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +20: [2023-05-08 11:23:00,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +20: [2023-05-08 11:23:00,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:23:00,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:23:00,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +30: [2023-05-08 11:23:00,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +23: [2023-05-08 11:23:00,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:23:00,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +30: [2023-05-08 11:23:00,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:23:00,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:23:00,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +31: [2023-05-08 11:23:00,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:23:00,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +31: [2023-05-08 11:23:00,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:23:00,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:23:00,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:23:00,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:23:00,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:23:00,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:23:00,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:23:00,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:23:00,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:23:00,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +25: [2023-05-08 11:23:00,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +25: [2023-05-08 11:23:00,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +25: [2023-05-08 11:23:00,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:23:00,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 7: [2023-05-08 11:23:00,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +30: [2023-05-08 11:23:00,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:23:00,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:23:00,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +11: [2023-05-08 11:23:00,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +20: [2023-05-08 11:23:00,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:23:00,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:23:00,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +16: [2023-05-08 11:23:00,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:23:00,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +30: [2023-05-08 11:23:00,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:23:00,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:23:00,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +20: [2023-05-08 11:23:00,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +30: [2023-05-08 11:23:00,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +16: [2023-05-08 11:23:00,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +28: [2023-05-08 11:23:00,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:23:00,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:23:00,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:23:00,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +28: [2023-05-08 11:23:00,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:23:00,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 0: [2023-05-08 11:23:00,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:23:00,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:23:00,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:23:00,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:23:00,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:23:00,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:23:00,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +28: [2023-05-08 11:23:00,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +28: [2023-05-08 11:23:00,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +28: [2023-05-08 11:23:00,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +28: [2023-05-08 11:23:00,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +31: [2023-05-08 11:23:00,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +25: [2023-05-08 11:23:00,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:23:00,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:23:00,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +16: [2023-05-08 11:23:00,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:23:00,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:23:00,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:23:00,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:23:00,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:23:00,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:23:00,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +19: [2023-05-08 11:23:00,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +16: [2023-05-08 11:23:00,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:23:00,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +19: [2023-05-08 11:23:00,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:23:00,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:23:00,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:23:00,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +19: [2023-05-08 11:23:00,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +19: [2023-05-08 11:23:00,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 9: [2023-05-08 11:23:00,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +16: [2023-05-08 11:23:00,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 4: [2023-05-08 11:23:00,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +20: [2023-05-08 11:23:00,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +31: [2023-05-08 11:23:00,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:23:00,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:23:00,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:23:00,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +25: [2023-05-08 11:23:00,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +19: [2023-05-08 11:23:00,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +16: [2023-05-08 11:23:00,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:23:00,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 6: [2023-05-08 11:23:00,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:23:00,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +19: [2023-05-08 11:23:00,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +23: [2023-05-08 11:23:00,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +23: [2023-05-08 11:23:00,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +28: [2023-05-08 11:23:00,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:23:00,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +28: [2023-05-08 11:23:00,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +25: [2023-05-08 11:23:00,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:23:00,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +10: [2023-05-08 11:23:00,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +10: [2023-05-08 11:23:00,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +10: [2023-05-08 11:23:00,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +30: [2023-05-08 11:23:00,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:23:00,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +31: [2023-05-08 11:23:00,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +10: [2023-05-08 11:23:00,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 0: [2023-05-08 11:23:00,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 0: [2023-05-08 11:23:00,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +31: [2023-05-08 11:23:00,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +31: [2023-05-08 11:23:00,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +28: [2023-05-08 11:23:00,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +25: [2023-05-08 11:23:00,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +28: [2023-05-08 11:23:00,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +13: [2023-05-08 11:23:00,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +10: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +13: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +13: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +10: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +10: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +26: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +26: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +13: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +13: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 9: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +25: [2023-05-08 11:23:00,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:23:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +25: [2023-05-08 11:23:00,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:23:00,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:23:00,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 7: [2023-05-08 11:23:00,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +23: [2023-05-08 11:23:00,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:23:00,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +10: [2023-05-08 11:23:00,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:23:00,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:23:00,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:23:00,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:23:00,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:23:00,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:23:00,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:23:00,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:23:00,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:23:00,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +28: [2023-05-08 11:23:00,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:23:00,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +19: [2023-05-08 11:23:00,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:23:00,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:23:00,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:23:00,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:23:00,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +28: [2023-05-08 11:23:00,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:23:00,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 0: [2023-05-08 11:23:00,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:23:00,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +15: [2023-05-08 11:23:00,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +15: [2023-05-08 11:23:00,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +15: [2023-05-08 11:23:00,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +15: [2023-05-08 11:23:00,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +12: [2023-05-08 11:23:00,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +12: [2023-05-08 11:23:00,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +12: [2023-05-08 11:23:00,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:23:00,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:23:00,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +12: [2023-05-08 11:23:00,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +12: [2023-05-08 11:23:00,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +15: [2023-05-08 11:23:00,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:23:00,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +15: [2023-05-08 11:23:00,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +13: [2023-05-08 11:23:00,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:23:00,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:23:00,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:23:00,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +27: [2023-05-08 11:23:00,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +15: [2023-05-08 11:23:00,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +27: [2023-05-08 11:23:00,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:23:00,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:23:00,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +27: [2023-05-08 11:23:00,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:23:00,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:23:00,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:23:00,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 7: [2023-05-08 11:23:00,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 0: [2023-05-08 11:23:00,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:23:00,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +14: [2023-05-08 11:23:00,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +27: [2023-05-08 11:23:00,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:23:00,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +14: [2023-05-08 11:23:00,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:23:00,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:23:00,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +13: [2023-05-08 11:23:00,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:23:00,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +14: [2023-05-08 11:23:00,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +14: [2023-05-08 11:23:00,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +14: [2023-05-08 11:23:00,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:23:00,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 5: [2023-05-08 11:23:00,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 5: [2023-05-08 11:23:00,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +10: [2023-05-08 11:23:00,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +10: [2023-05-08 11:23:00,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +23: [2023-05-08 11:23:00,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:23:00,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:23:00,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +23: [2023-05-08 11:23:00,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +19: [2023-05-08 11:23:00,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +19: [2023-05-08 11:23:00,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +19: [2023-05-08 11:23:00,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:23:00,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +10: [2023-05-08 11:23:00,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:23:00,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +12: [2023-05-08 11:23:00,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +15: [2023-05-08 11:23:00,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +26: [2023-05-08 11:23:00,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +12: [2023-05-08 11:23:00,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +19: [2023-05-08 11:23:00,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +26: [2023-05-08 11:23:00,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 1: [2023-05-08 11:23:00,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +15: [2023-05-08 11:23:00,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +15: [2023-05-08 11:23:00,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +26: [2023-05-08 11:23:00,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +13: [2023-05-08 11:23:00,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +26: [2023-05-08 11:23:00,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +26: [2023-05-08 11:23:00,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:23:00,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +26: [2023-05-08 11:23:00,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 0: [2023-05-08 11:23:00,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +15: [2023-05-08 11:23:00,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 9: [2023-05-08 11:23:00,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +15: [2023-05-08 11:23:00,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 9: [2023-05-08 11:23:00,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 9: [2023-05-08 11:23:00,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 9: [2023-05-08 11:23:00,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +27: [2023-05-08 11:23:00,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +10: [2023-05-08 11:23:00,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +22: [2023-05-08 11:23:00,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 1: [2023-05-08 11:23:00,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:23:00,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +22: [2023-05-08 11:23:00,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +22: [2023-05-08 11:23:00,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +14: [2023-05-08 11:23:00,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 5: [2023-05-08 11:23:00,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +12: [2023-05-08 11:23:00,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +12: [2023-05-08 11:23:00,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 4: [2023-05-08 11:23:00,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +19: [2023-05-08 11:23:00,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:23:00,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:23:00,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +13: [2023-05-08 11:23:00,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +13: [2023-05-08 11:23:00,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:23:00,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:23:00,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:23:00,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:23:00,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:23:00,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:23:00,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +10: [2023-05-08 11:23:00,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +10: [2023-05-08 11:23:00,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +10: [2023-05-08 11:23:00,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +12: [2023-05-08 11:23:00,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +12: [2023-05-08 11:23:00,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:23:00,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +18: [2023-05-08 11:23:00,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +19: [2023-05-08 11:23:00,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +19: [2023-05-08 11:23:00,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +26: [2023-05-08 11:23:00,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:23:00,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:23:00,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +14: [2023-05-08 11:23:00,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +19: [2023-05-08 11:23:00,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 0: [2023-05-08 11:23:00,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +18: [2023-05-08 11:23:00,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +26: [2023-05-08 11:23:00,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:23:00,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +15: [2023-05-08 11:23:00,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +15: [2023-05-08 11:23:00,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:23:00,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:23:00,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +12: [2023-05-08 11:23:00,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +12: [2023-05-08 11:23:00,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:23:00,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +22: [2023-05-08 11:23:00,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +18: [2023-05-08 11:23:00,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 1: [2023-05-08 11:23:00,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +26: [2023-05-08 11:23:00,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:23:00,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:23:00,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +26: [2023-05-08 11:23:00,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:23:00,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:23:00,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:23:00,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:23:00,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +26: [2023-05-08 11:23:00,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:23:00,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +19: [2023-05-08 11:23:00,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:23:00,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +15: [2023-05-08 11:23:00,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:23:00,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:23:00,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:23:00,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 9: [2023-05-08 11:23:00,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:23:00,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +14: [2023-05-08 11:23:00,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:23:00,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:23:00,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:23:00,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +27: [2023-05-08 11:23:00,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +27: [2023-05-08 11:23:00,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 0: [2023-05-08 11:23:00,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +29: [2023-05-08 11:23:00,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +29: [2023-05-08 11:23:00,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +29: [2023-05-08 11:23:00,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +15: [2023-05-08 11:23:00,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +29: [2023-05-08 11:23:00,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +18: [2023-05-08 11:23:00,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +18: [2023-05-08 11:23:00,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +18: [2023-05-08 11:23:00,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +27: [2023-05-08 11:23:00,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 5: [2023-05-08 11:23:00,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:23:00,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:23:00,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:23:00,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +18: [2023-05-08 11:23:00,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +27: [2023-05-08 11:23:00,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +26: [2023-05-08 11:23:00,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 1: [2023-05-08 11:23:00,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +19: [2023-05-08 11:23:00,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:23:00,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 4: [2023-05-08 11:23:00,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +19: [2023-05-08 11:23:00,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +27: [2023-05-08 11:23:00,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +27: [2023-05-08 11:23:00,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:23:00,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +26: [2023-05-08 11:23:00,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:23:00,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +15: [2023-05-08 11:23:00,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:23:00,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +26: [2023-05-08 11:23:00,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +26: [2023-05-08 11:23:00,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +29: [2023-05-08 11:23:00,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:23:00,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:23:00,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +22: [2023-05-08 11:23:00,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +22: [2023-05-08 11:23:00,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +22: [2023-05-08 11:23:00,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +22: [2023-05-08 11:23:00,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +29: [2023-05-08 11:23:00,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 3: [2023-05-08 11:23:00,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 1: [2023-05-08 11:23:00,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +15: [2023-05-08 11:23:00,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +22: [2023-05-08 11:23:00,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +15: [2023-05-08 11:23:00,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +22: [2023-05-08 11:23:00,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +22: [2023-05-08 11:23:00,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +22: [2023-05-08 11:23:00,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +26: [2023-05-08 11:23:00,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:23:00,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:23:00,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 1: [2023-05-08 11:23:00,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 5: [2023-05-08 11:23:00,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +27: [2023-05-08 11:23:00,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 4: [2023-05-08 11:23:00,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +15: [2023-05-08 11:23:00,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +27: [2023-05-08 11:23:00,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:23:00,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:23:00,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +18: [2023-05-08 11:23:00,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +18: [2023-05-08 11:23:00,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +18: [2023-05-08 11:23:00,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +26: [2023-05-08 11:23:00,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +26: [2023-05-08 11:23:00,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +21: [2023-05-08 11:23:00,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +21: [2023-05-08 11:23:00,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +21: [2023-05-08 11:23:00,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +21: [2023-05-08 11:23:00,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +26: [2023-05-08 11:23:00,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +18: [2023-05-08 11:23:00,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +18: [2023-05-08 11:23:00,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 1: [2023-05-08 11:23:00,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 1: [2023-05-08 11:23:00,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +27: [2023-05-08 11:23:00,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:23:00,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 5: [2023-05-08 11:23:00,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +26: [2023-05-08 11:23:00,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 3: [2023-05-08 11:23:00,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:23:00,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +18: [2023-05-08 11:23:00,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 5: [2023-05-08 11:23:00,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:23:00,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 5: [2023-05-08 11:23:00,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:23:00,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +17: [2023-05-08 11:23:00,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 3: [2023-05-08 11:23:00,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +27: [2023-05-08 11:23:00,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:23:00,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +15: [2023-05-08 11:23:00,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:23:00,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +15: [2023-05-08 11:23:00,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +21: [2023-05-08 11:23:00,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +17: [2023-05-08 11:23:00,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +21: [2023-05-08 11:23:00,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:23:00,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +27: [2023-05-08 11:23:00,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:23:00,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 5: [2023-05-08 11:23:00,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:23:00,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +27: [2023-05-08 11:23:00,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:23:00,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +17: [2023-05-08 11:23:00,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:23:00,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:23:00,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +21: [2023-05-08 11:23:00,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:23:00,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:23:00,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:23:00,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:23:00,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +21: [2023-05-08 11:23:00,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +21: [2023-05-08 11:23:00,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +21: [2023-05-08 11:23:00,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +30: [2023-05-08 11:23:00,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +30: [2023-05-08 11:23:00,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +14: [2023-05-08 11:23:00,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +14: [2023-05-08 11:23:00,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +14: [2023-05-08 11:23:00,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +14: [2023-05-08 11:23:00,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +30: [2023-05-08 11:23:00,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +30: [2023-05-08 11:23:00,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +30: [2023-05-08 11:23:00,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:23:00,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:23:00,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +17: [2023-05-08 11:23:00,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +17: [2023-05-08 11:23:00,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +17: [2023-05-08 11:23:00,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +17: [2023-05-08 11:23:00,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +17: [2023-05-08 11:23:00,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +17: [2023-05-08 11:23:00,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +17: [2023-05-08 11:23:00,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +17: [2023-05-08 11:23:00,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:23:00,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:23:00,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:23:00,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +17: [2023-05-08 11:23:00,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +17: [2023-05-08 11:23:00,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +17: [2023-05-08 11:23:00,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +16: [2023-05-08 11:23:00,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +16: [2023-05-08 11:23:00,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +16: [2023-05-08 11:23:00,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +30: [2023-05-08 11:23:00,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +25: [2023-05-08 11:23:00,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +25: [2023-05-08 11:23:00,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +25: [2023-05-08 11:23:00,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +17: [2023-05-08 11:23:00,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:23:00,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:23:00,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:23:00,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 6: [2023-05-08 11:23:00,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +24: [2023-05-08 11:23:00,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +24: [2023-05-08 11:23:00,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 6: [2023-05-08 11:23:00,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 6: [2023-05-08 11:23:00,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 6: [2023-05-08 11:23:00,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +30: [2023-05-08 11:23:00,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +25: [2023-05-08 11:23:00,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 2: [2023-05-08 11:23:00,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 2: [2023-05-08 11:23:00,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 2: [2023-05-08 11:23:00,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +24: [2023-05-08 11:23:00,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +16: [2023-05-08 11:23:00,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +17: [2023-05-08 11:23:00,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +17: [2023-05-08 11:23:00,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +12: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +12: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. + 8: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +12: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +12: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +29: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +18: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +18: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +18: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +18: [2023-05-08 11:23:00,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:23:00,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:23:00,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt. +29: [2023-05-08 11:23:00,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +29: [2023-05-08 11:23:00,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 7: [2023-05-08 11:23:00,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:23:00,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 7: [2023-05-08 11:23:00,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +29: [2023-05-08 11:23:00,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +16: [2023-05-08 11:23:00,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +25: [2023-05-08 11:23:00,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +16: [2023-05-08 11:23:00,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:23:00,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +17: [2023-05-08 11:23:00,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:23:00,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +25: [2023-05-08 11:23:00,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:23:00,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +16: [2023-05-08 11:23:00,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +25: [2023-05-08 11:23:00,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +25: [2023-05-08 11:23:00,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:23:00,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:23:00,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +30: [2023-05-08 11:23:00,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +30: [2023-05-08 11:23:00,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 8: [2023-05-08 11:23:00,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +24: [2023-05-08 11:23:00,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:23:00,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:23:00,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +30: [2023-05-08 11:23:00,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +12: [2023-05-08 11:23:00,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:23:00,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +14: [2023-05-08 11:23:00,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +14: [2023-05-08 11:23:00,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +12: [2023-05-08 11:23:00,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:23:00,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:23:00,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:23:00,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +14: [2023-05-08 11:23:00,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +14: [2023-05-08 11:23:00,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 8: [2023-05-08 11:23:00,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:23:00,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +18: [2023-05-08 11:23:00,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +18: [2023-05-08 11:23:00,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:23:00,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:23:00,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +30: [2023-05-08 11:23:00,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt... +17: [2023-05-08 11:23:00,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:23:00,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 8: [2023-05-08 11:23:00,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:23:00,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:23:00,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +30: [2023-05-08 11:23:00,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +14: [2023-05-08 11:23:00,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:23:00,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +24: [2023-05-08 11:23:00,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +14: [2023-05-08 11:23:00,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +24: [2023-05-08 11:23:00,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 8: [2023-05-08 11:23:00,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +16: [2023-05-08 11:23:00,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +16: [2023-05-08 11:23:00,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +25: [2023-05-08 11:23:00,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +25: [2023-05-08 11:23:00,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +17: [2023-05-08 11:23:00,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:23:00,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +24: [2023-05-08 11:23:00,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +18: [2023-05-08 11:23:00,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +18: [2023-05-08 11:23:00,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +18: [2023-05-08 11:23:00,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:23:00,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +16: [2023-05-08 11:23:00,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +16: [2023-05-08 11:23:00,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +17: [2023-05-08 11:23:00,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:23:00,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +16: [2023-05-08 11:23:00,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +16: [2023-05-08 11:23:00,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +12: [2023-05-08 11:23:00,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 7: [2023-05-08 11:23:00,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:23:00,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:23:00,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +17: [2023-05-08 11:23:00,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +18: [2023-05-08 11:23:00,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +16: [2023-05-08 11:23:00,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 6: [2023-05-08 11:23:00,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 6: [2023-05-08 11:23:00,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +12: [2023-05-08 11:23:00,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +12: [2023-05-08 11:23:00,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +12: [2023-05-08 11:23:00,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +12: [2023-05-08 11:23:00,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +29: [2023-05-08 11:23:00,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +16: [2023-05-08 11:23:00,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +25: [2023-05-08 11:23:00,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:23:00,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:23:00,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +12: [2023-05-08 11:23:00,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +25: [2023-05-08 11:23:00,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +18: [2023-05-08 11:23:00,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:23:00,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +17: [2023-05-08 11:23:00,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +31: [2023-05-08 11:23:00,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +31: [2023-05-08 11:23:00,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +31: [2023-05-08 11:23:00,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 2: [2023-05-08 11:23:00,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +25: [2023-05-08 11:23:00,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +24: [2023-05-08 11:23:00,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +24: [2023-05-08 11:23:00,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. +24: [2023-05-08 11:23:00,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +22: [2023-05-08 11:23:00,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +22: [2023-05-08 11:23:00,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +22: [2023-05-08 11:23:00,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:23:00,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:23:00,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 8: [2023-05-08 11:23:00,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 2: [2023-05-08 11:23:00,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... +25: [2023-05-08 11:23:00,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 2: [2023-05-08 11:23:00,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:23:00,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +22: [2023-05-08 11:23:00,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:23:00,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:23:00,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt... + 8: [2023-05-08 11:23:00,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:23:00,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 8: [2023-05-08 11:23:00,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:23:00,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +31: [2023-05-08 11:23:00,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +31: [2023-05-08 11:23:00,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 8: [2023-05-08 11:23:00,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +22: [2023-05-08 11:23:00,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +22: [2023-05-08 11:23:00,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:23:00,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +22: [2023-05-08 11:23:00,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:23:00,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:23:00,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:23:00,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:23:00,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:23:00,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:23:00,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +31: [2023-05-08 11:23:00,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +31: [2023-05-08 11:23:00,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +22: [2023-05-08 11:23:00,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:23:00,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:23:00,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:23:00,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:23:00,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +11: [2023-05-08 11:23:00,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +22: [2023-05-08 11:23:00,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:23:00,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:23:00,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:23:00,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:23:00,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:23:00,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:23:00,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:23:00,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:23:00,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:23:00,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +31: [2023-05-08 11:23:00,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +11: [2023-05-08 11:23:00,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 3: [2023-05-08 11:23:00,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 3: [2023-05-08 11:23:00,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +24: [2023-05-08 11:23:00,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 3: [2023-05-08 11:23:00,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +11: [2023-05-08 11:23:00,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:23:00,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:23:00,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:23:00,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:23:00,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:23:00,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:23:00,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:23:00,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +21: [2023-05-08 11:23:00,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +21: [2023-05-08 11:23:00,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +21: [2023-05-08 11:23:00,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +21: [2023-05-08 11:23:00,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +21: [2023-05-08 11:23:00,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:23:00,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +21: [2023-05-08 11:23:00,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +21: [2023-05-08 11:23:00,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +21: [2023-05-08 11:23:00,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:23:00,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:23:00,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:23:00,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:23:00,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 3: [2023-05-08 11:23:00,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:23:00,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 9: [2023-05-08 11:23:00,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 9: [2023-05-08 11:23:00,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 9: [2023-05-08 11:23:00,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 3: [2023-05-08 11:23:00,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +21: [2023-05-08 11:23:00,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:23:00,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:23:00,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +21: [2023-05-08 11:23:00,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +23: [2023-05-08 11:23:00,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +21: [2023-05-08 11:23:00,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +23: [2023-05-08 11:23:00,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +23: [2023-05-08 11:23:00,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +23: [2023-05-08 11:23:00,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +24: [2023-05-08 11:23:00,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:23:00,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +31: [2023-05-08 11:23:00,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +31: [2023-05-08 11:23:00,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:23:00,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:23:00,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +31: [2023-05-08 11:23:00,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:23:00,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +21: [2023-05-08 11:23:00,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +21: [2023-05-08 11:23:00,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:23:00,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +31: [2023-05-08 11:23:00,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:23:00,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +21: [2023-05-08 11:23:00,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:23:00,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:23:00,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:23:00,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:23:00,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:23:00,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +21: [2023-05-08 11:23:00,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:23:00,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:23:00,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:23:00,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:23:00,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:23:00,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:23:00,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:23:00,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:23:00,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:23:00,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +30: [2023-05-08 11:23:00,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:23:00,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:23:00,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +20: [2023-05-08 11:23:00,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:23:00,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +30: [2023-05-08 11:23:00,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:23:00,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:23:00,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:23:00,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:23:00,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:23:00,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +21: [2023-05-08 11:23:00,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:23:00,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:23:00,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:23:00,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:23:00,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +30: [2023-05-08 11:23:00,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:23:00,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +30: [2023-05-08 11:23:00,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +20: [2023-05-08 11:23:00,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 9: [2023-05-08 11:23:00,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 9: [2023-05-08 11:23:00,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +31: [2023-05-08 11:23:00,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:23:00,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:23:00,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +19: [2023-05-08 11:23:00,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +19: [2023-05-08 11:23:00,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +23: [2023-05-08 11:23:00,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +19: [2023-05-08 11:23:00,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 9: [2023-05-08 11:23:00,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 4: [2023-05-08 11:23:00,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:23:00,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +10: [2023-05-08 11:23:00,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +10: [2023-05-08 11:23:00,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +10: [2023-05-08 11:23:00,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +25: [2023-05-08 11:23:00,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:23:00,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:23:00,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +23: [2023-05-08 11:23:00,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +23: [2023-05-08 11:23:00,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +31: [2023-05-08 11:23:00,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:23:00,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:23:00,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +19: [2023-05-08 11:23:00,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:23:00,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:23:00,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:23:00,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:23:00,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:23:00,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:23:00,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:23:00,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:23:00,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +19: [2023-05-08 11:23:00,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +20: [2023-05-08 11:23:00,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +20: [2023-05-08 11:23:00,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +19: [2023-05-08 11:23:00,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +19: [2023-05-08 11:23:00,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +30: [2023-05-08 11:23:00,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:23:00,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +10: [2023-05-08 11:23:00,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +25: [2023-05-08 11:23:00,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +10: [2023-05-08 11:23:00,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:23:00,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:23:00,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +10: [2023-05-08 11:23:00,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +20: [2023-05-08 11:23:00,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +10: [2023-05-08 11:23:00,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +23: [2023-05-08 11:23:00,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:23:00,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:23:00,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:23:00,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:23:00,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:23:00,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:23:00,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 9: [2023-05-08 11:23:00,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +31: [2023-05-08 11:23:00,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +28: [2023-05-08 11:23:00,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +28: [2023-05-08 11:23:00,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +16: [2023-05-08 11:23:00,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +20: [2023-05-08 11:23:00,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:23:00,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +16: [2023-05-08 11:23:00,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +16: [2023-05-08 11:23:00,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +16: [2023-05-08 11:23:00,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +28: [2023-05-08 11:23:00,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +16: [2023-05-08 11:23:00,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +28: [2023-05-08 11:23:00,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +30: [2023-05-08 11:23:00,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +16: [2023-05-08 11:23:00,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +16: [2023-05-08 11:23:00,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:23:00,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +12: [2023-05-08 11:23:00,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +12: [2023-05-08 11:23:00,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +12: [2023-05-08 11:23:00,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +12: [2023-05-08 11:23:00,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +12: [2023-05-08 11:23:00,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:23:00,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +23: [2023-05-08 11:23:00,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:23:00,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:23:00,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:23:00,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:23:00,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:23:00,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:23:00,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +25: [2023-05-08 11:23:00,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:23:00,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:23:00,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:23:00,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +20: [2023-05-08 11:23:00,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +20: [2023-05-08 11:23:00,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +20: [2023-05-08 11:23:00,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +13: [2023-05-08 11:23:00,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +13: [2023-05-08 11:23:00,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 9: [2023-05-08 11:23:00,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:23:00,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +28: [2023-05-08 11:23:00,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +13: [2023-05-08 11:23:00,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +20: [2023-05-08 11:23:00,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:23:00,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +28: [2023-05-08 11:23:00,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:23:00,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +30: [2023-05-08 11:23:00,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:23:00,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +20: [2023-05-08 11:23:00,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:23:00,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +16: [2023-05-08 11:23:00,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:23:00,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:23:00,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:23:00,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +26: [2023-05-08 11:23:00,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +30: [2023-05-08 11:23:00,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:23:00,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:23:00,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:23:00,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 7: [2023-05-08 11:23:00,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +12: [2023-05-08 11:23:00,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:23:00,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +16: [2023-05-08 11:23:00,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:23:00,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +26: [2023-05-08 11:23:00,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +13: [2023-05-08 11:23:00,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +30: [2023-05-08 11:23:00,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:23:00,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +13: [2023-05-08 11:23:00,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +26: [2023-05-08 11:23:00,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +12: [2023-05-08 11:23:00,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +12: [2023-05-08 11:23:00,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +12: [2023-05-08 11:23:00,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:23:00,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +20: [2023-05-08 11:23:00,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:23:00,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:23:00,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +20: [2023-05-08 11:23:00,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:23:00,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:23:00,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +20: [2023-05-08 11:23:00,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +20: [2023-05-08 11:23:00,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +26: [2023-05-08 11:23:00,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +20: [2023-05-08 11:23:00,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 7: [2023-05-08 11:23:00,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +26: [2023-05-08 11:23:00,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:23:00,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:23:00,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:23:00,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:23:00,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +26: [2023-05-08 11:23:00,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +26: [2023-05-08 11:23:00,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +26: [2023-05-08 11:23:00,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +26: [2023-05-08 11:23:00,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:23:00,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +16: [2023-05-08 11:23:00,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:23:00,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:23:00,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +16: [2023-05-08 11:23:00,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:23:00,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:23:00,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:23:00,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:23:00,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +12: [2023-05-08 11:23:00,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +12: [2023-05-08 11:23:00,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:23:00,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:23:00,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:23:00,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:23:00,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +13: [2023-05-08 11:23:00,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +13: [2023-05-08 11:23:00,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:23:00,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:23:00,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +12: [2023-05-08 11:23:00,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:23:00,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:23:00,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +13: [2023-05-08 11:23:00,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +26: [2023-05-08 11:23:00,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 1: [2023-05-08 11:23:00,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +13: [2023-05-08 11:23:00,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +13: [2023-05-08 11:23:00,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 1: [2023-05-08 11:23:00,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +13: [2023-05-08 11:23:00,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +13: [2023-05-08 11:23:00,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +13: [2023-05-08 11:23:00,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +13: [2023-05-08 11:23:00,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +13: [2023-05-08 11:23:00,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +13: [2023-05-08 11:23:00,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +12: [2023-05-08 11:23:00,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +28: [2023-05-08 11:23:00,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +28: [2023-05-08 11:23:00,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +28: [2023-05-08 11:23:00,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 0: [2023-05-08 11:23:00,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 7: [2023-05-08 11:23:00,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +28: [2023-05-08 11:23:00,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +28: [2023-05-08 11:23:00,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +28: [2023-05-08 11:23:00,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +28: [2023-05-08 11:23:00,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +28: [2023-05-08 11:23:00,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:23:00,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +28: [2023-05-08 11:23:00,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:23:00,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +28: [2023-05-08 11:23:00,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +28: [2023-05-08 11:23:00,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:23:00,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +28: [2023-05-08 11:23:00,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:23:00,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 0: [2023-05-08 11:23:00,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +15: [2023-05-08 11:23:00,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:23:00,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:23:00,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:23:00,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:23:00,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:23:00,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:23:00,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:23:00,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +27: [2023-05-08 11:23:00,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +27: [2023-05-08 11:23:00,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +27: [2023-05-08 11:23:00,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +27: [2023-05-08 11:23:00,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +27: [2023-05-08 11:23:00,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +27: [2023-05-08 11:23:00,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:23:00,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +27: [2023-05-08 11:23:00,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 4: [2023-05-08 11:23:00,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 4: [2023-05-08 11:23:00,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 4: [2023-05-08 11:23:00,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 0: [2023-05-08 11:23:00,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +27: [2023-05-08 11:23:00,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:23:00,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +13: [2023-05-08 11:23:00,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +13: [2023-05-08 11:23:00,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +13: [2023-05-08 11:23:00,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +13: [2023-05-08 11:23:00,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 1: [2023-05-08 11:23:00,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +13: [2023-05-08 11:23:00,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:23:00,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +26: [2023-05-08 11:23:00,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 0: [2023-05-08 11:23:00,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:23:00,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:23:00,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:23:00,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:23:00,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:23:00,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +26: [2023-05-08 11:23:00,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:23:00,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 0: [2023-05-08 11:23:00,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:23:00,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:23:00,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +27: [2023-05-08 11:23:00,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:23:00,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +18: [2023-05-08 11:23:00,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:23:00,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:23:00,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +18: [2023-05-08 11:23:00,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +15: [2023-05-08 11:23:00,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:23:00,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +18: [2023-05-08 11:23:00,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:23:00,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:23:00,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +27: [2023-05-08 11:23:00,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +27: [2023-05-08 11:23:00,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +18: [2023-05-08 11:23:00,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +28: [2023-05-08 11:23:00,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:23:00,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:23:00,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +28: [2023-05-08 11:23:00,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:23:00,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +13: [2023-05-08 11:23:00,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:23:00,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +26: [2023-05-08 11:23:00,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +28: [2023-05-08 11:23:00,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:23:00,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:23:00,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:23:00,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:23:00,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:23:00,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +10: [2023-05-08 11:23:00,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +10: [2023-05-08 11:23:00,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +10: [2023-05-08 11:23:00,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:23:00,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +10: [2023-05-08 11:23:00,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:23:00,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +10: [2023-05-08 11:23:00,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +10: [2023-05-08 11:23:00,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:23:00,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +10: [2023-05-08 11:23:00,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +10: [2023-05-08 11:23:00,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +10: [2023-05-08 11:23:00,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:23:00,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:23:00,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:23:00,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +10: [2023-05-08 11:23:00,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:23:00,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +10: [2023-05-08 11:23:00,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +10: [2023-05-08 11:23:00,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +27: [2023-05-08 11:23:00,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +27: [2023-05-08 11:23:00,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +18: [2023-05-08 11:23:00,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:23:00,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +28: [2023-05-08 11:23:00,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:23:00,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +27: [2023-05-08 11:23:00,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 1: [2023-05-08 11:23:00,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +27: [2023-05-08 11:23:00,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +13: [2023-05-08 11:23:00,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +19: [2023-05-08 11:23:00,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:23:00,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:23:00,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +19: [2023-05-08 11:23:00,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +19: [2023-05-08 11:23:00,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +19: [2023-05-08 11:23:00,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +19: [2023-05-08 11:23:00,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 1: [2023-05-08 11:23:00,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:23:00,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +24: [2023-05-08 11:23:00,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +19: [2023-05-08 11:23:00,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +18: [2023-05-08 11:23:00,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +24: [2023-05-08 11:23:00,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +24: [2023-05-08 11:23:00,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +19: [2023-05-08 11:23:00,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +19: [2023-05-08 11:23:00,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +19: [2023-05-08 11:23:00,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +19: [2023-05-08 11:23:00,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +13: [2023-05-08 11:23:00,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:23:00,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:23:00,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:23:00,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +12: [2023-05-08 11:23:00,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +12: [2023-05-08 11:23:00,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +12: [2023-05-08 11:23:00,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +24: [2023-05-08 11:23:00,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +28: [2023-05-08 11:23:00,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:23:00,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:23:00,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:23:00,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 5: [2023-05-08 11:23:00,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +19: [2023-05-08 11:23:00,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +19: [2023-05-08 11:23:00,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:23:00,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +29: [2023-05-08 11:23:00,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +29: [2023-05-08 11:23:00,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:23:00,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:23:00,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +29: [2023-05-08 11:23:00,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:23:00,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:23:00,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:23:00,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +18: [2023-05-08 11:23:00,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:23:00,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 5: [2023-05-08 11:23:00,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 0: [2023-05-08 11:23:00,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +11: [2023-05-08 11:23:00,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 5: [2023-05-08 11:23:00,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +10: [2023-05-08 11:23:00,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +27: [2023-05-08 11:23:00,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:23:00,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +24: [2023-05-08 11:23:00,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:23:00,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +22: [2023-05-08 11:23:00,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:23:00,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:23:00,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:23:00,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:23:00,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:23:00,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:23:00,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:23:00,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:23:00,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +17: [2023-05-08 11:23:00,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:23:00,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:23:00,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:23:00,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:23:00,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:23:00,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +18: [2023-05-08 11:23:00,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:23:00,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +22: [2023-05-08 11:23:00,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:23:00,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:23:00,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +30: [2023-05-08 11:23:00,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +18: [2023-05-08 11:23:00,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:23:00,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +30: [2023-05-08 11:23:00,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +30: [2023-05-08 11:23:00,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +14: [2023-05-08 11:23:00,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +14: [2023-05-08 11:23:00,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +14: [2023-05-08 11:23:00,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:23:00,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +10: [2023-05-08 11:23:00,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +14: [2023-05-08 11:23:00,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +19: [2023-05-08 11:23:00,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +14: [2023-05-08 11:23:00,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:23:00,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:23:00,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +14: [2023-05-08 11:23:00,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:23:00,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +14: [2023-05-08 11:23:00,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:23:00,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +12: [2023-05-08 11:23:00,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:23:00,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +29: [2023-05-08 11:23:00,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 0: [2023-05-08 11:23:00,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 3: [2023-05-08 11:23:00,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:23:00,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +22: [2023-05-08 11:23:00,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:23:00,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +30: [2023-05-08 11:23:00,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +30: [2023-05-08 11:23:00,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +19: [2023-05-08 11:23:00,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:23:00,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +22: [2023-05-08 11:23:00,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:23:00,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +17: [2023-05-08 11:23:00,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:23:00,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:23:00,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +18: [2023-05-08 11:23:00,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +15: [2023-05-08 11:23:00,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +15: [2023-05-08 11:23:00,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +30: [2023-05-08 11:23:00,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:23:00,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +15: [2023-05-08 11:23:00,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +10: [2023-05-08 11:23:00,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:23:00,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +22: [2023-05-08 11:23:00,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:23:00,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +24: [2023-05-08 11:23:00,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:23:00,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:23:00,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +24: [2023-05-08 11:23:00,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 6: [2023-05-08 11:23:00,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +17: [2023-05-08 11:23:00,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:23:00,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:23:00,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 6: [2023-05-08 11:23:00,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:23:00,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:23:00,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +24: [2023-05-08 11:23:00,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:23:00,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 8: [2023-05-08 11:23:00,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:23:00,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 5: [2023-05-08 11:23:00,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +29: [2023-05-08 11:23:00,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:23:00,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +14: [2023-05-08 11:23:00,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +14: [2023-05-08 11:23:00,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +22: [2023-05-08 11:23:00,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:23:00,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +30: [2023-05-08 11:23:00,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +19: [2023-05-08 11:23:00,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +12: [2023-05-08 11:23:00,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +12: [2023-05-08 11:23:00,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +22: [2023-05-08 11:23:00,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:23:00,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +24: [2023-05-08 11:23:00,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +17: [2023-05-08 11:23:00,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:23:00,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 5: [2023-05-08 11:23:00,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +15: [2023-05-08 11:23:00,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:23:00,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:23:00,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +17: [2023-05-08 11:23:00,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +15: [2023-05-08 11:23:00,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:23:00,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:23:00,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +17: [2023-05-08 11:23:00,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:23:00,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +30: [2023-05-08 11:23:00,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +17: [2023-05-08 11:23:00,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +19: [2023-05-08 11:23:00,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +14: [2023-05-08 11:23:00,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +29: [2023-05-08 11:23:00,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:23:00,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:23:00,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +14: [2023-05-08 11:23:00,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:23:00,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:23:00,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +12: [2023-05-08 11:23:00,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 7: [2023-05-08 11:23:00,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 7: [2023-05-08 11:23:00,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:23:00,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:23:00,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +30: [2023-05-08 11:23:00,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 5: [2023-05-08 11:23:00,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 4: [2023-05-08 11:23:00,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:23:00,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +19: [2023-05-08 11:23:00,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:23:00,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:23:00,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:23:00,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:23:00,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 7: [2023-05-08 11:23:00,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:23:00,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 5: [2023-05-08 11:23:00,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +12: [2023-05-08 11:23:00,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 8: [2023-05-08 11:23:00,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:23:00,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:23:01,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +18: [2023-05-08 11:23:01,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +18: [2023-05-08 11:23:01,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +18: [2023-05-08 11:23:01,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +17: [2023-05-08 11:23:01,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +17: [2023-05-08 11:23:01,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +14: [2023-05-08 11:23:01,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:23:01,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +17: [2023-05-08 11:23:01,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +22: [2023-05-08 11:23:01,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +17: [2023-05-08 11:23:01,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 4: [2023-05-08 11:23:01,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +15: [2023-05-08 11:23:01,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:23:01,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +11: [2023-05-08 11:23:01,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +11: [2023-05-08 11:23:01,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +11: [2023-05-08 11:23:01,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 3: [2023-05-08 11:23:01,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:23:01,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 4: [2023-05-08 11:23:01,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +15: [2023-05-08 11:23:01,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:23:01,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:23:01,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:23:01,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +31: [2023-05-08 11:23:01,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +17: [2023-05-08 11:23:01,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:23:01,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +31: [2023-05-08 11:23:01,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +17: [2023-05-08 11:23:01,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:23:01,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +18: [2023-05-08 11:23:01,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:23:01,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +18: [2023-05-08 11:23:01,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +15: [2023-05-08 11:23:01,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +17: [2023-05-08 11:23:01,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 1: [2023-05-08 11:23:01,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 4: [2023-05-08 11:23:01,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:23:01,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +17: [2023-05-08 11:23:01,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +21: [2023-05-08 11:23:01,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:23:01,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:23:01,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:23:01,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +21: [2023-05-08 11:23:01,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +21: [2023-05-08 11:23:01,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +21: [2023-05-08 11:23:01,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:23:01,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +21: [2023-05-08 11:23:01,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:23:01,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +25: [2023-05-08 11:23:01,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +25: [2023-05-08 11:23:01,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +25: [2023-05-08 11:23:01,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +11: [2023-05-08 11:23:01,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +15: [2023-05-08 11:23:01,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +21: [2023-05-08 11:23:01,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +15: [2023-05-08 11:23:01,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +27: [2023-05-08 11:23:01,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +27: [2023-05-08 11:23:01,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +27: [2023-05-08 11:23:01,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +27: [2023-05-08 11:23:01,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +31: [2023-05-08 11:23:01,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:23:01,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +18: [2023-05-08 11:23:01,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:23:01,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:23:01,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +18: [2023-05-08 11:23:01,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 6: [2023-05-08 11:23:01,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 6: [2023-05-08 11:23:01,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 6: [2023-05-08 11:23:01,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 1: [2023-05-08 11:23:01,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:23:01,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +14: [2023-05-08 11:23:01,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 1: [2023-05-08 11:23:01,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:23:01,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 1: [2023-05-08 11:23:01,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 2: [2023-05-08 11:23:01,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 2: [2023-05-08 11:23:01,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 2: [2023-05-08 11:23:01,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +25: [2023-05-08 11:23:01,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:23:01,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +16: [2023-05-08 11:23:01,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +16: [2023-05-08 11:23:01,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +18: [2023-05-08 11:23:01,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +18: [2023-05-08 11:23:01,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +25: [2023-05-08 11:23:01,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:23:01,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +16: [2023-05-08 11:23:01,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +22: [2023-05-08 11:23:01,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +22: [2023-05-08 11:23:01,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +22: [2023-05-08 11:23:01,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +22: [2023-05-08 11:23:01,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +25: [2023-05-08 11:23:01,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:23:01,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +29: [2023-05-08 11:23:01,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +29: [2023-05-08 11:23:01,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +21: [2023-05-08 11:23:01,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:23:01,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:23:01,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:23:01,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:23:01,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:23:01,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +17: [2023-05-08 11:23:01,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +11: [2023-05-08 11:23:01,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 9: [2023-05-08 11:23:01,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 9: [2023-05-08 11:23:01,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +27: [2023-05-08 11:23:01,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +27: [2023-05-08 11:23:01,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +29: [2023-05-08 11:23:01,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +27: [2023-05-08 11:23:01,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +21: [2023-05-08 11:23:01,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:23:01,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:23:01,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +21: [2023-05-08 11:23:01,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:23:01,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:23:01,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +14: [2023-05-08 11:23:01,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:23:01,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:23:01,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:23:01,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:23:01,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:23:01,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:23:01,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +27: [2023-05-08 11:23:01,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:23:01,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:23:01,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:23:01,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +14: [2023-05-08 11:23:01,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:23:01,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +21: [2023-05-08 11:23:01,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:23:01,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:23:01,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:23:01,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:23:01,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:23:01,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:23:01,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 1: [2023-05-08 11:23:01,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +22: [2023-05-08 11:23:01,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 1: [2023-05-08 11:23:01,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +21: [2023-05-08 11:23:01,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +25: [2023-05-08 11:23:01,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:23:01,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:23:01,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +21: [2023-05-08 11:23:01,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. +29: [2023-05-08 11:23:01,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:23:01,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:23:01,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:23:01,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +31: [2023-05-08 11:23:01,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +31: [2023-05-08 11:23:01,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +31: [2023-05-08 11:23:01,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 1: [2023-05-08 11:23:01,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +11: [2023-05-08 11:23:01,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +25: [2023-05-08 11:23:01,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +25: [2023-05-08 11:23:01,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +27: [2023-05-08 11:23:01,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +27: [2023-05-08 11:23:01,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +17: [2023-05-08 11:23:01,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:23:01,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +21: [2023-05-08 11:23:01,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +21: [2023-05-08 11:23:01,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +17: [2023-05-08 11:23:01,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +21: [2023-05-08 11:23:01,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:23:01,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt. + 9: [2023-05-08 11:23:01,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +27: [2023-05-08 11:23:01,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:23:01,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +27: [2023-05-08 11:23:01,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +31: [2023-05-08 11:23:01,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +16: [2023-05-08 11:23:01,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:23:01,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 2: [2023-05-08 11:23:01,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 8: [2023-05-08 11:23:01,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +27: [2023-05-08 11:23:01,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +27: [2023-05-08 11:23:01,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +22: [2023-05-08 11:23:01,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:23:01,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +22: [2023-05-08 11:23:01,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +27: [2023-05-08 11:23:01,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +21: [2023-05-08 11:23:01,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +27: [2023-05-08 11:23:01,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +21: [2023-05-08 11:23:01,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +31: [2023-05-08 11:23:01,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +16: [2023-05-08 11:23:01,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:23:01,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +21: [2023-05-08 11:23:01,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:23:01,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +14: [2023-05-08 11:23:01,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +25: [2023-05-08 11:23:01,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:23:01,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:23:01,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 2: [2023-05-08 11:23:01,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 2: [2023-05-08 11:23:01,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +21: [2023-05-08 11:23:01,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +25: [2023-05-08 11:23:01,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +21: [2023-05-08 11:23:01,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:23:01,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:23:01,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +14: [2023-05-08 11:23:01,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +29: [2023-05-08 11:23:01,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +16: [2023-05-08 11:23:01,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:23:01,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 6: [2023-05-08 11:23:01,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +25: [2023-05-08 11:23:01,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +22: [2023-05-08 11:23:01,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +14: [2023-05-08 11:23:01,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +14: [2023-05-08 11:23:01,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +22: [2023-05-08 11:23:01,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +25: [2023-05-08 11:23:01,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:23:01,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +29: [2023-05-08 11:23:01,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +29: [2023-05-08 11:23:01,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +29: [2023-05-08 11:23:01,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +21: [2023-05-08 11:23:01,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:23:01,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +24: [2023-05-08 11:23:01,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:23:01,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:23:01,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +24: [2023-05-08 11:23:01,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:23:01,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +24: [2023-05-08 11:23:01,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +29: [2023-05-08 11:23:01,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +14: [2023-05-08 11:23:01,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:23:01,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:23:01,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:23:01,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +21: [2023-05-08 11:23:01,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +14: [2023-05-08 11:23:01,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 6: [2023-05-08 11:23:01,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +24: [2023-05-08 11:23:01,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:23:01,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +22: [2023-05-08 11:23:01,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +21: [2023-05-08 11:23:01,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +21: [2023-05-08 11:23:01,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 6: [2023-05-08 11:23:01,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +22: [2023-05-08 11:23:01,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +22: [2023-05-08 11:23:01,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 6: [2023-05-08 11:23:01,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +21: [2023-05-08 11:23:01,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:23:01,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:23:01,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt... +24: [2023-05-08 11:23:01,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +24: [2023-05-08 11:23:01,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 2: [2023-05-08 11:23:01,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 3: [2023-05-08 11:23:01,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 3: [2023-05-08 11:23:01,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 3: [2023-05-08 11:23:01,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +24: [2023-05-08 11:23:01,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +24: [2023-05-08 11:23:01,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +24: [2023-05-08 11:23:01,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +24: [2023-05-08 11:23:01,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +24: [2023-05-08 11:23:01,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 3: [2023-05-08 11:23:01,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 2: [2023-05-08 11:23:01,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +30: [2023-05-08 11:23:01,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +30: [2023-05-08 11:23:01,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:23:01,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +30: [2023-05-08 11:23:01,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +30: [2023-05-08 11:23:01,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:23:01,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:23:01,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +30: [2023-05-08 11:23:01,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +30: [2023-05-08 11:23:01,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +30: [2023-05-08 11:23:01,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +30: [2023-05-08 11:23:01,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +30: [2023-05-08 11:23:01,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +30: [2023-05-08 11:23:01,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +30: [2023-05-08 11:23:01,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +30: [2023-05-08 11:23:01,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +30: [2023-05-08 11:23:01,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:23:01,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:23:01,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:23:01,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:23:01,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 7: [2023-05-08 11:23:01,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 7: [2023-05-08 11:23:01,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 7: [2023-05-08 11:23:01,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 9: [2023-05-08 11:23:01,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:23:01,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:23:01,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:23:01,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:23:01,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 9: [2023-05-08 11:23:01,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 9: [2023-05-08 11:23:01,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 9: [2023-05-08 11:23:01,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 9: [2023-05-08 11:23:01,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 4: [2023-05-08 11:23:01,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +24: [2023-05-08 11:23:01,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 4: [2023-05-08 11:23:01,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 4: [2023-05-08 11:23:01,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +24: [2023-05-08 11:23:01,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +24: [2023-05-08 11:23:01,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +16: [2023-05-08 11:23:01,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:23:01,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +24: [2023-05-08 11:23:01,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +16: [2023-05-08 11:23:01,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:23:01,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:23:01,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:23:01,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +16: [2023-05-08 11:23:01,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +16: [2023-05-08 11:23:01,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:23:01,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:23:01,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:23:01,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:23:01,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:23:01,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:23:01,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:23:01,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:23:01,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:23:01,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:23:01,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:23:01,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +25: [2023-05-08 11:23:01,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:23:01,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +25: [2023-05-08 11:23:01,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +25: [2023-05-08 11:23:01,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:23:01,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:23:01,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +16: [2023-05-08 11:23:01,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:23:01,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:23:01,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +23: [2023-05-08 11:23:01,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +23: [2023-05-08 11:23:01,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +23: [2023-05-08 11:23:01,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:23:01,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +23: [2023-05-08 11:23:01,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:23:01,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:23:01,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +24: [2023-05-08 11:23:01,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +24: [2023-05-08 11:23:01,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 1: [2023-05-08 11:23:01,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:23:01,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:23:01,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:23:01,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:23:01,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +24: [2023-05-08 11:23:01,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +24: [2023-05-08 11:23:01,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 1: [2023-05-08 11:23:01,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:23:01,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:23:01,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +20: [2023-05-08 11:23:01,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:23:01,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +20: [2023-05-08 11:23:01,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:23:01,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:23:01,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:23:01,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:23:01,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 7: [2023-05-08 11:23:01,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +20: [2023-05-08 11:23:01,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +20: [2023-05-08 11:23:01,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:23:01,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:23:01,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +20: [2023-05-08 11:23:01,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:23:01,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:23:01,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:23:01,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:23:01,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +23: [2023-05-08 11:23:01,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +23: [2023-05-08 11:23:01,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +23: [2023-05-08 11:23:01,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +16: [2023-05-08 11:23:01,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +16: [2023-05-08 11:23:01,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 9: [2023-05-08 11:23:01,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +25: [2023-05-08 11:23:01,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:23:01,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:23:01,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:23:01,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +16: [2023-05-08 11:23:01,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:23:01,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:23:01,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:23:01,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:23:01,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:23:01,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:23:01,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:23:01,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:23:01,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +24: [2023-05-08 11:23:01,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +24: [2023-05-08 11:23:01,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:23:01,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:23:01,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +24: [2023-05-08 11:23:01,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +24: [2023-05-08 11:23:01,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +23: [2023-05-08 11:23:01,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 7: [2023-05-08 11:23:01,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +23: [2023-05-08 11:23:01,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:23:01,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:23:01,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:23:01,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:23:01,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:23:01,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:23:01,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +12: [2023-05-08 11:23:01,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +25: [2023-05-08 11:23:01,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:23:01,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:23:01,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +25: [2023-05-08 11:23:01,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:23:01,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:23:01,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:23:01,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:23:01,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:23:01,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:23:01,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:23:01,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:23:01,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +18: [2023-05-08 11:23:01,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +20: [2023-05-08 11:23:01,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:23:01,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:23:01,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:23:01,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +23: [2023-05-08 11:23:01,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:23:01,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:23:01,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +20: [2023-05-08 11:23:01,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:23:01,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:23:01,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:23:01,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:23:01,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:23:01,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:23:01,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:23:01,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:23:01,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:23:01,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:23:01,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:23:01,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:23:01,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +18: [2023-05-08 11:23:01,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:23:01,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:23:01,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:23:01,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:23:01,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +31: [2023-05-08 11:23:01,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +31: [2023-05-08 11:23:01,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +31: [2023-05-08 11:23:01,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:23:01,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:23:01,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:23:01,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:23:01,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:23:01,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:23:01,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +18: [2023-05-08 11:23:01,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:23:01,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:23:01,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +23: [2023-05-08 11:23:01,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 4: [2023-05-08 11:23:01,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:23:01,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:23:01,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:23:01,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:23:01,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:23:01,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:23:01,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:23:01,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:23:01,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:23:01,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 1: [2023-05-08 11:23:01,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:23:01,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:23:01,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:23:01,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +23: [2023-05-08 11:23:01,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:23:01,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:23:01,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +31: [2023-05-08 11:23:01,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:23:01,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:23:01,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:23:01,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:23:01,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:23:01,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:23:01,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +12: [2023-05-08 11:23:01,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:23:01,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +31: [2023-05-08 11:23:01,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:23:01,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:23:01,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:23:01,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +12: [2023-05-08 11:23:01,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +12: [2023-05-08 11:23:01,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +12: [2023-05-08 11:23:01,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:23:01,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 4: [2023-05-08 11:23:01,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:23:01,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:23:01,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:23:01,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:23:01,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:23:01,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +29: [2023-05-08 11:23:01,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +29: [2023-05-08 11:23:01,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:23:01,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +23: [2023-05-08 11:23:01,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:23:01,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:23:01,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +17: [2023-05-08 11:23:01,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:23:01,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +17: [2023-05-08 11:23:01,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:23:01,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:23:01,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:23:01,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +17: [2023-05-08 11:23:01,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:23:01,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:23:01,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:23:01,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:23:01,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +17: [2023-05-08 11:23:01,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:23:01,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:23:01,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:23:01,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:23:01,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +28: [2023-05-08 11:23:01,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +28: [2023-05-08 11:23:01,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +22: [2023-05-08 11:23:01,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:23:01,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 4: [2023-05-08 11:23:01,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:23:01,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:23:01,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:23:01,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:23:01,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:23:01,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +28: [2023-05-08 11:23:01,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:23:01,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:23:01,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +19: [2023-05-08 11:23:01,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:23:01,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +19: [2023-05-08 11:23:01,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +19: [2023-05-08 11:23:01,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:23:01,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:23:01,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +19: [2023-05-08 11:23:01,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +21: [2023-05-08 11:23:01,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:23:01,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +21: [2023-05-08 11:23:01,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:23:01,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:23:01,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:23:01,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +21: [2023-05-08 11:23:01,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +29: [2023-05-08 11:23:01,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:23:01,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:23:01,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +21: [2023-05-08 11:23:01,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:23:01,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:23:01,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:23:01,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:23:01,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:23:01,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +14: [2023-05-08 11:23:01,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:23:01,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:23:01,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:23:01,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +28: [2023-05-08 11:23:01,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +28: [2023-05-08 11:23:01,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +14: [2023-05-08 11:23:01,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:23:01,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +14: [2023-05-08 11:23:01,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:23:01,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +14: [2023-05-08 11:23:01,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:23:01,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +15: [2023-05-08 11:23:01,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +15: [2023-05-08 11:23:01,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +15: [2023-05-08 11:23:01,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +15: [2023-05-08 11:23:01,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +15: [2023-05-08 11:23:01,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:23:01,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +15: [2023-05-08 11:23:01,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +15: [2023-05-08 11:23:01,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +15: [2023-05-08 11:23:01,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +15: [2023-05-08 11:23:01,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +15: [2023-05-08 11:23:01,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +26: [2023-05-08 11:23:01,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +26: [2023-05-08 11:23:01,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 5: [2023-05-08 11:23:01,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 3: [2023-05-08 11:23:01,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:23:01,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +26: [2023-05-08 11:23:01,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 5: [2023-05-08 11:23:01,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +26: [2023-05-08 11:23:01,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:23:01,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +15: [2023-05-08 11:23:01,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +10: [2023-05-08 11:23:01,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:23:01,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:23:01,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:23:01,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +10: [2023-05-08 11:23:01,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +10: [2023-05-08 11:23:01,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +10: [2023-05-08 11:23:01,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +15: [2023-05-08 11:23:01,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +10: [2023-05-08 11:23:01,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:23:01,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +10: [2023-05-08 11:23:01,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:23:01,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +26: [2023-05-08 11:23:01,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 8: [2023-05-08 11:23:01,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:23:01,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +20: [2023-05-08 11:23:01,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +25: [2023-05-08 11:23:01,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +25: [2023-05-08 11:23:01,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +25: [2023-05-08 11:23:01,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +20: [2023-05-08 11:23:01,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +25: [2023-05-08 11:23:01,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +20: [2023-05-08 11:23:01,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 1: [2023-05-08 11:23:01,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 1: [2023-05-08 11:23:01,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +29: [2023-05-08 11:23:01,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:23:01,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +26: [2023-05-08 11:23:01,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:23:01,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +24: [2023-05-08 11:23:01,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +24: [2023-05-08 11:23:01,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +28: [2023-05-08 11:23:01,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +10: [2023-05-08 11:23:01,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +10: [2023-05-08 11:23:01,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +10: [2023-05-08 11:23:01,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +30: [2023-05-08 11:23:01,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +30: [2023-05-08 11:23:01,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +30: [2023-05-08 11:23:01,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +30: [2023-05-08 11:23:01,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 1: [2023-05-08 11:23:01,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +24: [2023-05-08 11:23:01,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:23:01,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +24: [2023-05-08 11:23:01,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +21: [2023-05-08 11:23:01,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +21: [2023-05-08 11:23:01,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +21: [2023-05-08 11:23:01,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:23:01,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:23:01,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:23:01,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:23:01,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +21: [2023-05-08 11:23:01,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +29: [2023-05-08 11:23:01,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:23:01,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +24: [2023-05-08 11:23:01,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +24: [2023-05-08 11:23:01,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:23:01,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +19: [2023-05-08 11:23:01,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +28: [2023-05-08 11:23:01,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +27: [2023-05-08 11:23:01,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +27: [2023-05-08 11:23:01,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +27: [2023-05-08 11:23:01,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 0: [2023-05-08 11:23:01,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +27: [2023-05-08 11:23:01,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 3: [2023-05-08 11:23:01,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:23:01,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:23:01,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 2: [2023-05-08 11:23:01,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +11: [2023-05-08 11:23:01,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 7: [2023-05-08 11:23:01,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +19: [2023-05-08 11:23:01,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 4: [2023-05-08 11:23:01,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:23:01,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 0: [2023-05-08 11:23:01,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +15: [2023-05-08 11:23:01,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 0: [2023-05-08 11:23:01,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +28: [2023-05-08 11:23:01,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +15: [2023-05-08 11:23:01,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +14: [2023-05-08 11:23:01,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +14: [2023-05-08 11:23:01,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:23:01,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:23:01,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +15: [2023-05-08 11:23:01,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:23:01,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:23:01,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +17: [2023-05-08 11:23:01,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:23:01,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:23:01,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:23:01,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +19: [2023-05-08 11:23:01,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:23:01,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:23:01,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +15: [2023-05-08 11:23:01,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:23:01,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:23:01,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:23:01,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:23:01,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +10: [2023-05-08 11:23:01,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +10: [2023-05-08 11:23:01,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +30: [2023-05-08 11:23:01,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +14: [2023-05-08 11:23:01,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:23:01,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +14: [2023-05-08 11:23:01,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:23:01,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +10: [2023-05-08 11:23:01,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:23:01,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:23:01,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +21: [2023-05-08 11:23:01,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:23:01,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:23:01,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:23:01,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 6: [2023-05-08 11:23:01,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:23:01,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +18: [2023-05-08 11:23:01,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 5: [2023-05-08 11:23:01,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:23:01,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +28: [2023-05-08 11:23:01,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +26: [2023-05-08 11:23:01,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +21: [2023-05-08 11:23:01,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:23:01,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +25: [2023-05-08 11:23:01,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:23:01,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +21: [2023-05-08 11:23:01,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:23:01,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +19: [2023-05-08 11:23:01,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +18: [2023-05-08 11:23:01,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 5: [2023-05-08 11:23:01,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:23:01,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +16: [2023-05-08 11:23:01,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 5: [2023-05-08 11:23:01,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +21: [2023-05-08 11:23:01,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +27: [2023-05-08 11:23:01,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +16: [2023-05-08 11:23:01,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +16: [2023-05-08 11:23:01,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +21: [2023-05-08 11:23:01,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:23:01,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +24: [2023-05-08 11:23:01,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:23:01,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 2: [2023-05-08 11:23:01,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:23:01,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +11: [2023-05-08 11:23:01,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +22: [2023-05-08 11:23:01,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +13: [2023-05-08 11:23:01,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +15: [2023-05-08 11:23:01,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +15: [2023-05-08 11:23:01,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +22: [2023-05-08 11:23:01,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +13: [2023-05-08 11:23:01,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +15: [2023-05-08 11:23:01,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +22: [2023-05-08 11:23:01,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +14: [2023-05-08 11:23:01,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +14: [2023-05-08 11:23:01,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +17: [2023-05-08 11:23:01,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +11: [2023-05-08 11:23:01,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 0: [2023-05-08 11:23:01,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:23:01,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +27: [2023-05-08 11:23:01,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +27: [2023-05-08 11:23:01,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +14: [2023-05-08 11:23:01,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:23:01,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +14: [2023-05-08 11:23:01,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +13: [2023-05-08 11:23:01,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. +15: [2023-05-08 11:23:01,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 6: [2023-05-08 11:23:01,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt. + 7: [2023-05-08 11:23:01,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:23:01,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +27: [2023-05-08 11:23:01,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:23:01,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:23:01,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:23:01,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 6: [2023-05-08 11:23:01,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:23:01,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +19: [2023-05-08 11:23:01,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:23:01,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:23:01,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:23:01,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:23:01,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:23:01,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:23:01,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:23:01,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:23:01,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:23:01,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +20: [2023-05-08 11:23:01,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +19: [2023-05-08 11:23:01,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:23:01,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:23:01,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +21: [2023-05-08 11:23:01,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:23:01,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:23:01,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:23:01,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:23:01,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +14: [2023-05-08 11:23:01,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:23:01,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +16: [2023-05-08 11:23:01,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +21: [2023-05-08 11:23:01,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +24: [2023-05-08 11:23:01,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +16: [2023-05-08 11:23:01,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +16: [2023-05-08 11:23:01,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +16: [2023-05-08 11:23:01,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:23:01,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:23:01,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:23:01,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:23:01,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:23:01,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:23:01,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 5: [2023-05-08 11:23:01,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:23:01,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +15: [2023-05-08 11:23:01,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +22: [2023-05-08 11:23:01,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:23:01,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:23:01,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:23:01,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +11: [2023-05-08 11:23:01,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +15: [2023-05-08 11:23:01,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:23:01,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:23:01,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:23:01,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:23:01,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:23:01,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +12: [2023-05-08 11:23:01,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +12: [2023-05-08 11:23:01,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:23:01,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:23:01,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:23:01,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:23:01,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:23:01,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:23:01,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:23:01,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:23:01,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +15: [2023-05-08 11:23:01,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:23:01,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +14: [2023-05-08 11:23:01,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:23:01,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:23:01,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:23:01,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:23:01,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:23:01,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +28: [2023-05-08 11:23:01,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:23:01,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 6: [2023-05-08 11:23:01,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:23:01,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +20: [2023-05-08 11:23:01,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +20: [2023-05-08 11:23:01,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +20: [2023-05-08 11:23:01,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +20: [2023-05-08 11:23:01,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 9: [2023-05-08 11:23:01,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +15: [2023-05-08 11:23:01,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +13: [2023-05-08 11:23:01,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +30: [2023-05-08 11:23:01,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:23:01,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 9: [2023-05-08 11:23:01,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:23:01,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +25: [2023-05-08 11:23:01,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:23:01,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:23:01,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +15: [2023-05-08 11:23:01,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +15: [2023-05-08 11:23:01,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:23:01,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:23:01,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +15: [2023-05-08 11:23:01,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +15: [2023-05-08 11:23:01,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 7: [2023-05-08 11:23:01,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:23:01,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:23:01,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +10: [2023-05-08 11:23:01,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:23:01,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +20: [2023-05-08 11:23:01,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:23:01,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:23:01,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +10: [2023-05-08 11:23:01,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +21: [2023-05-08 11:23:01,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:23:01,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:23:01,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:23:01,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:23:01,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 1: [2023-05-08 11:23:01,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:23:01,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 1: [2023-05-08 11:23:01,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +19: [2023-05-08 11:23:01,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +26: [2023-05-08 11:23:01,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +15: [2023-05-08 11:23:01,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +19: [2023-05-08 11:23:01,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +20: [2023-05-08 11:23:01,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +20: [2023-05-08 11:23:01,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:23:01,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:23:01,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 0: [2023-05-08 11:23:01,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:23:01,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +15: [2023-05-08 11:23:01,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +15: [2023-05-08 11:23:01,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +22: [2023-05-08 11:23:01,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:23:01,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +24: [2023-05-08 11:23:01,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:23:01,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +21: [2023-05-08 11:23:01,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +10: [2023-05-08 11:23:01,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:23:01,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:23:01,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +26: [2023-05-08 11:23:01,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +10: [2023-05-08 11:23:01,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +16: [2023-05-08 11:23:01,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +16: [2023-05-08 11:23:01,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +16: [2023-05-08 11:23:01,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +19: [2023-05-08 11:23:01,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +16: [2023-05-08 11:23:01,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +26: [2023-05-08 11:23:01,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:23:01,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +10: [2023-05-08 11:23:01,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +22: [2023-05-08 11:23:01,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +22: [2023-05-08 11:23:01,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 1: [2023-05-08 11:23:01,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +26: [2023-05-08 11:23:01,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 6: [2023-05-08 11:23:01,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:23:01,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 1: [2023-05-08 11:23:01,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 6: [2023-05-08 11:23:01,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +18: [2023-05-08 11:23:01,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:23:01,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:23:01,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:23:01,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:23:01,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +11: [2023-05-08 11:23:01,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:23:01,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:23:01,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:23:01,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +16: [2023-05-08 11:23:01,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +16: [2023-05-08 11:23:01,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:23:01,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +18: [2023-05-08 11:23:01,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +16: [2023-05-08 11:23:01,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +16: [2023-05-08 11:23:01,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 2: [2023-05-08 11:23:01,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:23:01,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 0: [2023-05-08 11:23:01,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 2: [2023-05-08 11:23:01,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +22: [2023-05-08 11:23:01,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +22: [2023-05-08 11:23:01,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +28: [2023-05-08 11:23:01,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +17: [2023-05-08 11:23:01,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +17: [2023-05-08 11:23:01,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 9: [2023-05-08 11:23:01,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:23:01,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:23:01,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:23:01,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +10: [2023-05-08 11:23:01,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 5: [2023-05-08 11:23:01,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:23:01,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +13: [2023-05-08 11:23:01,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:23:01,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +25: [2023-05-08 11:23:01,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 2: [2023-05-08 11:23:01,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:23:01,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:23:01,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:23:01,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:23:01,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +21: [2023-05-08 11:23:01,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +18: [2023-05-08 11:23:01,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:23:01,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +10: [2023-05-08 11:23:01,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:23:01,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 2: [2023-05-08 11:23:01,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +22: [2023-05-08 11:23:01,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:23:01,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +22: [2023-05-08 11:23:01,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +19: [2023-05-08 11:23:01,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +13: [2023-05-08 11:23:01,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +20: [2023-05-08 11:23:01,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:23:01,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:23:01,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:23:01,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:23:01,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:23:01,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:23:01,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +14: [2023-05-08 11:23:01,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +13: [2023-05-08 11:23:01,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 5: [2023-05-08 11:23:01,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +13: [2023-05-08 11:23:01,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:23:01,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:23:01,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +28: [2023-05-08 11:23:01,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +18: [2023-05-08 11:23:01,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:23:01,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +24: [2023-05-08 11:23:01,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +21: [2023-05-08 11:23:01,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:23:01,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:23:01,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:23:01,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +11: [2023-05-08 11:23:01,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:23:01,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 1: [2023-05-08 11:23:01,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +25: [2023-05-08 11:23:01,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 9: [2023-05-08 11:23:01,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:23:01,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:23:01,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:23:01,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 9: [2023-05-08 11:23:01,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +14: [2023-05-08 11:23:01,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:23:01,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +12: [2023-05-08 11:23:01,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:23:01,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +20: [2023-05-08 11:23:01,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:23:01,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:23:01,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +11: [2023-05-08 11:23:01,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:23:01,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 1: [2023-05-08 11:23:01,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:23:01,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:23:01,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 0: [2023-05-08 11:23:01,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:23:01,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +17: [2023-05-08 11:23:01,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:23:01,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:23:01,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:23:01,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 6: [2023-05-08 11:23:01,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 2: [2023-05-08 11:23:01,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:23:01,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:23:01,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:23:01,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:23:01,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 2: [2023-05-08 11:23:01,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 2: [2023-05-08 11:23:01,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:23:01,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:23:01,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +13: [2023-05-08 11:23:01,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:23:01,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +13: [2023-05-08 11:23:01,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:23:01,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 3: [2023-05-08 11:23:01,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 9: [2023-05-08 11:23:01,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 3: [2023-05-08 11:23:01,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 3: [2023-05-08 11:23:01,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 9: [2023-05-08 11:23:01,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +14: [2023-05-08 11:23:01,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +26: [2023-05-08 11:23:01,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:23:01,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +13: [2023-05-08 11:23:01,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +14: [2023-05-08 11:23:01,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +20: [2023-05-08 11:23:01,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:23:01,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:23:01,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +13: [2023-05-08 11:23:01,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:23:01,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:23:01,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:23:01,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +17: [2023-05-08 11:23:01,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:23:01,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +17: [2023-05-08 11:23:01,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +13: [2023-05-08 11:23:01,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +17: [2023-05-08 11:23:01,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:23:01,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 1: [2023-05-08 11:23:01,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 8: [2023-05-08 11:23:01,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 8: [2023-05-08 11:23:01,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 8: [2023-05-08 11:23:01,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +20: [2023-05-08 11:23:01,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +13: [2023-05-08 11:23:01,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:23:01,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:23:01,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 1: [2023-05-08 11:23:01,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:23:01,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +27: [2023-05-08 11:23:01,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +27: [2023-05-08 11:23:01,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +27: [2023-05-08 11:23:01,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:23:01,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 3: [2023-05-08 11:23:01,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +27: [2023-05-08 11:23:01,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +27: [2023-05-08 11:23:01,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +27: [2023-05-08 11:23:01,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +27: [2023-05-08 11:23:01,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +27: [2023-05-08 11:23:01,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +27: [2023-05-08 11:23:01,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +27: [2023-05-08 11:23:01,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +27: [2023-05-08 11:23:01,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... +27: [2023-05-08 11:23:01,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... + 9: [2023-05-08 11:23:01,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 9: [2023-05-08 11:23:01,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 9: [2023-05-08 11:23:01,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +27: [2023-05-08 11:23:01,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +27: [2023-05-08 11:23:01,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt... +27: [2023-05-08 11:23:01,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 9: [2023-05-08 11:23:01,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 9: [2023-05-08 11:23:01,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 0: [2023-05-08 11:23:01,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +27: [2023-05-08 11:23:01,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +27: [2023-05-08 11:23:01,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +27: [2023-05-08 11:23:01,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +27: [2023-05-08 11:23:01,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +14: [2023-05-08 11:23:01,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:23:01,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +14: [2023-05-08 11:23:01,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:23:01,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 0: [2023-05-08 11:23:01,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:23:01,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +14: [2023-05-08 11:23:01,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +31: [2023-05-08 11:23:01,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +31: [2023-05-08 11:23:01,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +14: [2023-05-08 11:23:01,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +14: [2023-05-08 11:23:01,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +14: [2023-05-08 11:23:01,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:23:01,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:23:01,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:23:01,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:23:01,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:23:01,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +31: [2023-05-08 11:23:01,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +31: [2023-05-08 11:23:01,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +27: [2023-05-08 11:23:01,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 0: [2023-05-08 11:23:01,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +14: [2023-05-08 11:23:01,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +27: [2023-05-08 11:23:01,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +25: [2023-05-08 11:23:01,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:23:01,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:23:01,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:23:01,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:23:01,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:23:01,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:23:01,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:23:01,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:23:01,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:23:01,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:23:01,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +14: [2023-05-08 11:23:01,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:23:01,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:23:01,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:23:01,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +31: [2023-05-08 11:23:01,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:23:01,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +31: [2023-05-08 11:23:01,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +28: [2023-05-08 11:23:01,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:23:01,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:23:01,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:23:01,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:23:01,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +28: [2023-05-08 11:23:01,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +28: [2023-05-08 11:23:01,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:23:01,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:23:01,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +25: [2023-05-08 11:23:01,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:23:01,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:23:01,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:23:01,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:23:01,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:23:01,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:23:01,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:23:01,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:23:01,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:23:01,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:23:01,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:23:01,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:23:01,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:23:01,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:23:01,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:23:01,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:23:01,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +15: [2023-05-08 11:23:01,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +15: [2023-05-08 11:23:01,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +10: [2023-05-08 11:23:01,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +10: [2023-05-08 11:23:01,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +10: [2023-05-08 11:23:01,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +10: [2023-05-08 11:23:01,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +10: [2023-05-08 11:23:01,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +10: [2023-05-08 11:23:01,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +15: [2023-05-08 11:23:01,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +15: [2023-05-08 11:23:01,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +10: [2023-05-08 11:23:01,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +10: [2023-05-08 11:23:01,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:23:01,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:23:01,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +15: [2023-05-08 11:23:01,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:23:01,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +25: [2023-05-08 11:23:01,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:23:01,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:23:01,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:23:01,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:23:01,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:23:01,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:23:01,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:23:01,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:23:01,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:23:01,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:23:01,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:23:01,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:23:01,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +15: [2023-05-08 11:23:01,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:23:01,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:23:01,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:23:01,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:23:01,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +10: [2023-05-08 11:23:01,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:23:01,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:23:01,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:23:01,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:23:01,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:23:01,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +10: [2023-05-08 11:23:01,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +10: [2023-05-08 11:23:01,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:23:01,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:23:01,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:23:01,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +15: [2023-05-08 11:23:01,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:23:01,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +12: [2023-05-08 11:23:01,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +12: [2023-05-08 11:23:01,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +12: [2023-05-08 11:23:01,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +19: [2023-05-08 11:23:01,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:23:01,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:23:01,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:23:01,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:23:01,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:23:01,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +10: [2023-05-08 11:23:01,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:23:01,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +26: [2023-05-08 11:23:01,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:23:01,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:23:01,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:23:01,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:23:01,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:23:01,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:23:01,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:23:01,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:23:01,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:23:01,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +15: [2023-05-08 11:23:01,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:23:01,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:23:01,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:23:01,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +26: [2023-05-08 11:23:01,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +26: [2023-05-08 11:23:01,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +26: [2023-05-08 11:23:01,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +27: [2023-05-08 11:23:01,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:23:01,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +27: [2023-05-08 11:23:01,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:23:01,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:23:01,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +27: [2023-05-08 11:23:01,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +27: [2023-05-08 11:23:01,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:23:01,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:23:01,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:23:01,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +12: [2023-05-08 11:23:01,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:23:01,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +27: [2023-05-08 11:23:01,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:23:01,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:23:01,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:23:01,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +23: [2023-05-08 11:23:01,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +23: [2023-05-08 11:23:01,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +23: [2023-05-08 11:23:01,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +23: [2023-05-08 11:23:01,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +12: [2023-05-08 11:23:01,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:23:01,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:23:01,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +12: [2023-05-08 11:23:01,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +27: [2023-05-08 11:23:01,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +27: [2023-05-08 11:23:01,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +27: [2023-05-08 11:23:01,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:23:01,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:23:01,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:23:01,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +27: [2023-05-08 11:23:01,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:23:01,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:23:01,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +23: [2023-05-08 11:23:01,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +23: [2023-05-08 11:23:01,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +23: [2023-05-08 11:23:01,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +23: [2023-05-08 11:23:01,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +23: [2023-05-08 11:23:01,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +24: [2023-05-08 11:23:01,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +24: [2023-05-08 11:23:01,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +24: [2023-05-08 11:23:01,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +24: [2023-05-08 11:23:01,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +11: [2023-05-08 11:23:01,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +11: [2023-05-08 11:23:01,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +14: [2023-05-08 11:23:01,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +26: [2023-05-08 11:23:01,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +29: [2023-05-08 11:23:01,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +14: [2023-05-08 11:23:01,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +14: [2023-05-08 11:23:01,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +29: [2023-05-08 11:23:01,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +29: [2023-05-08 11:23:01,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +11: [2023-05-08 11:23:01,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +14: [2023-05-08 11:23:01,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +26: [2023-05-08 11:23:01,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +29: [2023-05-08 11:23:01,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +23: [2023-05-08 11:23:01,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:23:01,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +26: [2023-05-08 11:23:01,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 7: [2023-05-08 11:23:01,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 7: [2023-05-08 11:23:01,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +23: [2023-05-08 11:23:01,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 7: [2023-05-08 11:23:01,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 1: [2023-05-08 11:23:01,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 1: [2023-05-08 11:23:01,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 7: [2023-05-08 11:23:01,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +22: [2023-05-08 11:23:01,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +22: [2023-05-08 11:23:01,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +22: [2023-05-08 11:23:01,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +22: [2023-05-08 11:23:01,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 1: [2023-05-08 11:23:01,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +11: [2023-05-08 11:23:01,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 1: [2023-05-08 11:23:01,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 2: [2023-05-08 11:23:01,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +28: [2023-05-08 11:23:01,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 2: [2023-05-08 11:23:01,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 2: [2023-05-08 11:23:01,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 2: [2023-05-08 11:23:01,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +28: [2023-05-08 11:23:01,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +28: [2023-05-08 11:23:01,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +28: [2023-05-08 11:23:01,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +24: [2023-05-08 11:23:01,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +24: [2023-05-08 11:23:01,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +24: [2023-05-08 11:23:01,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 9: [2023-05-08 11:23:01,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 6: [2023-05-08 11:23:01,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 6: [2023-05-08 11:23:01,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 6: [2023-05-08 11:23:01,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 9: [2023-05-08 11:23:01,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 9: [2023-05-08 11:23:01,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 5: [2023-05-08 11:23:01,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 5: [2023-05-08 11:23:01,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 5: [2023-05-08 11:23:01,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +21: [2023-05-08 11:23:01,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +21: [2023-05-08 11:23:01,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +21: [2023-05-08 11:23:01,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +11: [2023-05-08 11:23:01,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +21: [2023-05-08 11:23:01,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +11: [2023-05-08 11:23:01,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:23:01,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:23:01,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +24: [2023-05-08 11:23:01,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:23:01,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:23:01,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +29: [2023-05-08 11:23:01,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +29: [2023-05-08 11:23:01,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:23:01,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +29: [2023-05-08 11:23:01,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:23:01,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:23:01,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +30: [2023-05-08 11:23:01,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +30: [2023-05-08 11:23:01,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +14: [2023-05-08 11:23:01,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:23:01,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +26: [2023-05-08 11:23:01,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:23:01,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +14: [2023-05-08 11:23:01,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +22: [2023-05-08 11:23:01,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 8: [2023-05-08 11:23:01,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 8: [2023-05-08 11:23:01,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +11: [2023-05-08 11:23:01,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:23:01,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:23:01,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:23:01,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:23:01,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:23:01,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:23:01,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:23:01,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:23:01,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:23:01,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:23:01,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:23:01,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +21: [2023-05-08 11:23:01,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +17: [2023-05-08 11:23:01,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +17: [2023-05-08 11:23:01,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +17: [2023-05-08 11:23:01,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +30: [2023-05-08 11:23:01,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +30: [2023-05-08 11:23:01,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +30: [2023-05-08 11:23:01,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +30: [2023-05-08 11:23:01,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 3: [2023-05-08 11:23:01,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +17: [2023-05-08 11:23:01,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +21: [2023-05-08 11:23:01,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:23:01,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +28: [2023-05-08 11:23:01,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:23:01,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:23:01,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:23:01,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +28: [2023-05-08 11:23:01,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:23:01,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:23:01,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 9: [2023-05-08 11:23:01,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 5: [2023-05-08 11:23:01,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:23:01,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +21: [2023-05-08 11:23:01,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:23:01,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +30: [2023-05-08 11:23:01,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:23:01,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:23:01,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +26: [2023-05-08 11:23:01,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +24: [2023-05-08 11:23:01,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:23:01,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +26: [2023-05-08 11:23:01,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:23:01,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:23:01,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +24: [2023-05-08 11:23:01,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +30: [2023-05-08 11:23:01,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:23:01,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:23:01,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +26: [2023-05-08 11:23:01,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:23:01,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 2: [2023-05-08 11:23:01,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:23:01,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +26: [2023-05-08 11:23:01,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 7: [2023-05-08 11:23:01,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:23:01,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:23:01,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:23:01,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 7: [2023-05-08 11:23:01,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 3: [2023-05-08 11:23:01,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +18: [2023-05-08 11:23:01,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:23:01,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:23:01,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:23:01,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:23:01,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:23:01,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:23:01,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:23:01,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +29: [2023-05-08 11:23:01,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +29: [2023-05-08 11:23:01,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +29: [2023-05-08 11:23:01,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +14: [2023-05-08 11:23:01,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +14: [2023-05-08 11:23:01,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +14: [2023-05-08 11:23:01,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 6: [2023-05-08 11:23:01,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:23:01,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:23:01,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 1: [2023-05-08 11:23:01,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 5: [2023-05-08 11:23:01,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +21: [2023-05-08 11:23:01,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:23:01,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +30: [2023-05-08 11:23:01,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +17: [2023-05-08 11:23:01,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +30: [2023-05-08 11:23:01,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +24: [2023-05-08 11:23:01,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 2: [2023-05-08 11:23:01,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:23:01,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +21: [2023-05-08 11:23:01,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +24: [2023-05-08 11:23:01,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 1: [2023-05-08 11:23:01,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:23:01,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +14: [2023-05-08 11:23:01,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +14: [2023-05-08 11:23:01,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 7: [2023-05-08 11:23:01,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 7: [2023-05-08 11:23:01,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +16: [2023-05-08 11:23:01,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +16: [2023-05-08 11:23:01,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +16: [2023-05-08 11:23:01,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 5: [2023-05-08 11:23:01,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +16: [2023-05-08 11:23:01,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +21: [2023-05-08 11:23:01,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +15: [2023-05-08 11:23:01,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +15: [2023-05-08 11:23:01,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +21: [2023-05-08 11:23:01,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +15: [2023-05-08 11:23:01,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +29: [2023-05-08 11:23:01,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:23:01,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:23:01,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +30: [2023-05-08 11:23:01,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:23:01,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 8: [2023-05-08 11:23:01,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 1: [2023-05-08 11:23:01,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 6: [2023-05-08 11:23:01,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 6: [2023-05-08 11:23:01,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +29: [2023-05-08 11:23:01,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +22: [2023-05-08 11:23:01,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 5: [2023-05-08 11:23:01,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +19: [2023-05-08 11:23:01,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +19: [2023-05-08 11:23:01,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +30: [2023-05-08 11:23:01,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:23:01,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 7: [2023-05-08 11:23:01,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:23:01,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +19: [2023-05-08 11:23:01,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +28: [2023-05-08 11:23:01,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +14: [2023-05-08 11:23:01,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:23:01,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:23:01,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +19: [2023-05-08 11:23:01,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +24: [2023-05-08 11:23:01,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:23:01,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:23:01,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:23:01,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:23:01,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:23:01,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +29: [2023-05-08 11:23:01,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +29: [2023-05-08 11:23:01,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:23:01,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +29: [2023-05-08 11:23:01,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +31: [2023-05-08 11:23:01,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:23:01,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:23:01,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +10: [2023-05-08 11:23:01,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +10: [2023-05-08 11:23:01,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +29: [2023-05-08 11:23:01,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:23:01,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +31: [2023-05-08 11:23:01,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:23:01,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:23:01,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +31: [2023-05-08 11:23:01,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:23:01,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:23:01,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:23:01,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +14: [2023-05-08 11:23:01,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +31: [2023-05-08 11:23:01,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:23:01,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:23:01,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 0: [2023-05-08 11:23:01,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 0: [2023-05-08 11:23:01,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +18: [2023-05-08 11:23:01,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:23:01,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +20: [2023-05-08 11:23:01,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +20: [2023-05-08 11:23:01,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +22: [2023-05-08 11:23:01,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +20: [2023-05-08 11:23:01,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +18: [2023-05-08 11:23:01,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:23:01,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:23:01,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +13: [2023-05-08 11:23:01,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +10: [2023-05-08 11:23:01,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +13: [2023-05-08 11:23:01,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +13: [2023-05-08 11:23:01,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +22: [2023-05-08 11:23:01,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +16: [2023-05-08 11:23:01,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:23:01,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:23:01,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +16: [2023-05-08 11:23:01,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +28: [2023-05-08 11:23:01,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:23:01,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:23:01,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:23:01,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:23:01,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +30: [2023-05-08 11:23:01,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +15: [2023-05-08 11:23:01,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:23:01,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:23:01,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +28: [2023-05-08 11:23:01,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +30: [2023-05-08 11:23:01,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +30: [2023-05-08 11:23:01,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +30: [2023-05-08 11:23:01,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +28: [2023-05-08 11:23:01,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:23:01,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +15: [2023-05-08 11:23:01,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +30: [2023-05-08 11:23:01,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:23:01,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:23:01,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:23:01,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +11: [2023-05-08 11:23:01,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:23:01,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:23:01,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +11: [2023-05-08 11:23:01,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +11: [2023-05-08 11:23:01,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +11: [2023-05-08 11:23:01,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +11: [2023-05-08 11:23:01,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +11: [2023-05-08 11:23:01,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +11: [2023-05-08 11:23:01,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +11: [2023-05-08 11:23:01,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +11: [2023-05-08 11:23:01,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +11: [2023-05-08 11:23:01,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +11: [2023-05-08 11:23:01,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +11: [2023-05-08 11:23:01,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +20: [2023-05-08 11:23:01,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:23:01,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:23:01,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:23:01,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:23:01,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +24: [2023-05-08 11:23:01,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:23:01,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:23:01,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:23:01,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:23:01,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:23:01,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +16: [2023-05-08 11:23:01,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +16: [2023-05-08 11:23:01,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +16: [2023-05-08 11:23:01,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +16: [2023-05-08 11:23:01,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:23:01,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +16: [2023-05-08 11:23:01,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +16: [2023-05-08 11:23:01,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:23:01,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +16: [2023-05-08 11:23:01,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +16: [2023-05-08 11:23:01,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 4: [2023-05-08 11:23:01,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 0: [2023-05-08 11:23:01,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:23:01,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +29: [2023-05-08 11:23:01,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:23:01,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:23:01,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:23:01,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +10: [2023-05-08 11:23:01,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +10: [2023-05-08 11:23:01,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:23:01,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +20: [2023-05-08 11:23:01,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:23:01,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +20: [2023-05-08 11:23:01,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:23:01,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +20: [2023-05-08 11:23:01,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +20: [2023-05-08 11:23:01,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +30: [2023-05-08 11:23:01,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +16: [2023-05-08 11:23:01,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:23:01,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +20: [2023-05-08 11:23:01,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:23:01,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:23:01,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:23:01,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:23:01,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +31: [2023-05-08 11:23:01,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:23:01,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:23:01,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +29: [2023-05-08 11:23:01,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:23:01,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +18: [2023-05-08 11:23:01,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +18: [2023-05-08 11:23:01,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +18: [2023-05-08 11:23:01,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +16: [2023-05-08 11:23:01,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:23:01,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +15: [2023-05-08 11:23:01,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +15: [2023-05-08 11:23:01,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +24: [2023-05-08 11:23:01,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:23:01,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:23:01,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:23:01,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +31: [2023-05-08 11:23:01,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +30: [2023-05-08 11:23:01,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:23:01,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:23:01,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +11: [2023-05-08 11:23:01,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:23:01,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +24: [2023-05-08 11:23:01,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:23:01,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:23:01,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +30: [2023-05-08 11:23:01,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:23:01,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:23:01,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +30: [2023-05-08 11:23:01,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +16: [2023-05-08 11:23:01,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:23:01,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +24: [2023-05-08 11:23:01,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:23:01,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +15: [2023-05-08 11:23:01,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +13: [2023-05-08 11:23:01,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:23:01,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +18: [2023-05-08 11:23:01,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +31: [2023-05-08 11:23:01,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +10: [2023-05-08 11:23:01,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:23:01,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +10: [2023-05-08 11:23:01,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +29: [2023-05-08 11:23:01,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +29: [2023-05-08 11:23:01,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +31: [2023-05-08 11:23:01,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:23:01,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:23:01,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +18: [2023-05-08 11:23:01,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:23:01,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:23:01,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +31: [2023-05-08 11:23:01,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +16: [2023-05-08 11:23:01,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:23:01,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:23:01,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +31: [2023-05-08 11:23:01,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +16: [2023-05-08 11:23:01,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +13: [2023-05-08 11:23:01,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:23:01,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +20: [2023-05-08 11:23:01,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +20: [2023-05-08 11:23:01,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +16: [2023-05-08 11:23:01,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +16: [2023-05-08 11:23:01,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +20: [2023-05-08 11:23:01,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:23:01,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:23:01,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +16: [2023-05-08 11:23:01,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:23:01,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +11: [2023-05-08 11:23:01,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +10: [2023-05-08 11:23:01,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +10: [2023-05-08 11:23:01,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +10: [2023-05-08 11:23:01,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +10: [2023-05-08 11:23:01,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +20: [2023-05-08 11:23:01,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:23:01,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +20: [2023-05-08 11:23:01,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:23:01,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +29: [2023-05-08 11:23:01,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:23:01,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:23:01,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:23:01,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +12: [2023-05-08 11:23:01,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:23:01,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:23:01,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +12: [2023-05-08 11:23:01,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +12: [2023-05-08 11:23:01,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +12: [2023-05-08 11:23:01,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:23:01,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +12: [2023-05-08 11:23:01,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:23:01,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +20: [2023-05-08 11:23:01,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +20: [2023-05-08 11:23:01,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +25: [2023-05-08 11:23:01,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +31: [2023-05-08 11:23:01,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +25: [2023-05-08 11:23:01,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +31: [2023-05-08 11:23:01,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 4: [2023-05-08 11:23:01,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:23:01,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 4: [2023-05-08 11:23:01,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +19: [2023-05-08 11:23:01,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +12: [2023-05-08 11:23:01,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:23:01,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +11: [2023-05-08 11:23:01,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:23:01,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:23:01,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +16: [2023-05-08 11:23:01,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +16: [2023-05-08 11:23:01,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +16: [2023-05-08 11:23:01,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:23:01,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +11: [2023-05-08 11:23:01,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:23:01,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +27: [2023-05-08 11:23:01,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +27: [2023-05-08 11:23:01,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. + 3: [2023-05-08 11:23:01,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 3: [2023-05-08 11:23:01,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:23:01,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:23:01,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 3: [2023-05-08 11:23:01,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 3: [2023-05-08 11:23:01,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 3: [2023-05-08 11:23:01,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +10: [2023-05-08 11:23:01,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:23:01,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 0: [2023-05-08 11:23:01,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:23:01,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +10: [2023-05-08 11:23:01,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +27: [2023-05-08 11:23:01,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt. +20: [2023-05-08 11:23:01,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:23:01,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 4: [2023-05-08 11:23:01,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:23:01,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:23:01,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:23:01,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:23:01,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +16: [2023-05-08 11:23:01,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +31: [2023-05-08 11:23:01,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +11: [2023-05-08 11:23:01,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +25: [2023-05-08 11:23:01,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +27: [2023-05-08 11:23:01,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +25: [2023-05-08 11:23:01,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +25: [2023-05-08 11:23:01,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +27: [2023-05-08 11:23:01,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +31: [2023-05-08 11:23:01,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:23:01,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:23:01,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:23:01,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +12: [2023-05-08 11:23:01,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +27: [2023-05-08 11:23:01,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:23:01,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:23:01,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +27: [2023-05-08 11:23:01,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +12: [2023-05-08 11:23:01,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +20: [2023-05-08 11:23:01,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:23:01,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:23:01,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:23:01,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:23:01,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:23:01,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +25: [2023-05-08 11:23:01,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:23:01,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +27: [2023-05-08 11:23:01,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:23:01,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:23:01,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +27: [2023-05-08 11:23:01,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +27: [2023-05-08 11:23:01,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:23:01,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 3: [2023-05-08 11:23:01,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +27: [2023-05-08 11:23:01,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:23:01,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:23:01,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... +27: [2023-05-08 11:23:01,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 3: [2023-05-08 11:23:01,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:23:01,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +31: [2023-05-08 11:23:01,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +12: [2023-05-08 11:23:01,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +31: [2023-05-08 11:23:01,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +31: [2023-05-08 11:23:01,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +25: [2023-05-08 11:23:01,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:23:01,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt... + 7: [2023-05-08 11:23:01,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +17: [2023-05-08 11:23:01,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +17: [2023-05-08 11:23:01,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +17: [2023-05-08 11:23:01,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +17: [2023-05-08 11:23:01,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +17: [2023-05-08 11:23:01,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +17: [2023-05-08 11:23:01,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +17: [2023-05-08 11:23:01,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +17: [2023-05-08 11:23:01,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:23:01,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:23:01,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:23:01,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:23:01,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +17: [2023-05-08 11:23:01,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +17: [2023-05-08 11:23:01,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +17: [2023-05-08 11:23:01,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +31: [2023-05-08 11:23:01,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +17: [2023-05-08 11:23:01,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +12: [2023-05-08 11:23:01,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +12: [2023-05-08 11:23:01,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +31: [2023-05-08 11:23:01,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +12: [2023-05-08 11:23:01,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +12: [2023-05-08 11:23:01,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +31: [2023-05-08 11:23:01,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +31: [2023-05-08 11:23:01,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 7: [2023-05-08 11:23:01,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:23:01,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:23:01,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:23:01,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:23:01,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +12: [2023-05-08 11:23:01,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:23:01,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +17: [2023-05-08 11:23:01,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +17: [2023-05-08 11:23:01,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +12: [2023-05-08 11:23:01,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +12: [2023-05-08 11:23:01,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +12: [2023-05-08 11:23:01,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +12: [2023-05-08 11:23:01,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +17: [2023-05-08 11:23:01,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +17: [2023-05-08 11:23:01,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:23:01,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +12: [2023-05-08 11:23:01,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +12: [2023-05-08 11:23:01,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +12: [2023-05-08 11:23:01,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +21: [2023-05-08 11:23:01,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +17: [2023-05-08 11:23:01,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +17: [2023-05-08 11:23:01,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +21: [2023-05-08 11:23:01,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:23:01,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:23:01,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +21: [2023-05-08 11:23:01,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:23:01,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +21: [2023-05-08 11:23:01,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +13: [2023-05-08 11:23:01,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:23:01,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +13: [2023-05-08 11:23:01,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:23:01,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +13: [2023-05-08 11:23:01,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:23:01,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +13: [2023-05-08 11:23:01,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +21: [2023-05-08 11:23:01,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +13: [2023-05-08 11:23:01,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:23:01,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:23:01,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:23:01,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:23:01,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:23:01,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:23:01,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:23:01,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 9: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 2: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 9: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 0: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:23:01,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +23: [2023-05-08 11:23:01,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +23: [2023-05-08 11:23:01,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +23: [2023-05-08 11:23:01,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:23:01,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:23:01,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +23: [2023-05-08 11:23:01,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:23:01,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:23:01,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:23:01,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:23:01,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:23:01,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:23:01,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:23:01,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:23:01,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:23:01,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:23:01,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:23:01,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:23:01,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:23:01,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:23:01,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:23:01,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:23:01,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:23:01,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:23:01,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:23:01,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:23:01,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:23:01,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:23:01,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:23:01,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:23:01,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:23:01,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:23:01,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:23:01,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:23:01,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +23: [2023-05-08 11:23:01,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:23:01,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:23:01,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:23:01,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:23:01,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:23:01,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:23:01,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:23:01,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:23:01,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:23:01,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +10: [2023-05-08 11:23:01,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:23:01,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:23:01,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +22: [2023-05-08 11:23:01,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +23: [2023-05-08 11:23:01,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +23: [2023-05-08 11:23:01,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +10: [2023-05-08 11:23:01,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:23:01,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:23:01,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:23:01,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:23:01,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +14: [2023-05-08 11:23:01,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 5: [2023-05-08 11:23:01,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:23:01,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:23:01,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +10: [2023-05-08 11:23:01,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:23:01,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:23:01,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:23:01,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +22: [2023-05-08 11:23:01,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +23: [2023-05-08 11:23:01,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +10: [2023-05-08 11:23:01,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +10: [2023-05-08 11:23:01,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:23:01,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +26: [2023-05-08 11:23:01,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +26: [2023-05-08 11:23:01,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +26: [2023-05-08 11:23:01,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +26: [2023-05-08 11:23:01,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +26: [2023-05-08 11:23:01,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:23:01,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +23: [2023-05-08 11:23:01,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +23: [2023-05-08 11:23:01,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:23:01,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +26: [2023-05-08 11:23:01,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:23:01,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +23: [2023-05-08 11:23:01,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:23:01,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:23:01,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:23:01,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +22: [2023-05-08 11:23:01,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:23:01,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:23:01,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:23:01,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:23:01,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:23:01,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:23:01,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +23: [2023-05-08 11:23:01,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:23:01,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:23:01,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:23:01,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:23:01,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:23:01,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:23:01,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +23: [2023-05-08 11:23:01,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +23: [2023-05-08 11:23:01,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +14: [2023-05-08 11:23:01,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +23: [2023-05-08 11:23:01,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +14: [2023-05-08 11:23:01,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +26: [2023-05-08 11:23:01,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +26: [2023-05-08 11:23:01,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:23:01,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:23:01,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +23: [2023-05-08 11:23:01,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:23:01,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:23:01,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:23:01,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:23:01,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:23:01,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +14: [2023-05-08 11:23:01,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +23: [2023-05-08 11:23:01,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:23:01,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:23:01,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:23:01,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:23:01,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:23:01,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:23:01,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:23:01,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +21: [2023-05-08 11:23:01,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +21: [2023-05-08 11:23:01,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +21: [2023-05-08 11:23:01,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +23: [2023-05-08 11:23:01,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +23: [2023-05-08 11:23:01,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +14: [2023-05-08 11:23:01,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:23:01,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +27: [2023-05-08 11:23:01,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +26: [2023-05-08 11:23:01,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +26: [2023-05-08 11:23:01,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +27: [2023-05-08 11:23:01,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +27: [2023-05-08 11:23:01,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +27: [2023-05-08 11:23:01,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +27: [2023-05-08 11:23:01,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +27: [2023-05-08 11:23:01,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:23:01,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 7: [2023-05-08 11:23:01,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 7: [2023-05-08 11:23:01,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +27: [2023-05-08 11:23:01,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +23: [2023-05-08 11:23:01,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +23: [2023-05-08 11:23:01,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +19: [2023-05-08 11:23:01,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:23:01,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:23:01,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:23:01,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +27: [2023-05-08 11:23:01,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:23:01,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:23:01,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:23:01,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +24: [2023-05-08 11:23:01,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +19: [2023-05-08 11:23:01,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:23:01,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:23:01,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:23:01,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:23:01,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:23:01,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:23:01,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:23:01,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +21: [2023-05-08 11:23:01,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:23:01,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:23:01,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:23:01,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:23:01,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:23:01,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:23:01,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:23:01,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +22: [2023-05-08 11:23:01,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +21: [2023-05-08 11:23:01,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:23:01,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +23: [2023-05-08 11:23:01,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +23: [2023-05-08 11:23:01,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +15: [2023-05-08 11:23:01,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:23:01,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:23:01,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +22: [2023-05-08 11:23:01,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +22: [2023-05-08 11:23:01,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +22: [2023-05-08 11:23:01,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 5: [2023-05-08 11:23:01,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +24: [2023-05-08 11:23:01,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 5: [2023-05-08 11:23:01,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 5: [2023-05-08 11:23:01,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +24: [2023-05-08 11:23:01,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +27: [2023-05-08 11:23:01,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:23:01,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:23:01,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +27: [2023-05-08 11:23:01,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +27: [2023-05-08 11:23:01,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 1: [2023-05-08 11:23:01,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 1: [2023-05-08 11:23:01,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +24: [2023-05-08 11:23:01,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:23:01,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:23:01,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:23:01,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:23:01,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:23:01,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +26: [2023-05-08 11:23:01,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +26: [2023-05-08 11:23:01,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +22: [2023-05-08 11:23:01,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +26: [2023-05-08 11:23:01,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +26: [2023-05-08 11:23:01,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +21: [2023-05-08 11:23:01,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:23:01,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:23:01,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +21: [2023-05-08 11:23:01,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +25: [2023-05-08 11:23:01,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +25: [2023-05-08 11:23:01,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +25: [2023-05-08 11:23:01,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +25: [2023-05-08 11:23:01,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +27: [2023-05-08 11:23:01,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +25: [2023-05-08 11:23:01,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +25: [2023-05-08 11:23:01,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:23:01,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:23:01,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:23:01,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:23:01,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +25: [2023-05-08 11:23:01,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:23:01,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:23:01,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:23:01,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:23:01,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +24: [2023-05-08 11:23:01,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:23:01,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +27: [2023-05-08 11:23:01,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:23:01,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +27: [2023-05-08 11:23:01,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:23:01,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:23:01,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:23:01,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 1: [2023-05-08 11:23:01,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:23:01,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:23:01,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +22: [2023-05-08 11:23:01,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +24: [2023-05-08 11:23:01,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:23:01,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +19: [2023-05-08 11:23:01,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:23:01,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +26: [2023-05-08 11:23:01,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +26: [2023-05-08 11:23:01,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:23:01,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +27: [2023-05-08 11:23:01,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +19: [2023-05-08 11:23:01,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +28: [2023-05-08 11:23:01,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:23:01,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +26: [2023-05-08 11:23:01,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 6: [2023-05-08 11:23:01,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 6: [2023-05-08 11:23:01,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 6: [2023-05-08 11:23:01,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 5: [2023-05-08 11:23:01,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 5: [2023-05-08 11:23:01,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 2: [2023-05-08 11:23:01,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 2: [2023-05-08 11:23:01,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 5: [2023-05-08 11:23:01,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 2: [2023-05-08 11:23:01,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 2: [2023-05-08 11:23:01,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +22: [2023-05-08 11:23:01,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:23:01,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +24: [2023-05-08 11:23:01,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:23:01,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +28: [2023-05-08 11:23:01,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +24: [2023-05-08 11:23:01,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +22: [2023-05-08 11:23:01,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:23:01,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:23:01,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +22: [2023-05-08 11:23:01,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +28: [2023-05-08 11:23:01,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +28: [2023-05-08 11:23:01,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +25: [2023-05-08 11:23:01,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +25: [2023-05-08 11:23:01,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:23:01,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +27: [2023-05-08 11:23:01,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 1: [2023-05-08 11:23:01,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 1: [2023-05-08 11:23:01,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 7: [2023-05-08 11:23:01,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 6: [2023-05-08 11:23:01,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 6: [2023-05-08 11:23:01,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:23:01,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +29: [2023-05-08 11:23:01,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +29: [2023-05-08 11:23:01,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +29: [2023-05-08 11:23:01,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 2: [2023-05-08 11:23:01,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:23:01,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:23:01,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:23:01,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:23:01,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:23:01,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:23:01,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:23:01,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:23:01,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +16: [2023-05-08 11:23:01,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +16: [2023-05-08 11:23:01,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +16: [2023-05-08 11:23:01,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 5: [2023-05-08 11:23:01,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +18: [2023-05-08 11:23:01,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 1: [2023-05-08 11:23:01,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:23:01,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +18: [2023-05-08 11:23:01,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +18: [2023-05-08 11:23:01,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +25: [2023-05-08 11:23:01,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +25: [2023-05-08 11:23:01,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +16: [2023-05-08 11:23:01,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 1: [2023-05-08 11:23:01,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +18: [2023-05-08 11:23:01,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:23:01,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:23:01,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:23:01,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:23:01,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:23:01,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:23:01,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:23:01,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:23:01,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +26: [2023-05-08 11:23:01,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 2: [2023-05-08 11:23:01,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +29: [2023-05-08 11:23:01,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +26: [2023-05-08 11:23:01,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +26: [2023-05-08 11:23:01,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +26: [2023-05-08 11:23:01,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +29: [2023-05-08 11:23:01,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:23:01,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +16: [2023-05-08 11:23:01,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:23:01,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +16: [2023-05-08 11:23:01,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +26: [2023-05-08 11:23:01,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +18: [2023-05-08 11:23:01,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:23:01,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +26: [2023-05-08 11:23:01,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +16: [2023-05-08 11:23:01,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:23:01,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:23:01,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +26: [2023-05-08 11:23:01,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +16: [2023-05-08 11:23:01,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 6: [2023-05-08 11:23:01,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +26: [2023-05-08 11:23:01,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +31: [2023-05-08 11:23:01,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +31: [2023-05-08 11:23:01,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +31: [2023-05-08 11:23:01,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +18: [2023-05-08 11:23:01,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:23:01,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:23:01,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:23:01,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 8: [2023-05-08 11:23:01,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 8: [2023-05-08 11:23:01,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 8: [2023-05-08 11:23:01,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 8: [2023-05-08 11:23:01,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 3: [2023-05-08 11:23:01,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +17: [2023-05-08 11:23:01,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +17: [2023-05-08 11:23:01,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 3: [2023-05-08 11:23:01,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 3: [2023-05-08 11:23:01,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 3: [2023-05-08 11:23:01,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +17: [2023-05-08 11:23:01,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +17: [2023-05-08 11:23:01,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +12: [2023-05-08 11:23:01,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +11: [2023-05-08 11:23:01,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +28: [2023-05-08 11:23:01,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:23:01,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 6: [2023-05-08 11:23:01,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +12: [2023-05-08 11:23:01,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +28: [2023-05-08 11:23:01,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +28: [2023-05-08 11:23:01,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +12: [2023-05-08 11:23:01,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +11: [2023-05-08 11:23:01,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +28: [2023-05-08 11:23:01,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +12: [2023-05-08 11:23:01,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +11: [2023-05-08 11:23:01,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +11: [2023-05-08 11:23:01,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +29: [2023-05-08 11:23:01,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:23:01,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 6: [2023-05-08 11:23:01,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +11: [2023-05-08 11:23:01,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:23:01,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +11: [2023-05-08 11:23:01,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +11: [2023-05-08 11:23:01,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:23:01,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:23:01,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +11: [2023-05-08 11:23:01,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +11: [2023-05-08 11:23:01,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +11: [2023-05-08 11:23:01,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +11: [2023-05-08 11:23:01,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +11: [2023-05-08 11:23:01,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:23:01,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:23:01,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +31: [2023-05-08 11:23:01,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:23:01,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:23:01,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +16: [2023-05-08 11:23:01,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +16: [2023-05-08 11:23:01,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +18: [2023-05-08 11:23:01,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 2: [2023-05-08 11:23:01,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +16: [2023-05-08 11:23:01,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +16: [2023-05-08 11:23:01,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 8: [2023-05-08 11:23:01,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:23:01,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +11: [2023-05-08 11:23:01,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +11: [2023-05-08 11:23:01,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +29: [2023-05-08 11:23:01,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +29: [2023-05-08 11:23:01,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +29: [2023-05-08 11:23:01,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +29: [2023-05-08 11:23:01,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 3: [2023-05-08 11:23:01,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +11: [2023-05-08 11:23:01,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:23:01,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +16: [2023-05-08 11:23:01,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +20: [2023-05-08 11:23:01,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +20: [2023-05-08 11:23:01,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +17: [2023-05-08 11:23:01,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:23:01,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:23:01,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +17: [2023-05-08 11:23:01,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +17: [2023-05-08 11:23:01,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +17: [2023-05-08 11:23:01,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:23:01,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +12: [2023-05-08 11:23:01,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +16: [2023-05-08 11:23:01,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +18: [2023-05-08 11:23:01,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 0: [2023-05-08 11:23:01,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +18: [2023-05-08 11:23:01,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:23:01,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:23:01,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:23:01,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:23:01,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:23:01,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:23:01,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +18: [2023-05-08 11:23:01,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:23:01,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:23:01,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:23:01,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:23:01,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +30: [2023-05-08 11:23:01,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:23:01,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:23:01,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +30: [2023-05-08 11:23:01,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +30: [2023-05-08 11:23:01,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:23:01,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:23:01,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:23:01,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +20: [2023-05-08 11:23:01,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:23:01,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +16: [2023-05-08 11:23:01,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +29: [2023-05-08 11:23:01,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +18: [2023-05-08 11:23:01,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:23:01,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +16: [2023-05-08 11:23:01,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 0: [2023-05-08 11:23:01,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:23:01,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +11: [2023-05-08 11:23:01,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:23:01,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:23:01,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:23:01,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:23:01,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:23:01,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +20: [2023-05-08 11:23:01,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +11: [2023-05-08 11:23:01,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +20: [2023-05-08 11:23:01,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:23:01,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:23:01,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +12: [2023-05-08 11:23:01,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +12: [2023-05-08 11:23:01,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +12: [2023-05-08 11:23:01,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +12: [2023-05-08 11:23:01,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:23:01,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:23:01,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:23:01,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:23:01,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:23:01,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +11: [2023-05-08 11:23:01,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +12: [2023-05-08 11:23:01,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +11: [2023-05-08 11:23:01,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +11: [2023-05-08 11:23:01,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +12: [2023-05-08 11:23:01,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +12: [2023-05-08 11:23:01,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +30: [2023-05-08 11:23:01,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:23:01,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:23:01,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +15: [2023-05-08 11:23:01,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +11: [2023-05-08 11:23:01,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:23:01,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +15: [2023-05-08 11:23:01,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +15: [2023-05-08 11:23:01,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 9: [2023-05-08 11:23:01,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 9: [2023-05-08 11:23:01,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 9: [2023-05-08 11:23:01,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +13: [2023-05-08 11:23:01,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 9: [2023-05-08 11:23:01,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +13: [2023-05-08 11:23:01,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +13: [2023-05-08 11:23:01,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +13: [2023-05-08 11:23:01,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +11: [2023-05-08 11:23:01,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +11: [2023-05-08 11:23:01,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 3: [2023-05-08 11:23:01,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 3: [2023-05-08 11:23:01,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 3: [2023-05-08 11:23:01,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 3: [2023-05-08 11:23:01,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 3: [2023-05-08 11:23:01,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:23:01,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +20: [2023-05-08 11:23:01,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +20: [2023-05-08 11:23:01,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +20: [2023-05-08 11:23:01,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +30: [2023-05-08 11:23:01,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 8: [2023-05-08 11:23:01,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +30: [2023-05-08 11:23:01,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:23:01,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:23:01,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +12: [2023-05-08 11:23:01,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +15: [2023-05-08 11:23:01,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +11: [2023-05-08 11:23:01,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:23:01,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:23:01,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:23:01,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +12: [2023-05-08 11:23:01,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +12: [2023-05-08 11:23:01,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +15: [2023-05-08 11:23:01,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:23:01,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +12: [2023-05-08 11:23:01,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +13: [2023-05-08 11:23:01,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +13: [2023-05-08 11:23:01,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +13: [2023-05-08 11:23:01,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +11: [2023-05-08 11:23:01,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +11: [2023-05-08 11:23:01,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +11: [2023-05-08 11:23:01,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:23:01,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +11: [2023-05-08 11:23:01,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +20: [2023-05-08 11:23:01,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:23:01,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:23:01,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +20: [2023-05-08 11:23:01,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 9: [2023-05-08 11:23:01,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:23:01,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +25: [2023-05-08 11:23:01,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +25: [2023-05-08 11:23:01,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +25: [2023-05-08 11:23:01,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 3: [2023-05-08 11:23:01,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:23:01,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +25: [2023-05-08 11:23:01,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +30: [2023-05-08 11:23:01,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:23:01,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +12: [2023-05-08 11:23:01,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 3: [2023-05-08 11:23:01,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +12: [2023-05-08 11:23:01,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +12: [2023-05-08 11:23:01,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:23:01,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +11: [2023-05-08 11:23:01,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +12: [2023-05-08 11:23:01,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +12: [2023-05-08 11:23:01,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +24: [2023-05-08 11:23:01,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +24: [2023-05-08 11:23:01,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +24: [2023-05-08 11:23:01,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +24: [2023-05-08 11:23:01,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +24: [2023-05-08 11:23:01,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +24: [2023-05-08 11:23:01,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:23:01,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +31: [2023-05-08 11:23:01,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:23:01,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:23:01,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:23:01,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +29: [2023-05-08 11:23:01,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +29: [2023-05-08 11:23:01,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +31: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:23:01,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +31: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +31: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +29: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +31: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +31: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +31: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +31: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +30: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +31: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +31: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +31: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +31: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +17: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +17: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +17: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +17: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +17: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +17: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +17: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +17: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +17: [2023-05-08 11:23:01,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +17: [2023-05-08 11:23:01,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +17: [2023-05-08 11:23:01,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +24: [2023-05-08 11:23:01,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +24: [2023-05-08 11:23:01,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +17: [2023-05-08 11:23:01,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +17: [2023-05-08 11:23:01,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +15: [2023-05-08 11:23:01,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +17: [2023-05-08 11:23:01,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +17: [2023-05-08 11:23:01,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:23:01,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:23:01,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +16: [2023-05-08 11:23:01,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +17: [2023-05-08 11:23:01,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +16: [2023-05-08 11:23:01,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:23:01,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:23:01,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +16: [2023-05-08 11:23:01,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +16: [2023-05-08 11:23:01,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 4: [2023-05-08 11:23:01,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +16: [2023-05-08 11:23:01,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +16: [2023-05-08 11:23:01,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +16: [2023-05-08 11:23:01,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:23:01,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:23:01,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:23:01,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:23:01,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +15: [2023-05-08 11:23:01,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +15: [2023-05-08 11:23:01,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +13: [2023-05-08 11:23:01,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +15: [2023-05-08 11:23:01,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:23:01,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 4: [2023-05-08 11:23:01,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 4: [2023-05-08 11:23:01,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +25: [2023-05-08 11:23:01,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:23:01,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +25: [2023-05-08 11:23:01,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +13: [2023-05-08 11:23:01,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:23:01,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +20: [2023-05-08 11:23:01,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +19: [2023-05-08 11:23:01,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +19: [2023-05-08 11:23:01,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +19: [2023-05-08 11:23:01,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +27: [2023-05-08 11:23:01,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +27: [2023-05-08 11:23:01,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +27: [2023-05-08 11:23:01,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +27: [2023-05-08 11:23:01,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 8: [2023-05-08 11:23:01,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +10: [2023-05-08 11:23:01,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +10: [2023-05-08 11:23:01,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +10: [2023-05-08 11:23:01,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +20: [2023-05-08 11:23:01,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +10: [2023-05-08 11:23:01,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 3: [2023-05-08 11:23:01,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:23:01,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:23:01,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +19: [2023-05-08 11:23:01,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 4: [2023-05-08 11:23:01,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +24: [2023-05-08 11:23:01,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +24: [2023-05-08 11:23:01,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +17: [2023-05-08 11:23:01,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +16: [2023-05-08 11:23:01,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +30: [2023-05-08 11:23:01,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +30: [2023-05-08 11:23:01,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. + 4: [2023-05-08 11:23:01,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:23:01,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:23:01,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:23:01,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:23:01,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:23:01,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:23:01,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:23:01,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:23:01,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:23:01,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:23:01,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:23:01,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:23:01,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:23:01,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:23:01,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:23:01,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:23:01,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +27: [2023-05-08 11:23:01,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:23:01,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +27: [2023-05-08 11:23:01,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:23:01,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:23:01,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +30: [2023-05-08 11:23:01,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +19: [2023-05-08 11:23:01,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:23:01,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:23:01,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:23:01,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +13: [2023-05-08 11:23:01,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +25: [2023-05-08 11:23:01,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +30: [2023-05-08 11:23:01,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt. +31: [2023-05-08 11:23:01,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:23:01,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:23:01,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:23:01,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:23:01,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +14: [2023-05-08 11:23:01,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +14: [2023-05-08 11:23:01,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +14: [2023-05-08 11:23:01,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +17: [2023-05-08 11:23:01,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:23:01,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +14: [2023-05-08 11:23:01,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +24: [2023-05-08 11:23:01,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:23:01,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:23:01,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:23:01,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +16: [2023-05-08 11:23:01,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +16: [2023-05-08 11:23:01,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:23:01,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:23:01,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:23:01,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:23:01,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:23:01,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:23:01,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +17: [2023-05-08 11:23:01,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 7: [2023-05-08 11:23:01,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 8: [2023-05-08 11:23:01,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:23:01,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:23:01,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +17: [2023-05-08 11:23:01,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:23:01,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:23:01,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:23:01,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:23:01,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +31: [2023-05-08 11:23:01,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:23:01,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:23:01,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:23:01,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:23:01,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:23:01,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:23:01,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +24: [2023-05-08 11:23:01,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +24: [2023-05-08 11:23:01,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:23:01,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:23:01,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +25: [2023-05-08 11:23:01,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +25: [2023-05-08 11:23:01,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +25: [2023-05-08 11:23:01,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +14: [2023-05-08 11:23:01,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +16: [2023-05-08 11:23:01,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:23:01,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:23:01,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:23:01,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:23:01,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:23:01,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:23:01,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:23:01,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:23:01,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:23:01,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:23:01,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +31: [2023-05-08 11:23:01,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:23:01,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +27: [2023-05-08 11:23:01,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +27: [2023-05-08 11:23:01,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +19: [2023-05-08 11:23:01,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:23:01,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:23:01,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +19: [2023-05-08 11:23:01,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +17: [2023-05-08 11:23:01,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:23:01,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +30: [2023-05-08 11:23:01,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +27: [2023-05-08 11:23:01,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +17: [2023-05-08 11:23:01,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:23:01,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:23:01,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:23:01,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:23:01,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +27: [2023-05-08 11:23:01,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... + 7: [2023-05-08 11:23:01,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +10: [2023-05-08 11:23:01,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +10: [2023-05-08 11:23:01,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +10: [2023-05-08 11:23:01,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +30: [2023-05-08 11:23:01,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +30: [2023-05-08 11:23:01,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +24: [2023-05-08 11:23:01,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +24: [2023-05-08 11:23:01,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:23:01,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:23:01,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +13: [2023-05-08 11:23:01,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:23:01,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +30: [2023-05-08 11:23:01,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +30: [2023-05-08 11:23:01,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +30: [2023-05-08 11:23:01,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +10: [2023-05-08 11:23:01,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:23:01,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +13: [2023-05-08 11:23:01,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:23:01,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:23:01,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:23:01,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt. +27: [2023-05-08 11:23:01,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +19: [2023-05-08 11:23:01,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt... +15: [2023-05-08 11:23:01,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +15: [2023-05-08 11:23:01,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:23:01,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +15: [2023-05-08 11:23:01,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:23:01,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +15: [2023-05-08 11:23:01,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +15: [2023-05-08 11:23:01,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:23:01,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:23:01,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +15: [2023-05-08 11:23:01,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:23:01,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +15: [2023-05-08 11:23:01,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +21: [2023-05-08 11:23:01,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +21: [2023-05-08 11:23:01,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +21: [2023-05-08 11:23:01,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +15: [2023-05-08 11:23:01,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:23:01,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +15: [2023-05-08 11:23:01,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 1: [2023-05-08 11:23:01,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 1: [2023-05-08 11:23:01,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 1: [2023-05-08 11:23:01,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +15: [2023-05-08 11:23:01,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:23:01,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 6: [2023-05-08 11:23:01,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 6: [2023-05-08 11:23:01,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 6: [2023-05-08 11:23:01,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 2: [2023-05-08 11:23:01,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +21: [2023-05-08 11:23:01,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:23:01,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:23:01,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:23:01,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 5: [2023-05-08 11:23:01,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +28: [2023-05-08 11:23:01,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 6: [2023-05-08 11:23:01,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:23:01,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +28: [2023-05-08 11:23:01,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +28: [2023-05-08 11:23:01,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 6: [2023-05-08 11:23:01,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 6: [2023-05-08 11:23:01,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:23:01,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 1: [2023-05-08 11:23:01,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +14: [2023-05-08 11:23:01,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +14: [2023-05-08 11:23:01,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +14: [2023-05-08 11:23:01,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:23:01,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:23:01,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:23:01,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +14: [2023-05-08 11:23:01,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +14: [2023-05-08 11:23:01,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +14: [2023-05-08 11:23:01,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +14: [2023-05-08 11:23:01,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +14: [2023-05-08 11:23:01,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +14: [2023-05-08 11:23:01,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +14: [2023-05-08 11:23:01,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +14: [2023-05-08 11:23:01,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:23:01,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +14: [2023-05-08 11:23:01,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:23:01,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 5: [2023-05-08 11:23:01,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:23:01,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 0: [2023-05-08 11:23:01,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 0: [2023-05-08 11:23:01,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 0: [2023-05-08 11:23:01,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 5: [2023-05-08 11:23:01,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:23:01,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:23:01,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:23:01,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 2: [2023-05-08 11:23:01,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:23:01,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +14: [2023-05-08 11:23:01,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +14: [2023-05-08 11:23:01,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:23:01,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 9: [2023-05-08 11:23:01,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 9: [2023-05-08 11:23:01,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 9: [2023-05-08 11:23:01,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:23:01,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 9: [2023-05-08 11:23:01,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:23:01,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:23:01,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:23:01,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:23:01,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 0: [2023-05-08 11:23:01,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +23: [2023-05-08 11:23:01,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +23: [2023-05-08 11:23:01,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +23: [2023-05-08 11:23:01,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +23: [2023-05-08 11:23:01,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +23: [2023-05-08 11:23:01,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +23: [2023-05-08 11:23:01,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +23: [2023-05-08 11:23:01,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +23: [2023-05-08 11:23:01,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:23:01,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +14: [2023-05-08 11:23:01,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:23:01,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:23:01,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:23:01,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:23:01,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +22: [2023-05-08 11:23:01,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +22: [2023-05-08 11:23:01,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:23:01,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:23:01,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:23:01,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:23:01,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:23:01,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:23:01,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:23:01,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:23:01,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:23:01,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:23:02,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:23:02,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:23:02,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:23:02,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:23:02,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:23:02,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:23:02,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:23:02,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:23:02,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:23:02,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +28: [2023-05-08 11:23:02,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:23:02,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +28: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +27: [2023-05-08 11:23:02,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:23:02,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:23:02,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +27: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +27: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +27: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 6: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 5: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 5: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 6: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 5: [2023-05-08 11:23:02,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 5: [2023-05-08 11:23:02,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +21: [2023-05-08 11:23:02,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +21: [2023-05-08 11:23:02,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:23:02,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +10: [2023-05-08 11:23:02,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +10: [2023-05-08 11:23:02,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +10: [2023-05-08 11:23:02,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +10: [2023-05-08 11:23:02,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:23:02,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +21: [2023-05-08 11:23:02,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +21: [2023-05-08 11:23:02,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:23:02,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +21: [2023-05-08 11:23:02,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +21: [2023-05-08 11:23:02,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +21: [2023-05-08 11:23:02,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +28: [2023-05-08 11:23:02,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +28: [2023-05-08 11:23:02,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +28: [2023-05-08 11:23:02,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:23:02,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:23:02,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +21: [2023-05-08 11:23:02,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +21: [2023-05-08 11:23:02,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 5: [2023-05-08 11:23:02,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +28: [2023-05-08 11:23:02,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 5: [2023-05-08 11:23:02,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 1: [2023-05-08 11:23:02,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +10: [2023-05-08 11:23:02,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +10: [2023-05-08 11:23:02,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +10: [2023-05-08 11:23:02,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:23:02,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +21: [2023-05-08 11:23:02,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:23:02,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +28: [2023-05-08 11:23:02,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:23:02,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:23:02,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +10: [2023-05-08 11:23:02,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +28: [2023-05-08 11:23:02,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:23:02,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +18: [2023-05-08 11:23:02,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +28: [2023-05-08 11:23:02,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +18: [2023-05-08 11:23:02,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 9: [2023-05-08 11:23:02,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:23:02,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +28: [2023-05-08 11:23:02,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 0: [2023-05-08 11:23:02,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:23:02,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:23:02,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:23:02,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +23: [2023-05-08 11:23:02,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +26: [2023-05-08 11:23:02,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +26: [2023-05-08 11:23:02,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +26: [2023-05-08 11:23:02,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +29: [2023-05-08 11:23:02,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +29: [2023-05-08 11:23:02,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +29: [2023-05-08 11:23:02,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +29: [2023-05-08 11:23:02,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +23: [2023-05-08 11:23:02,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +23: [2023-05-08 11:23:02,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +23: [2023-05-08 11:23:02,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +23: [2023-05-08 11:23:02,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +23: [2023-05-08 11:23:02,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 9: [2023-05-08 11:23:02,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:23:02,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +23: [2023-05-08 11:23:02,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +28: [2023-05-08 11:23:02,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +14: [2023-05-08 11:23:02,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:23:02,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +24: [2023-05-08 11:23:02,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +24: [2023-05-08 11:23:02,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +24: [2023-05-08 11:23:02,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +24: [2023-05-08 11:23:02,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 9: [2023-05-08 11:23:02,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +26: [2023-05-08 11:23:02,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +18: [2023-05-08 11:23:02,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:23:02,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:23:02,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:23:02,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +10: [2023-05-08 11:23:02,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:23:02,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:23:02,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:23:02,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:23:02,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 6: [2023-05-08 11:23:02,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:23:02,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 0: [2023-05-08 11:23:02,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +20: [2023-05-08 11:23:02,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +20: [2023-05-08 11:23:02,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +21: [2023-05-08 11:23:02,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +21: [2023-05-08 11:23:02,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 2: [2023-05-08 11:23:02,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 9: [2023-05-08 11:23:02,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 5: [2023-05-08 11:23:02,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:23:02,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:23:02,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:23:02,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:23:02,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +22: [2023-05-08 11:23:02,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:23:02,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +26: [2023-05-08 11:23:02,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +22: [2023-05-08 11:23:02,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +28: [2023-05-08 11:23:02,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:23:02,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:23:02,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +10: [2023-05-08 11:23:02,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +10: [2023-05-08 11:23:02,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:23:02,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:23:02,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +22: [2023-05-08 11:23:02,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 5: [2023-05-08 11:23:02,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:23:02,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +23: [2023-05-08 11:23:02,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:23:02,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +22: [2023-05-08 11:23:02,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 7: [2023-05-08 11:23:02,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 7: [2023-05-08 11:23:02,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 7: [2023-05-08 11:23:02,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +28: [2023-05-08 11:23:02,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +24: [2023-05-08 11:23:02,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +29: [2023-05-08 11:23:02,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +29: [2023-05-08 11:23:02,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +29: [2023-05-08 11:23:02,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +29: [2023-05-08 11:23:02,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +16: [2023-05-08 11:23:02,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +16: [2023-05-08 11:23:02,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +10: [2023-05-08 11:23:02,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +24: [2023-05-08 11:23:02,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:23:02,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:23:02,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:23:02,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +24: [2023-05-08 11:23:02,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +23: [2023-05-08 11:23:02,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:23:02,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +16: [2023-05-08 11:23:02,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 0: [2023-05-08 11:23:02,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +28: [2023-05-08 11:23:02,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +23: [2023-05-08 11:23:02,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:23:02,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:23:02,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:23:02,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:23:02,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +24: [2023-05-08 11:23:02,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:23:02,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +21: [2023-05-08 11:23:02,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:23:02,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:23:02,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:23:02,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:23:02,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +20: [2023-05-08 11:23:02,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:23:02,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:23:02,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:23:02,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:23:02,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:23:02,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +23: [2023-05-08 11:23:02,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 2: [2023-05-08 11:23:02,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +22: [2023-05-08 11:23:02,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +19: [2023-05-08 11:23:02,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:23:02,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +10: [2023-05-08 11:23:02,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:23:02,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:23:02,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +26: [2023-05-08 11:23:02,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:23:02,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:23:02,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:23:02,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:23:02,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:23:02,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:23:02,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +10: [2023-05-08 11:23:02,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:23:02,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 5: [2023-05-08 11:23:02,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +19: [2023-05-08 11:23:02,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:23:02,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:23:02,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +16: [2023-05-08 11:23:02,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:23:02,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +16: [2023-05-08 11:23:02,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +16: [2023-05-08 11:23:02,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +16: [2023-05-08 11:23:02,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:23:02,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +21: [2023-05-08 11:23:02,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:23:02,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +23: [2023-05-08 11:23:02,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +21: [2023-05-08 11:23:02,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +23: [2023-05-08 11:23:02,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +23: [2023-05-08 11:23:02,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +26: [2023-05-08 11:23:02,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:23:02,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:23:02,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +26: [2023-05-08 11:23:02,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +22: [2023-05-08 11:23:02,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +20: [2023-05-08 11:23:02,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:23:02,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:23:02,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +10: [2023-05-08 11:23:02,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +23: [2023-05-08 11:23:02,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:23:02,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:23:02,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 7: [2023-05-08 11:23:02,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:23:02,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 4: [2023-05-08 11:23:02,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:23:02,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:23:02,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +24: [2023-05-08 11:23:02,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:23:02,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:23:02,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +24: [2023-05-08 11:23:02,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +24: [2023-05-08 11:23:02,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +24: [2023-05-08 11:23:02,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +26: [2023-05-08 11:23:02,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +26: [2023-05-08 11:23:02,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +26: [2023-05-08 11:23:02,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +26: [2023-05-08 11:23:02,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:23:02,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:23:02,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:23:02,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +20: [2023-05-08 11:23:02,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:23:02,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:23:02,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:23:02,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +20: [2023-05-08 11:23:02,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +29: [2023-05-08 11:23:02,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +28: [2023-05-08 11:23:02,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:23:02,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +21: [2023-05-08 11:23:02,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +24: [2023-05-08 11:23:02,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +24: [2023-05-08 11:23:02,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +29: [2023-05-08 11:23:02,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 2: [2023-05-08 11:23:02,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +29: [2023-05-08 11:23:02,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:23:02,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +10: [2023-05-08 11:23:02,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:23:02,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +29: [2023-05-08 11:23:02,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:23:02,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 4: [2023-05-08 11:23:02,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:23:02,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:23:02,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +16: [2023-05-08 11:23:02,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:23:02,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +13: [2023-05-08 11:23:02,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +22: [2023-05-08 11:23:02,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:23:02,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 7: [2023-05-08 11:23:02,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +16: [2023-05-08 11:23:02,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +13: [2023-05-08 11:23:02,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +16: [2023-05-08 11:23:02,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 4: [2023-05-08 11:23:02,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +15: [2023-05-08 11:23:02,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +15: [2023-05-08 11:23:02,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +15: [2023-05-08 11:23:02,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 4: [2023-05-08 11:23:02,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +15: [2023-05-08 11:23:02,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 4: [2023-05-08 11:23:02,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +19: [2023-05-08 11:23:02,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 7: [2023-05-08 11:23:02,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +19: [2023-05-08 11:23:02,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 2: [2023-05-08 11:23:02,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:23:02,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 2: [2023-05-08 11:23:02,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +19: [2023-05-08 11:23:02,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 4: [2023-05-08 11:23:02,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 4: [2023-05-08 11:23:02,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:23:02,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +30: [2023-05-08 11:23:02,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +30: [2023-05-08 11:23:02,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +30: [2023-05-08 11:23:02,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +24: [2023-05-08 11:23:02,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:23:02,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:23:02,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +16: [2023-05-08 11:23:02,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +16: [2023-05-08 11:23:02,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +26: [2023-05-08 11:23:02,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:23:02,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +24: [2023-05-08 11:23:02,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +23: [2023-05-08 11:23:02,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:23:02,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:23:02,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +16: [2023-05-08 11:23:02,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +15: [2023-05-08 11:23:02,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:23:02,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:23:02,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +15: [2023-05-08 11:23:02,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +23: [2023-05-08 11:23:02,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +16: [2023-05-08 11:23:02,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +19: [2023-05-08 11:23:02,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:23:02,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 7: [2023-05-08 11:23:02,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +15: [2023-05-08 11:23:02,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:23:02,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:23:02,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:23:02,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +30: [2023-05-08 11:23:02,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +30: [2023-05-08 11:23:02,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +30: [2023-05-08 11:23:02,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +30: [2023-05-08 11:23:02,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:23:02,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:23:02,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:23:02,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +13: [2023-05-08 11:23:02,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +15: [2023-05-08 11:23:02,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:23:02,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:23:02,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +15: [2023-05-08 11:23:02,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +13: [2023-05-08 11:23:02,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +13: [2023-05-08 11:23:02,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +30: [2023-05-08 11:23:02,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:23:02,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 4: [2023-05-08 11:23:02,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +30: [2023-05-08 11:23:02,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:23:02,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +19: [2023-05-08 11:23:02,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:23:02,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:23:02,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +19: [2023-05-08 11:23:02,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +27: [2023-05-08 11:23:02,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +27: [2023-05-08 11:23:02,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +15: [2023-05-08 11:23:02,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +15: [2023-05-08 11:23:02,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +27: [2023-05-08 11:23:02,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +25: [2023-05-08 11:23:02,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +25: [2023-05-08 11:23:02,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +25: [2023-05-08 11:23:02,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +27: [2023-05-08 11:23:02,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +25: [2023-05-08 11:23:02,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. + 4: [2023-05-08 11:23:02,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 4: [2023-05-08 11:23:02,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +10: [2023-05-08 11:23:02,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +10: [2023-05-08 11:23:02,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +10: [2023-05-08 11:23:02,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +15: [2023-05-08 11:23:02,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:23:02,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:23:02,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:23:02,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 4: [2023-05-08 11:23:02,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +15: [2023-05-08 11:23:02,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 8: [2023-05-08 11:23:02,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 8: [2023-05-08 11:23:02,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 8: [2023-05-08 11:23:02,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 4: [2023-05-08 11:23:02,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... + 8: [2023-05-08 11:23:02,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +27: [2023-05-08 11:23:02,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:23:02,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:23:02,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +10: [2023-05-08 11:23:02,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt. +27: [2023-05-08 11:23:02,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +30: [2023-05-08 11:23:02,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +25: [2023-05-08 11:23:02,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:23:02,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +15: [2023-05-08 11:23:02,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +27: [2023-05-08 11:23:02,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +25: [2023-05-08 11:23:02,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +25: [2023-05-08 11:23:02,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +25: [2023-05-08 11:23:02,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:23:02,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +30: [2023-05-08 11:23:02,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:23:02,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +10: [2023-05-08 11:23:02,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +10: [2023-05-08 11:23:02,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... +10: [2023-05-08 11:23:02,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:23:02,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:23:02,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +10: [2023-05-08 11:23:02,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:23:02,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:23:02,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +27: [2023-05-08 11:23:02,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:23:02,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +27: [2023-05-08 11:23:02,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:23:02,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +25: [2023-05-08 11:23:02,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +25: [2023-05-08 11:23:02,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +10: [2023-05-08 11:23:02,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +25: [2023-05-08 11:23:02,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +25: [2023-05-08 11:23:02,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +10: [2023-05-08 11:23:02,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +25: [2023-05-08 11:23:02,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +25: [2023-05-08 11:23:02,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +10: [2023-05-08 11:23:02,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +10: [2023-05-08 11:23:02,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +10: [2023-05-08 11:23:02,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +10: [2023-05-08 11:23:02,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +18: [2023-05-08 11:23:02,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:23:02,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:23:02,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:23:02,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +25: [2023-05-08 11:23:02,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:23:02,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:23:02,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:23:02,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:23:02,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +25: [2023-05-08 11:23:02,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +10: [2023-05-08 11:23:02,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:23:02,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:23:02,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt... +18: [2023-05-08 11:23:02,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:23:02,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:23:02,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:23:02,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:23:02,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:23:02,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:23:02,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 3: [2023-05-08 11:23:02,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:23:02,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:23:02,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:23:02,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:23:02,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:23:02,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:23:02,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:23:02,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:23:02,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +11: [2023-05-08 11:23:02,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:23:02,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +11: [2023-05-08 11:23:02,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +11: [2023-05-08 11:23:02,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:23:02,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +11: [2023-05-08 11:23:02,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:23:02,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:23:02,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 3: [2023-05-08 11:23:02,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 3: [2023-05-08 11:23:02,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 3: [2023-05-08 11:23:02,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 7: [2023-05-08 11:23:02,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:23:02,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:23:02,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:23:02,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +11: [2023-05-08 11:23:02,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +11: [2023-05-08 11:23:02,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +11: [2023-05-08 11:23:02,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 7: [2023-05-08 11:23:02,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:23:02,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 7: [2023-05-08 11:23:02,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +11: [2023-05-08 11:23:02,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 3: [2023-05-08 11:23:02,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:23:02,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:23:02,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:23:02,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 3: [2023-05-08 11:23:02,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:23:02,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +11: [2023-05-08 11:23:02,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:23:02,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 3: [2023-05-08 11:23:02,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:23:02,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +12: [2023-05-08 11:23:02,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +12: [2023-05-08 11:23:02,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +12: [2023-05-08 11:23:02,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 3: [2023-05-08 11:23:02,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:23:02,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 7: [2023-05-08 11:23:02,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 3: [2023-05-08 11:23:02,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:23:02,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:23:02,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:23:02,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:23:02,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:23:02,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +12: [2023-05-08 11:23:02,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +12: [2023-05-08 11:23:02,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:23:02,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:23:02,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +12: [2023-05-08 11:23:02,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +12: [2023-05-08 11:23:02,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:23:02,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +11: [2023-05-08 11:23:02,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:23:02,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +17: [2023-05-08 11:23:02,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +17: [2023-05-08 11:23:02,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +17: [2023-05-08 11:23:02,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +17: [2023-05-08 11:23:02,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 3: [2023-05-08 11:23:02,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:23:02,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:23:02,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 3: [2023-05-08 11:23:02,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 3: [2023-05-08 11:23:02,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:23:02,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 3: [2023-05-08 11:23:02,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +11: [2023-05-08 11:23:02,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +11: [2023-05-08 11:23:02,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 3: [2023-05-08 11:23:02,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +17: [2023-05-08 11:23:02,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:23:02,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:23:02,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:23:02,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:23:02,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:23:02,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +25: [2023-05-08 11:23:02,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:23:02,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +25: [2023-05-08 11:23:02,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:23:02,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:23:02,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:23:02,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +25: [2023-05-08 11:23:02,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +25: [2023-05-08 11:23:02,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:23:02,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +11: [2023-05-08 11:23:02,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +25: [2023-05-08 11:23:02,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:23:02,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:23:02,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +31: [2023-05-08 11:23:02,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +31: [2023-05-08 11:23:02,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +31: [2023-05-08 11:23:02,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +31: [2023-05-08 11:23:02,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +25: [2023-05-08 11:23:02,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +25: [2023-05-08 11:23:02,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +25: [2023-05-08 11:23:02,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +31: [2023-05-08 11:23:02,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +31: [2023-05-08 11:23:02,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +31: [2023-05-08 11:23:02,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +31: [2023-05-08 11:23:02,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:23:02,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +30: [2023-05-08 11:23:02,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:23:02,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +30: [2023-05-08 11:23:02,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +30: [2023-05-08 11:23:02,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:23:02,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:23:02,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +30: [2023-05-08 11:23:02,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +25: [2023-05-08 11:23:02,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:23:02,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:23:02,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:23:02,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:23:02,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:23:02,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:23:02,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:23:02,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +12: [2023-05-08 11:23:02,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:23:02,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +12: [2023-05-08 11:23:02,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +12: [2023-05-08 11:23:02,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +12: [2023-05-08 11:23:02,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +12: [2023-05-08 11:23:02,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +12: [2023-05-08 11:23:02,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:23:02,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:23:02,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:23:02,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +14: [2023-05-08 11:23:02,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +14: [2023-05-08 11:23:02,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +20: [2023-05-08 11:23:02,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +20: [2023-05-08 11:23:02,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +20: [2023-05-08 11:23:02,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:23:02,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +12: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 8: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 8: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 8: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 8: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 8: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 8: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 8: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +20: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +20: [2023-05-08 11:23:02,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:23:02,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:23:02,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:23:02,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:23:02,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +20: [2023-05-08 11:23:02,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:23:02,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +20: [2023-05-08 11:23:02,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 8: [2023-05-08 11:23:02,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 9: [2023-05-08 11:23:02,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +25: [2023-05-08 11:23:02,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +17: [2023-05-08 11:23:02,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +17: [2023-05-08 11:23:02,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 8: [2023-05-08 11:23:02,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:23:02,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 8: [2023-05-08 11:23:02,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 8: [2023-05-08 11:23:02,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:23:02,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +20: [2023-05-08 11:23:02,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:23:02,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +15: [2023-05-08 11:23:02,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +15: [2023-05-08 11:23:02,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:23:02,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +15: [2023-05-08 11:23:02,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +15: [2023-05-08 11:23:02,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:23:02,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +15: [2023-05-08 11:23:02,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:23:02,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:23:02,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +30: [2023-05-08 11:23:02,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +13: [2023-05-08 11:23:02,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +13: [2023-05-08 11:23:02,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:23:02,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +13: [2023-05-08 11:23:02,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:23:02,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +13: [2023-05-08 11:23:02,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +30: [2023-05-08 11:23:02,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:23:02,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:23:02,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +12: [2023-05-08 11:23:02,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:23:02,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 8: [2023-05-08 11:23:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +20: [2023-05-08 11:23:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:23:02,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +20: [2023-05-08 11:23:02,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:23:02,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +24: [2023-05-08 11:23:02,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:23:02,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:23:02,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +24: [2023-05-08 11:23:02,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +14: [2023-05-08 11:23:02,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:23:02,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:23:02,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:23:02,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:23:02,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:23:02,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:23:02,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +15: [2023-05-08 11:23:02,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +30: [2023-05-08 11:23:02,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:23:02,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +15: [2023-05-08 11:23:02,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:23:02,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +20: [2023-05-08 11:23:02,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +13: [2023-05-08 11:23:02,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:23:02,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +31: [2023-05-08 11:23:02,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +31: [2023-05-08 11:23:02,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +31: [2023-05-08 11:23:02,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +31: [2023-05-08 11:23:02,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +31: [2023-05-08 11:23:02,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +31: [2023-05-08 11:23:02,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +31: [2023-05-08 11:23:02,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:23:02,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +31: [2023-05-08 11:23:02,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +31: [2023-05-08 11:23:02,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +31: [2023-05-08 11:23:02,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +31: [2023-05-08 11:23:02,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +31: [2023-05-08 11:23:02,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +31: [2023-05-08 11:23:02,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +31: [2023-05-08 11:23:02,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +12: [2023-05-08 11:23:02,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +20: [2023-05-08 11:23:02,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +14: [2023-05-08 11:23:02,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +31: [2023-05-08 11:23:02,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:23:02,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:23:02,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +31: [2023-05-08 11:23:02,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +14: [2023-05-08 11:23:02,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:23:02,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +15: [2023-05-08 11:23:02,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +17: [2023-05-08 11:23:02,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:23:02,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:23:02,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 8: [2023-05-08 11:23:02,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +12: [2023-05-08 11:23:02,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:23:02,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +14: [2023-05-08 11:23:02,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:23:02,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +15: [2023-05-08 11:23:02,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 8: [2023-05-08 11:23:02,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:23:02,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +14: [2023-05-08 11:23:02,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +15: [2023-05-08 11:23:02,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +14: [2023-05-08 11:23:02,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +24: [2023-05-08 11:23:02,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:23:02,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +20: [2023-05-08 11:23:02,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +15: [2023-05-08 11:23:02,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +13: [2023-05-08 11:23:02,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:23:02,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 8: [2023-05-08 11:23:02,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +12: [2023-05-08 11:23:02,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +17: [2023-05-08 11:23:02,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:23:02,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +14: [2023-05-08 11:23:02,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:23:02,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +31: [2023-05-08 11:23:02,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:23:02,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:23:02,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +14: [2023-05-08 11:23:02,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:23:02,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:23:02,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +15: [2023-05-08 11:23:02,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:23:02,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 8: [2023-05-08 11:23:02,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +31: [2023-05-08 11:23:02,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +15: [2023-05-08 11:23:02,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +14: [2023-05-08 11:23:02,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +31: [2023-05-08 11:23:02,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:23:02,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:23:02,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:23:02,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:23:02,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +14: [2023-05-08 11:23:02,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +24: [2023-05-08 11:23:02,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +31: [2023-05-08 11:23:02,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:23:02,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:23:02,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:23:02,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:23:02,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +31: [2023-05-08 11:23:02,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +31: [2023-05-08 11:23:02,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +31: [2023-05-08 11:23:02,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:23:02,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:23:02,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +14: [2023-05-08 11:23:02,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:23:02,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:23:02,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +14: [2023-05-08 11:23:02,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:23:02,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +14: [2023-05-08 11:23:02,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 0: [2023-05-08 11:23:02,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +31: [2023-05-08 11:23:02,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 1: [2023-05-08 11:23:02,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 0: [2023-05-08 11:23:02,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 9: [2023-05-08 11:23:02,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 9: [2023-05-08 11:23:02,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 9: [2023-05-08 11:23:02,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 9: [2023-05-08 11:23:02,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 1: [2023-05-08 11:23:02,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 6: [2023-05-08 11:23:02,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 6: [2023-05-08 11:23:02,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 6: [2023-05-08 11:23:02,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +19: [2023-05-08 11:23:02,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +19: [2023-05-08 11:23:02,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +19: [2023-05-08 11:23:02,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +19: [2023-05-08 11:23:02,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 1: [2023-05-08 11:23:02,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:23:02,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:23:02,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:23:02,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:23:02,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:23:02,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:23:02,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:23:02,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:23:02,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +19: [2023-05-08 11:23:02,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +19: [2023-05-08 11:23:02,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +19: [2023-05-08 11:23:02,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +19: [2023-05-08 11:23:02,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:23:02,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +18: [2023-05-08 11:23:02,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +18: [2023-05-08 11:23:02,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +18: [2023-05-08 11:23:02,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 1: [2023-05-08 11:23:02,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 1: [2023-05-08 11:23:02,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +18: [2023-05-08 11:23:02,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 1: [2023-05-08 11:23:02,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 9: [2023-05-08 11:23:02,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +13: [2023-05-08 11:23:02,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +13: [2023-05-08 11:23:02,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +13: [2023-05-08 11:23:02,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +13: [2023-05-08 11:23:02,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +16: [2023-05-08 11:23:02,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:23:02,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:23:02,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:23:02,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 4: [2023-05-08 11:23:02,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 4: [2023-05-08 11:23:02,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 4: [2023-05-08 11:23:02,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 5: [2023-05-08 11:23:02,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:23:02,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:23:02,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +29: [2023-05-08 11:23:02,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +29: [2023-05-08 11:23:02,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +29: [2023-05-08 11:23:02,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:23:02,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:23:02,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:23:02,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:23:02,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:23:02,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +22: [2023-05-08 11:23:02,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +22: [2023-05-08 11:23:02,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:23:02,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:23:02,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +22: [2023-05-08 11:23:02,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +22: [2023-05-08 11:23:02,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:23:02,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:23:02,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:23:02,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:23:02,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:23:02,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:23:02,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:23:02,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:23:02,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:23:02,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:23:02,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:23:02,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +10: [2023-05-08 11:23:02,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:23:02,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:23:02,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:23:02,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:23:02,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +13: [2023-05-08 11:23:02,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:23:02,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:23:02,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:23:02,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:23:02,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:23:02,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +21: [2023-05-08 11:23:02,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:23:02,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:23:02,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:23:02,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:23:02,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:23:02,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +21: [2023-05-08 11:23:02,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:23:02,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +10: [2023-05-08 11:23:02,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +10: [2023-05-08 11:23:02,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:23:02,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:23:02,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:23:02,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:23:02,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +16: [2023-05-08 11:23:02,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +16: [2023-05-08 11:23:02,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +16: [2023-05-08 11:23:02,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 4: [2023-05-08 11:23:02,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 6: [2023-05-08 11:23:02,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 6: [2023-05-08 11:23:02,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 6: [2023-05-08 11:23:02,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 6: [2023-05-08 11:23:02,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:23:02,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +29: [2023-05-08 11:23:02,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:23:02,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +29: [2023-05-08 11:23:02,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +29: [2023-05-08 11:23:02,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 0: [2023-05-08 11:23:02,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 0: [2023-05-08 11:23:02,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +28: [2023-05-08 11:23:02,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:23:02,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +18: [2023-05-08 11:23:02,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +21: [2023-05-08 11:23:02,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +21: [2023-05-08 11:23:02,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:23:02,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:23:02,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:23:02,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:23:02,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 1: [2023-05-08 11:23:02,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +13: [2023-05-08 11:23:02,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +13: [2023-05-08 11:23:02,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +13: [2023-05-08 11:23:02,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 1: [2023-05-08 11:23:02,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 1: [2023-05-08 11:23:02,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:23:02,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +16: [2023-05-08 11:23:02,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:23:02,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:23:02,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:23:02,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +21: [2023-05-08 11:23:02,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:23:02,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 5: [2023-05-08 11:23:02,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 5: [2023-05-08 11:23:02,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +18: [2023-05-08 11:23:02,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:23:02,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 5: [2023-05-08 11:23:02,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 2: [2023-05-08 11:23:02,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 6: [2023-05-08 11:23:02,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 2: [2023-05-08 11:23:02,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 2: [2023-05-08 11:23:02,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +22: [2023-05-08 11:23:02,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +22: [2023-05-08 11:23:02,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +28: [2023-05-08 11:23:02,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 1: [2023-05-08 11:23:02,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +28: [2023-05-08 11:23:02,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:23:02,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +28: [2023-05-08 11:23:02,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +22: [2023-05-08 11:23:02,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +28: [2023-05-08 11:23:02,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +28: [2023-05-08 11:23:02,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:23:02,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:23:02,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +29: [2023-05-08 11:23:02,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:23:02,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +29: [2023-05-08 11:23:02,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:23:02,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:23:02,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +29: [2023-05-08 11:23:02,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 5: [2023-05-08 11:23:02,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:23:02,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:23:02,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:23:02,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:23:02,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +13: [2023-05-08 11:23:02,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 5: [2023-05-08 11:23:02,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:23:02,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +16: [2023-05-08 11:23:02,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +13: [2023-05-08 11:23:02,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +21: [2023-05-08 11:23:02,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:23:02,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:23:02,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 0: [2023-05-08 11:23:02,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +29: [2023-05-08 11:23:02,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +20: [2023-05-08 11:23:02,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 6: [2023-05-08 11:23:02,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:23:02,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +20: [2023-05-08 11:23:02,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +20: [2023-05-08 11:23:02,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 2: [2023-05-08 11:23:02,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:23:02,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +10: [2023-05-08 11:23:02,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:23:02,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:23:02,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:23:02,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:23:02,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:23:02,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:23:02,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 4: [2023-05-08 11:23:02,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:23:02,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:23:02,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:23:02,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:23:02,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:23:02,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:23:02,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:23:02,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +20: [2023-05-08 11:23:02,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +20: [2023-05-08 11:23:02,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:23:02,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:23:02,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +10: [2023-05-08 11:23:02,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:23:02,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:23:02,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:23:02,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +25: [2023-05-08 11:23:02,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +25: [2023-05-08 11:23:02,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +25: [2023-05-08 11:23:02,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +16: [2023-05-08 11:23:02,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 4: [2023-05-08 11:23:02,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +27: [2023-05-08 11:23:02,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +27: [2023-05-08 11:23:02,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +27: [2023-05-08 11:23:02,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +27: [2023-05-08 11:23:02,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:23:02,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +25: [2023-05-08 11:23:02,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 5: [2023-05-08 11:23:02,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 5: [2023-05-08 11:23:02,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +27: [2023-05-08 11:23:02,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +27: [2023-05-08 11:23:02,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:23:02,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 5: [2023-05-08 11:23:02,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +27: [2023-05-08 11:23:02,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 2: [2023-05-08 11:23:02,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +28: [2023-05-08 11:23:02,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:23:02,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +25: [2023-05-08 11:23:02,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:23:02,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:23:02,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:23:02,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +25: [2023-05-08 11:23:02,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +25: [2023-05-08 11:23:02,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:23:02,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +23: [2023-05-08 11:23:02,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +23: [2023-05-08 11:23:02,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:23:02,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:23:02,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +23: [2023-05-08 11:23:02,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +19: [2023-05-08 11:23:02,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +19: [2023-05-08 11:23:02,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +19: [2023-05-08 11:23:02,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +23: [2023-05-08 11:23:02,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +19: [2023-05-08 11:23:02,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +19: [2023-05-08 11:23:02,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +23: [2023-05-08 11:23:02,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +19: [2023-05-08 11:23:02,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +19: [2023-05-08 11:23:02,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +19: [2023-05-08 11:23:02,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +19: [2023-05-08 11:23:02,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +19: [2023-05-08 11:23:02,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +19: [2023-05-08 11:23:02,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +19: [2023-05-08 11:23:02,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +19: [2023-05-08 11:23:02,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +19: [2023-05-08 11:23:02,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +19: [2023-05-08 11:23:02,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 2: [2023-05-08 11:23:02,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +19: [2023-05-08 11:23:02,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +20: [2023-05-08 11:23:02,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +20: [2023-05-08 11:23:02,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +20: [2023-05-08 11:23:02,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:23:02,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:23:02,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:23:02,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +20: [2023-05-08 11:23:02,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +26: [2023-05-08 11:23:02,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +26: [2023-05-08 11:23:02,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +26: [2023-05-08 11:23:02,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:23:02,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:23:02,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:23:02,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +27: [2023-05-08 11:23:02,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:23:02,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +28: [2023-05-08 11:23:02,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +20: [2023-05-08 11:23:02,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +20: [2023-05-08 11:23:02,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +20: [2023-05-08 11:23:02,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +20: [2023-05-08 11:23:02,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +27: [2023-05-08 11:23:02,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:23:02,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:23:02,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:23:02,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +25: [2023-05-08 11:23:02,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +25: [2023-05-08 11:23:02,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:23:02,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:23:02,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +28: [2023-05-08 11:23:02,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +27: [2023-05-08 11:23:02,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +27: [2023-05-08 11:23:02,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +25: [2023-05-08 11:23:02,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +23: [2023-05-08 11:23:02,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +23: [2023-05-08 11:23:02,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:23:02,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +19: [2023-05-08 11:23:02,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:23:02,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +10: [2023-05-08 11:23:02,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +10: [2023-05-08 11:23:02,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +30: [2023-05-08 11:23:02,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +26: [2023-05-08 11:23:02,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +26: [2023-05-08 11:23:02,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +30: [2023-05-08 11:23:02,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 7: [2023-05-08 11:23:02,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +11: [2023-05-08 11:23:02,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +11: [2023-05-08 11:23:02,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +11: [2023-05-08 11:23:02,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +11: [2023-05-08 11:23:02,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 7: [2023-05-08 11:23:02,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +27: [2023-05-08 11:23:02,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:23:02,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +19: [2023-05-08 11:23:02,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:23:02,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +19: [2023-05-08 11:23:02,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +30: [2023-05-08 11:23:02,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +30: [2023-05-08 11:23:02,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +24: [2023-05-08 11:23:02,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +24: [2023-05-08 11:23:02,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +24: [2023-05-08 11:23:02,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +15: [2023-05-08 11:23:02,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +15: [2023-05-08 11:23:02,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +27: [2023-05-08 11:23:02,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +23: [2023-05-08 11:23:02,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:23:02,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +24: [2023-05-08 11:23:02,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +23: [2023-05-08 11:23:02,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:23:02,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +15: [2023-05-08 11:23:02,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 7: [2023-05-08 11:23:02,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +27: [2023-05-08 11:23:02,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +15: [2023-05-08 11:23:02,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +27: [2023-05-08 11:23:02,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +30: [2023-05-08 11:23:02,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:23:02,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +27: [2023-05-08 11:23:02,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 7: [2023-05-08 11:23:02,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 7: [2023-05-08 11:23:02,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:23:02,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +10: [2023-05-08 11:23:02,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +10: [2023-05-08 11:23:02,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +19: [2023-05-08 11:23:02,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:23:02,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:23:02,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:23:02,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:23:02,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:23:02,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:23:02,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:23:02,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:23:02,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:23:02,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:23:02,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:23:02,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +19: [2023-05-08 11:23:02,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:23:02,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +19: [2023-05-08 11:23:02,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:23:02,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +24: [2023-05-08 11:23:02,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:23:02,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:23:02,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:23:02,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:23:02,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +24: [2023-05-08 11:23:02,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:23:02,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +15: [2023-05-08 11:23:02,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:23:02,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:23:02,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +26: [2023-05-08 11:23:02,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:23:02,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +24: [2023-05-08 11:23:02,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:23:02,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +19: [2023-05-08 11:23:02,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +27: [2023-05-08 11:23:02,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +26: [2023-05-08 11:23:02,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:23:02,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:23:02,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:23:02,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +17: [2023-05-08 11:23:02,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +30: [2023-05-08 11:23:02,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +26: [2023-05-08 11:23:02,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:23:02,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +17: [2023-05-08 11:23:02,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +10: [2023-05-08 11:23:02,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:23:02,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:23:02,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +30: [2023-05-08 11:23:02,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +30: [2023-05-08 11:23:02,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 7: [2023-05-08 11:23:02,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:23:02,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +15: [2023-05-08 11:23:02,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:23:02,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:23:02,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +15: [2023-05-08 11:23:02,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +19: [2023-05-08 11:23:02,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:23:02,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +30: [2023-05-08 11:23:02,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +27: [2023-05-08 11:23:02,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:23:02,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:23:02,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +27: [2023-05-08 11:23:02,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +24: [2023-05-08 11:23:02,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:23:02,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +30: [2023-05-08 11:23:02,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +30: [2023-05-08 11:23:02,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +15: [2023-05-08 11:23:02,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +15: [2023-05-08 11:23:02,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +10: [2023-05-08 11:23:02,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:23:02,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 3: [2023-05-08 11:23:02,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 3: [2023-05-08 11:23:02,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 3: [2023-05-08 11:23:02,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 3: [2023-05-08 11:23:02,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +27: [2023-05-08 11:23:02,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +27: [2023-05-08 11:23:02,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +15: [2023-05-08 11:23:02,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +15: [2023-05-08 11:23:02,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +26: [2023-05-08 11:23:02,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:23:02,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +27: [2023-05-08 11:23:02,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +27: [2023-05-08 11:23:02,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +17: [2023-05-08 11:23:02,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:23:02,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +17: [2023-05-08 11:23:02,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:23:02,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:23:02,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:23:02,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +15: [2023-05-08 11:23:02,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +15: [2023-05-08 11:23:02,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 7: [2023-05-08 11:23:02,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:23:02,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:23:02,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 7: [2023-05-08 11:23:02,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 7: [2023-05-08 11:23:02,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +26: [2023-05-08 11:23:02,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:23:02,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +29: [2023-05-08 11:23:02,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +29: [2023-05-08 11:23:02,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +29: [2023-05-08 11:23:02,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +23: [2023-05-08 11:23:02,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +26: [2023-05-08 11:23:02,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +23: [2023-05-08 11:23:02,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 3: [2023-05-08 11:23:02,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:23:02,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +22: [2023-05-08 11:23:02,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 3: [2023-05-08 11:23:02,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:23:02,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +22: [2023-05-08 11:23:02,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +22: [2023-05-08 11:23:02,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 3: [2023-05-08 11:23:02,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 3: [2023-05-08 11:23:02,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:23:02,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +24: [2023-05-08 11:23:02,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +29: [2023-05-08 11:23:02,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +26: [2023-05-08 11:23:02,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:23:02,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 7: [2023-05-08 11:23:02,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +22: [2023-05-08 11:23:02,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +27: [2023-05-08 11:23:02,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:23:02,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +26: [2023-05-08 11:23:02,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +27: [2023-05-08 11:23:02,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +26: [2023-05-08 11:23:02,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:23:02,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 7: [2023-05-08 11:23:02,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:23:02,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:23:02,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:23:02,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:23:02,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:23:02,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:23:02,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:23:02,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:23:02,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:23:02,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:23:02,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:23:02,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:23:02,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:23:02,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +18: [2023-05-08 11:23:02,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +18: [2023-05-08 11:23:02,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:23:02,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +18: [2023-05-08 11:23:02,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +12: [2023-05-08 11:23:02,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +18: [2023-05-08 11:23:02,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:23:02,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +18: [2023-05-08 11:23:02,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +12: [2023-05-08 11:23:02,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +12: [2023-05-08 11:23:02,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +18: [2023-05-08 11:23:02,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:23:02,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +11: [2023-05-08 11:23:02,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +11: [2023-05-08 11:23:02,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +11: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +11: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +11: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:23:02,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 3: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +11: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +11: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +11: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 3: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +11: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +31: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +11: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 3: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 3: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 3: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +31: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +31: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +31: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +11: [2023-05-08 11:23:02,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +22: [2023-05-08 11:23:02,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +22: [2023-05-08 11:23:02,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:23:02,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +25: [2023-05-08 11:23:02,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +25: [2023-05-08 11:23:02,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +25: [2023-05-08 11:23:02,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +25: [2023-05-08 11:23:02,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +25: [2023-05-08 11:23:02,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:23:02,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +25: [2023-05-08 11:23:02,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +25: [2023-05-08 11:23:02,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:23:02,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +20: [2023-05-08 11:23:02,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +20: [2023-05-08 11:23:02,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:23:02,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:23:02,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +20: [2023-05-08 11:23:02,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:23:02,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +25: [2023-05-08 11:23:02,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 3: [2023-05-08 11:23:02,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:23:02,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:23:02,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +12: [2023-05-08 11:23:02,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:23:02,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +12: [2023-05-08 11:23:02,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 3: [2023-05-08 11:23:02,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:23:02,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +22: [2023-05-08 11:23:02,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +22: [2023-05-08 11:23:02,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +22: [2023-05-08 11:23:02,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +20: [2023-05-08 11:23:02,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:23:02,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +23: [2023-05-08 11:23:02,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:23:02,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +29: [2023-05-08 11:23:02,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +23: [2023-05-08 11:23:02,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +23: [2023-05-08 11:23:02,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +12: [2023-05-08 11:23:02,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:23:02,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +29: [2023-05-08 11:23:02,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +18: [2023-05-08 11:23:02,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +18: [2023-05-08 11:23:02,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:23:02,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:23:02,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:23:02,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:23:02,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:23:02,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:23:02,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:23:02,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:23:02,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:23:02,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +23: [2023-05-08 11:23:02,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +31: [2023-05-08 11:23:02,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +31: [2023-05-08 11:23:02,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:23:02,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +29: [2023-05-08 11:23:02,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +29: [2023-05-08 11:23:02,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +29: [2023-05-08 11:23:02,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +12: [2023-05-08 11:23:02,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:23:02,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:23:02,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +12: [2023-05-08 11:23:02,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +12: [2023-05-08 11:23:02,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +31: [2023-05-08 11:23:02,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +12: [2023-05-08 11:23:02,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:23:02,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:23:02,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 3: [2023-05-08 11:23:02,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:23:02,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:23:02,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:23:02,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +20: [2023-05-08 11:23:02,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +18: [2023-05-08 11:23:02,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +11: [2023-05-08 11:23:02,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +11: [2023-05-08 11:23:02,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +11: [2023-05-08 11:23:02,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +11: [2023-05-08 11:23:02,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:23:02,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:23:02,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:23:02,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:23:02,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:23:02,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +17: [2023-05-08 11:23:02,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:23:02,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:23:02,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +17: [2023-05-08 11:23:02,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:23:02,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:23:02,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +17: [2023-05-08 11:23:02,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +17: [2023-05-08 11:23:02,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +17: [2023-05-08 11:23:02,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +17: [2023-05-08 11:23:02,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +18: [2023-05-08 11:23:02,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:23:02,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +12: [2023-05-08 11:23:02,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:23:02,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +20: [2023-05-08 11:23:02,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:23:02,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:23:02,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 3: [2023-05-08 11:23:02,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +18: [2023-05-08 11:23:02,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:23:02,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +13: [2023-05-08 11:23:02,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +13: [2023-05-08 11:23:02,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:23:02,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +13: [2023-05-08 11:23:02,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +13: [2023-05-08 11:23:02,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:23:02,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:23:02,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +13: [2023-05-08 11:23:02,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +12: [2023-05-08 11:23:02,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:23:02,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +18: [2023-05-08 11:23:02,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +25: [2023-05-08 11:23:02,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +25: [2023-05-08 11:23:02,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +25: [2023-05-08 11:23:02,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:23:02,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +21: [2023-05-08 11:23:02,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +21: [2023-05-08 11:23:02,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +21: [2023-05-08 11:23:02,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. +21: [2023-05-08 11:23:02,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt. + 3: [2023-05-08 11:23:02,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:23:02,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:23:02,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:23:02,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +11: [2023-05-08 11:23:02,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:23:02,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:23:02,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:23:02,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:23:02,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:23:02,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:23:02,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:23:02,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:23:02,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:23:02,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:23:02,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:23:02,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:23:02,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:23:02,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:23:02,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +20: [2023-05-08 11:23:02,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:23:02,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:23:02,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:02,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +13: [2023-05-08 11:23:02,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +25: [2023-05-08 11:23:02,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:23:02,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:23:02,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:23:02,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:23:02,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:23:02,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt... +20: [2023-05-08 11:23:02,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:23:02,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:23:02,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +20: [2023-05-08 11:23:02,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:23:02,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:23:02,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +25: [2023-05-08 11:23:02,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:23:02,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:02,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +13: [2023-05-08 11:23:02,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:23:02,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +30: [2023-05-08 11:23:02,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +30: [2023-05-08 11:23:02,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:23:02,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:02,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:23:02,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:23:02,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:23:02,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:23:02,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:23:02,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +20: [2023-05-08 11:23:02,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:02,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 8: [2023-05-08 11:23:02,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 8: [2023-05-08 11:23:02,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +30: [2023-05-08 11:23:02,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:23:02,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:02,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +17: [2023-05-08 11:23:02,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +13: [2023-05-08 11:23:02,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:23:02,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +21: [2023-05-08 11:23:02,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +21: [2023-05-08 11:23:02,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +21: [2023-05-08 11:23:02,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +12: [2023-05-08 11:23:02,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:23:02,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +21: [2023-05-08 11:23:02,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... +21: [2023-05-08 11:23:02,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt. +21: [2023-05-08 11:23:02,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt... + 8: [2023-05-08 11:23:02,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +17: [2023-05-08 11:23:02,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 9: [2023-05-08 11:23:02,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 9: [2023-05-08 11:23:02,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 9: [2023-05-08 11:23:02,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +18: [2023-05-08 11:23:02,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +18: [2023-05-08 11:23:02,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +18: [2023-05-08 11:23:02,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +18: [2023-05-08 11:23:02,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +14: [2023-05-08 11:23:02,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +14: [2023-05-08 11:23:02,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +14: [2023-05-08 11:23:02,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +14: [2023-05-08 11:23:02,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +15: [2023-05-08 11:23:02,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:23:02,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:23:02,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:23:02,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +15: [2023-05-08 11:23:02,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +15: [2023-05-08 11:23:02,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:23:02,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +15: [2023-05-08 11:23:02,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:23:02,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:23:02,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:23:02,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:23:02,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +14: [2023-05-08 11:23:02,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +14: [2023-05-08 11:23:02,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +14: [2023-05-08 11:23:02,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +14: [2023-05-08 11:23:02,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +15: [2023-05-08 11:23:02,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:23:02,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:23:02,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:23:02,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:23:02,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:23:02,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +31: [2023-05-08 11:23:02,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:23:02,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:23:02,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:23:02,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:23:02,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:23:02,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +31: [2023-05-08 11:23:02,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +31: [2023-05-08 11:23:02,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +31: [2023-05-08 11:23:02,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +18: [2023-05-08 11:23:02,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +18: [2023-05-08 11:23:02,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +18: [2023-05-08 11:23:02,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +18: [2023-05-08 11:23:02,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +31: [2023-05-08 11:23:02,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +31: [2023-05-08 11:23:02,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:23:02,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:23:02,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +31: [2023-05-08 11:23:02,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:23:02,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +18: [2023-05-08 11:23:02,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +18: [2023-05-08 11:23:02,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +18: [2023-05-08 11:23:02,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +14: [2023-05-08 11:23:02,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:23:02,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +14: [2023-05-08 11:23:02,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +14: [2023-05-08 11:23:02,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +14: [2023-05-08 11:23:02,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +15: [2023-05-08 11:23:02,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:23:02,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:23:02,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +14: [2023-05-08 11:23:02,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +14: [2023-05-08 11:23:02,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:23:02,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +16: [2023-05-08 11:23:02,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:23:02,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +16: [2023-05-08 11:23:02,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +14: [2023-05-08 11:23:02,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:23:02,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +14: [2023-05-08 11:23:02,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:23:02,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +16: [2023-05-08 11:23:02,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +16: [2023-05-08 11:23:02,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:23:02,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +31: [2023-05-08 11:23:02,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:23:02,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +14: [2023-05-08 11:23:02,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +14: [2023-05-08 11:23:02,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +14: [2023-05-08 11:23:02,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +14: [2023-05-08 11:23:02,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:23:02,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +14: [2023-05-08 11:23:02,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +14: [2023-05-08 11:23:02,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +14: [2023-05-08 11:23:02,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +16: [2023-05-08 11:23:02,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +14: [2023-05-08 11:23:02,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +14: [2023-05-08 11:23:02,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +31: [2023-05-08 11:23:02,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +16: [2023-05-08 11:23:02,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +14: [2023-05-08 11:23:02,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +14: [2023-05-08 11:23:02,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +31: [2023-05-08 11:23:02,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +14: [2023-05-08 11:23:02,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +16: [2023-05-08 11:23:02,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +14: [2023-05-08 11:23:02,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:23:02,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +31: [2023-05-08 11:23:02,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 6: [2023-05-08 11:23:02,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 6: [2023-05-08 11:23:02,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 6: [2023-05-08 11:23:02,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +16: [2023-05-08 11:23:02,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +14: [2023-05-08 11:23:02,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +16: [2023-05-08 11:23:02,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:23:02,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +14: [2023-05-08 11:23:02,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +16: [2023-05-08 11:23:02,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:23:02,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:23:02,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:23:02,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:23:02,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:23:02,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:23:02,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:23:02,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:23:02,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:23:02,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:23:02,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:23:02,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 2: [2023-05-08 11:23:02,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:23:02,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:23:02,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:23:02,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:23:02,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:23:02,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 2: [2023-05-08 11:23:02,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:23:02,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +14: [2023-05-08 11:23:02,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:23:02,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +21: [2023-05-08 11:23:02,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:23:02,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:23:02,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 8: [2023-05-08 11:23:02,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:23:02,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:23:02,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 0: [2023-05-08 11:23:02,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +21: [2023-05-08 11:23:02,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +21: [2023-05-08 11:23:02,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 9: [2023-05-08 11:23:02,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 9: [2023-05-08 11:23:02,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 9: [2023-05-08 11:23:02,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 9: [2023-05-08 11:23:02,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +21: [2023-05-08 11:23:02,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +21: [2023-05-08 11:23:02,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:23:02,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:23:02,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 1: [2023-05-08 11:23:02,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 1: [2023-05-08 11:23:02,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 4: [2023-05-08 11:23:02,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 4: [2023-05-08 11:23:02,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 2: [2023-05-08 11:23:02,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:23:02,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 4: [2023-05-08 11:23:02,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 2: [2023-05-08 11:23:02,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:23:02,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:23:02,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:23:02,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:23:02,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:23:02,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:23:02,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 1: [2023-05-08 11:23:02,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:23:02,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:23:02,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:23:02,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:23:02,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:23:02,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:23:02,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:23:02,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +28: [2023-05-08 11:23:02,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +28: [2023-05-08 11:23:02,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:23:02,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:23:02,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:23:02,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +28: [2023-05-08 11:23:02,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:23:02,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:23:02,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:23:02,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:23:02,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:23:02,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:23:02,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 6: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +26: [2023-05-08 11:23:02,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 6: [2023-05-08 11:23:02,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +23: [2023-05-08 11:23:02,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:23:02,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:23:02,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:23:02,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:23:02,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:23:02,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:23:02,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:23:02,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:23:02,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:23:02,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:23:02,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:23:02,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:23:02,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:23:02,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:23:02,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:23:02,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +15: [2023-05-08 11:23:02,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 0: [2023-05-08 11:23:02,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:23:02,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +15: [2023-05-08 11:23:02,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +15: [2023-05-08 11:23:02,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 0: [2023-05-08 11:23:02,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 0: [2023-05-08 11:23:02,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 0: [2023-05-08 11:23:02,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:23:02,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +28: [2023-05-08 11:23:02,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:23:02,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +22: [2023-05-08 11:23:02,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:23:02,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:23:02,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:23:02,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:23:02,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +19: [2023-05-08 11:23:02,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:23:02,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:23:02,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:23:02,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:23:02,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:23:02,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +22: [2023-05-08 11:23:02,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +28: [2023-05-08 11:23:02,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:23:02,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +27: [2023-05-08 11:23:02,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +28: [2023-05-08 11:23:02,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +27: [2023-05-08 11:23:02,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +27: [2023-05-08 11:23:02,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +27: [2023-05-08 11:23:02,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +21: [2023-05-08 11:23:02,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +19: [2023-05-08 11:23:02,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +27: [2023-05-08 11:23:02,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +27: [2023-05-08 11:23:02,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +27: [2023-05-08 11:23:02,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +27: [2023-05-08 11:23:02,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:23:02,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:23:02,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:23:02,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +15: [2023-05-08 11:23:02,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +15: [2023-05-08 11:23:02,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:23:02,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:23:02,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:23:02,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +29: [2023-05-08 11:23:02,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:23:02,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:23:02,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:23:02,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 4: [2023-05-08 11:23:02,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:23:02,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:23:02,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:23:02,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 4: [2023-05-08 11:23:02,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 4: [2023-05-08 11:23:02,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:23:02,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:23:02,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +22: [2023-05-08 11:23:02,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +29: [2023-05-08 11:23:02,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +28: [2023-05-08 11:23:02,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:23:02,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +29: [2023-05-08 11:23:02,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 1: [2023-05-08 11:23:02,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 0: [2023-05-08 11:23:02,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:23:02,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:23:02,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +28: [2023-05-08 11:23:02,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 1: [2023-05-08 11:23:02,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 6: [2023-05-08 11:23:02,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:23:02,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:23:02,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:23:02,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:23:02,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:23:02,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:23:02,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +28: [2023-05-08 11:23:02,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +23: [2023-05-08 11:23:02,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:23:02,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +23: [2023-05-08 11:23:02,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +29: [2023-05-08 11:23:02,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 6: [2023-05-08 11:23:02,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +27: [2023-05-08 11:23:02,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:23:02,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +27: [2023-05-08 11:23:02,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +19: [2023-05-08 11:23:02,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:23:02,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:23:02,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +26: [2023-05-08 11:23:02,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +27: [2023-05-08 11:23:02,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:23:02,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 4: [2023-05-08 11:23:02,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:23:02,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:23:02,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +27: [2023-05-08 11:23:02,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:23:02,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +29: [2023-05-08 11:23:02,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:23:02,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:02,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +13: [2023-05-08 11:23:02,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +15: [2023-05-08 11:23:02,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:23:02,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +15: [2023-05-08 11:23:02,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:23:02,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 5: [2023-05-08 11:23:02,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 5: [2023-05-08 11:23:02,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 5: [2023-05-08 11:23:02,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 5: [2023-05-08 11:23:02,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +24: [2023-05-08 11:23:02,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:23:02,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:23:02,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +28: [2023-05-08 11:23:02,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:23:02,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +26: [2023-05-08 11:23:02,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 6: [2023-05-08 11:23:02,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:23:02,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +22: [2023-05-08 11:23:02,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:02,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +26: [2023-05-08 11:23:02,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:23:02,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:23:02,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +28: [2023-05-08 11:23:02,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +27: [2023-05-08 11:23:02,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +27: [2023-05-08 11:23:02,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:23:02,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:23:02,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:23:02,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:23:02,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +19: [2023-05-08 11:23:02,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +29: [2023-05-08 11:23:02,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +27: [2023-05-08 11:23:02,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +27: [2023-05-08 11:23:02,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:02,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:23:02,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:23:02,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +15: [2023-05-08 11:23:02,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:23:02,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +15: [2023-05-08 11:23:02,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +13: [2023-05-08 11:23:02,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +19: [2023-05-08 11:23:02,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:23:02,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:23:02,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +19: [2023-05-08 11:23:02,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:23:02,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:23:02,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +28: [2023-05-08 11:23:02,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +25: [2023-05-08 11:23:02,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +25: [2023-05-08 11:23:02,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +25: [2023-05-08 11:23:02,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +25: [2023-05-08 11:23:02,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +10: [2023-05-08 11:23:02,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +10: [2023-05-08 11:23:02,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 1: [2023-05-08 11:23:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +19: [2023-05-08 11:23:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:23:02,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:23:02,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +24: [2023-05-08 11:23:02,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +24: [2023-05-08 11:23:02,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +24: [2023-05-08 11:23:02,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +24: [2023-05-08 11:23:02,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +20: [2023-05-08 11:23:02,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +20: [2023-05-08 11:23:02,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +20: [2023-05-08 11:23:02,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +13: [2023-05-08 11:23:02,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:23:02,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +10: [2023-05-08 11:23:02,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 5: [2023-05-08 11:23:02,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +13: [2023-05-08 11:23:02,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +20: [2023-05-08 11:23:02,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 7: [2023-05-08 11:23:02,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 7: [2023-05-08 11:23:02,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 7: [2023-05-08 11:23:02,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 7: [2023-05-08 11:23:02,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 5: [2023-05-08 11:23:02,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +13: [2023-05-08 11:23:02,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +13: [2023-05-08 11:23:02,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +13: [2023-05-08 11:23:02,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +13: [2023-05-08 11:23:02,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +28: [2023-05-08 11:23:02,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +28: [2023-05-08 11:23:02,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +19: [2023-05-08 11:23:02,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +19: [2023-05-08 11:23:02,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +19: [2023-05-08 11:23:02,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +19: [2023-05-08 11:23:02,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +13: [2023-05-08 11:23:02,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +13: [2023-05-08 11:23:02,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 2: [2023-05-08 11:23:02,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +16: [2023-05-08 11:23:02,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +16: [2023-05-08 11:23:02,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 2: [2023-05-08 11:23:02,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 2: [2023-05-08 11:23:02,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. + 2: [2023-05-08 11:23:02,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +25: [2023-05-08 11:23:02,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +27: [2023-05-08 11:23:02,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +27: [2023-05-08 11:23:02,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +29: [2023-05-08 11:23:02,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +16: [2023-05-08 11:23:02,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +29: [2023-05-08 11:23:02,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +29: [2023-05-08 11:23:02,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +29: [2023-05-08 11:23:02,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +25: [2023-05-08 11:23:02,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:23:02,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +25: [2023-05-08 11:23:02,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:23:02,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:23:02,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:23:02,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +26: [2023-05-08 11:23:02,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +30: [2023-05-08 11:23:02,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +30: [2023-05-08 11:23:02,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +25: [2023-05-08 11:23:02,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:23:02,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +27: [2023-05-08 11:23:02,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +30: [2023-05-08 11:23:02,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +23: [2023-05-08 11:23:02,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +27: [2023-05-08 11:23:02,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +30: [2023-05-08 11:23:02,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +10: [2023-05-08 11:23:02,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:23:02,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +10: [2023-05-08 11:23:02,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:23:02,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +23: [2023-05-08 11:23:02,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +23: [2023-05-08 11:23:02,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +21: [2023-05-08 11:23:02,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +28: [2023-05-08 11:23:02,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +21: [2023-05-08 11:23:02,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +24: [2023-05-08 11:23:02,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:23:02,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +10: [2023-05-08 11:23:02,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:23:02,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +22: [2023-05-08 11:23:02,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +22: [2023-05-08 11:23:02,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +24: [2023-05-08 11:23:02,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:23:02,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:23:02,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt. +10: [2023-05-08 11:23:02,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:23:02,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:23:02,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:23:02,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:23:02,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:23:02,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +19: [2023-05-08 11:23:02,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +19: [2023-05-08 11:23:02,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:23:02,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:23:02,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 2: [2023-05-08 11:23:02,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +19: [2023-05-08 11:23:02,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +16: [2023-05-08 11:23:02,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +19: [2023-05-08 11:23:02,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +27: [2023-05-08 11:23:02,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +27: [2023-05-08 11:23:02,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:23:02,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:23:02,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:23:02,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:23:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:23:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:23:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:23:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:23:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +27: [2023-05-08 11:23:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:23:02,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:23:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:23:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:23:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:23:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:23:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:23:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +27: [2023-05-08 11:23:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +21: [2023-05-08 11:23:02,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:23:02,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:23:02,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:23:02,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:23:02,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:23:02,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:23:02,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 7: [2023-05-08 11:23:02,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:23:02,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:23:02,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:23:02,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:23:02,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:23:02,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 7: [2023-05-08 11:23:02,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:23:02,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 5: [2023-05-08 11:23:02,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +24: [2023-05-08 11:23:02,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 7: [2023-05-08 11:23:02,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:23:02,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:23:02,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +24: [2023-05-08 11:23:02,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:23:02,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:23:02,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +24: [2023-05-08 11:23:02,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +20: [2023-05-08 11:23:02,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +25: [2023-05-08 11:23:02,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +20: [2023-05-08 11:23:02,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +20: [2023-05-08 11:23:02,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +25: [2023-05-08 11:23:02,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +20: [2023-05-08 11:23:02,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 2: [2023-05-08 11:23:02,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +25: [2023-05-08 11:23:02,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +20: [2023-05-08 11:23:02,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +25: [2023-05-08 11:23:02,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +20: [2023-05-08 11:23:02,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +30: [2023-05-08 11:23:02,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:23:02,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:23:02,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +10: [2023-05-08 11:23:02,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:23:02,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:23:02,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +10: [2023-05-08 11:23:02,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +30: [2023-05-08 11:23:02,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +19: [2023-05-08 11:23:02,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:23:02,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:23:02,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:23:02,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +27: [2023-05-08 11:23:02,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:23:02,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +19: [2023-05-08 11:23:02,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +29: [2023-05-08 11:23:02,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:23:02,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +22: [2023-05-08 11:23:02,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +27: [2023-05-08 11:23:02,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +22: [2023-05-08 11:23:02,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +30: [2023-05-08 11:23:02,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:23:02,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +27: [2023-05-08 11:23:02,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +27: [2023-05-08 11:23:02,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 2: [2023-05-08 11:23:02,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +20: [2023-05-08 11:23:02,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +16: [2023-05-08 11:23:02,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +20: [2023-05-08 11:23:02,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +21: [2023-05-08 11:23:02,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +21: [2023-05-08 11:23:02,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +16: [2023-05-08 11:23:02,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +27: [2023-05-08 11:23:02,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +27: [2023-05-08 11:23:02,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +10: [2023-05-08 11:23:02,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:23:02,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +21: [2023-05-08 11:23:02,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +23: [2023-05-08 11:23:02,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:23:02,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +21: [2023-05-08 11:23:02,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +23: [2023-05-08 11:23:02,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +23: [2023-05-08 11:23:02,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 2: [2023-05-08 11:23:02,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:23:02,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 2: [2023-05-08 11:23:02,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +27: [2023-05-08 11:23:02,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +27: [2023-05-08 11:23:02,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 7: [2023-05-08 11:23:02,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +22: [2023-05-08 11:23:02,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:23:02,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +25: [2023-05-08 11:23:02,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:23:02,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +16: [2023-05-08 11:23:02,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +25: [2023-05-08 11:23:02,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +29: [2023-05-08 11:23:02,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:23:02,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +24: [2023-05-08 11:23:02,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +21: [2023-05-08 11:23:02,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +21: [2023-05-08 11:23:02,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +19: [2023-05-08 11:23:02,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:23:02,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:23:02,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 7: [2023-05-08 11:23:02,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 7: [2023-05-08 11:23:02,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +19: [2023-05-08 11:23:02,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +22: [2023-05-08 11:23:02,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:23:02,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +19: [2023-05-08 11:23:02,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +29: [2023-05-08 11:23:02,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:23:02,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 7: [2023-05-08 11:23:02,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +16: [2023-05-08 11:23:02,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +16: [2023-05-08 11:23:02,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +16: [2023-05-08 11:23:02,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:23:02,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +16: [2023-05-08 11:23:02,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +30: [2023-05-08 11:23:02,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 7: [2023-05-08 11:23:02,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +30: [2023-05-08 11:23:02,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +21: [2023-05-08 11:23:02,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:23:02,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +30: [2023-05-08 11:23:02,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +21: [2023-05-08 11:23:02,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +22: [2023-05-08 11:23:02,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... +29: [2023-05-08 11:23:02,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:23:02,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 7: [2023-05-08 11:23:02,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:23:02,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt... + 7: [2023-05-08 11:23:02,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:23:02,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +11: [2023-05-08 11:23:02,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +11: [2023-05-08 11:23:02,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +11: [2023-05-08 11:23:02,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +11: [2023-05-08 11:23:02,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +11: [2023-05-08 11:23:02,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:23:02,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:23:02,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:23:02,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +31: [2023-05-08 11:23:02,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +31: [2023-05-08 11:23:02,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +31: [2023-05-08 11:23:02,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +31: [2023-05-08 11:23:02,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +14: [2023-05-08 11:23:02,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +14: [2023-05-08 11:23:02,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +14: [2023-05-08 11:23:02,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +14: [2023-05-08 11:23:02,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +31: [2023-05-08 11:23:02,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +31: [2023-05-08 11:23:02,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +14: [2023-05-08 11:23:02,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +31: [2023-05-08 11:23:02,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:23:02,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +31: [2023-05-08 11:23:02,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:23:02,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +12: [2023-05-08 11:23:02,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +12: [2023-05-08 11:23:02,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +14: [2023-05-08 11:23:02,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +14: [2023-05-08 11:23:02,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +14: [2023-05-08 11:23:02,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:23:02,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:23:02,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:23:02,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:23:02,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:23:02,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 3: [2023-05-08 11:23:02,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 3: [2023-05-08 11:23:02,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 3: [2023-05-08 11:23:02,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 6: [2023-05-08 11:23:02,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 6: [2023-05-08 11:23:02,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 6: [2023-05-08 11:23:02,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 6: [2023-05-08 11:23:02,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 0: [2023-05-08 11:23:02,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 0: [2023-05-08 11:23:02,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 0: [2023-05-08 11:23:02,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 0: [2023-05-08 11:23:02,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +15: [2023-05-08 11:23:02,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +15: [2023-05-08 11:23:02,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +15: [2023-05-08 11:23:02,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 3: [2023-05-08 11:23:02,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:23:02,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 3: [2023-05-08 11:23:02,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:23:02,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:23:02,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 6: [2023-05-08 11:23:02,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:23:02,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:23:02,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:23:02,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:23:02,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:23:02,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:23:02,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +17: [2023-05-08 11:23:02,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +17: [2023-05-08 11:23:02,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +17: [2023-05-08 11:23:02,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +17: [2023-05-08 11:23:02,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:23:02,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:23:02,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:23:02,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:23:02,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:23:02,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:23:02,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:23:02,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:23:02,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:23:02,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:23:02,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:23:02,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:23:02,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +20: [2023-05-08 11:23:02,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:23:02,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +20: [2023-05-08 11:23:02,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:23:02,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +20: [2023-05-08 11:23:02,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +20: [2023-05-08 11:23:02,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:23:02,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:23:02,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:23:02,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:23:02,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:23:02,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +20: [2023-05-08 11:23:02,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +20: [2023-05-08 11:23:02,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:23:02,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:23:02,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +20: [2023-05-08 11:23:02,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:23:02,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:23:02,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 8: [2023-05-08 11:23:02,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +28: [2023-05-08 11:23:02,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +28: [2023-05-08 11:23:02,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +28: [2023-05-08 11:23:02,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +28: [2023-05-08 11:23:02,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 8: [2023-05-08 11:23:02,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +20: [2023-05-08 11:23:02,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 9: [2023-05-08 11:23:02,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 9: [2023-05-08 11:23:02,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +20: [2023-05-08 11:23:02,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 9: [2023-05-08 11:23:02,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +26: [2023-05-08 11:23:02,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 9: [2023-05-08 11:23:02,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +18: [2023-05-08 11:23:02,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +18: [2023-05-08 11:23:02,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +18: [2023-05-08 11:23:02,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +18: [2023-05-08 11:23:02,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +26: [2023-05-08 11:23:02,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 1: [2023-05-08 11:23:02,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 1: [2023-05-08 11:23:02,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 1: [2023-05-08 11:23:02,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 4: [2023-05-08 11:23:02,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 4: [2023-05-08 11:23:02,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 1: [2023-05-08 11:23:02,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 4: [2023-05-08 11:23:02,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 8: [2023-05-08 11:23:02,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +28: [2023-05-08 11:23:02,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:23:02,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:23:02,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 8: [2023-05-08 11:23:02,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:23:02,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:23:02,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +26: [2023-05-08 11:23:02,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 8: [2023-05-08 11:23:02,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:23:02,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:02,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +13: [2023-05-08 11:23:02,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +18: [2023-05-08 11:23:02,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:02,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 9: [2023-05-08 11:23:02,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:02,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 9: [2023-05-08 11:23:02,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 9: [2023-05-08 11:23:02,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:23:02,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:23:02,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:23:02,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:23:02,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:23:02,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:23:02,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:23:02,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:23:02,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 4: [2023-05-08 11:23:02,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:23:02,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +10: [2023-05-08 11:23:02,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +10: [2023-05-08 11:23:02,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 4: [2023-05-08 11:23:02,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:23:02,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:23:02,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:23:02,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 1: [2023-05-08 11:23:02,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:02,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:02,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:23:02,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +13: [2023-05-08 11:23:02,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:02,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:23:02,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:23:02,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 2: [2023-05-08 11:23:02,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 2: [2023-05-08 11:23:02,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 2: [2023-05-08 11:23:02,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +10: [2023-05-08 11:23:02,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:23:02,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:23:02,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:23:02,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 2: [2023-05-08 11:23:02,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:23:02,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:23:02,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 2: [2023-05-08 11:23:02,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 2: [2023-05-08 11:23:02,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:23:02,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:23:02,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +18: [2023-05-08 11:23:02,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:23:02,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +19: [2023-05-08 11:23:02,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +19: [2023-05-08 11:23:02,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +19: [2023-05-08 11:23:02,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +19: [2023-05-08 11:23:02,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 5: [2023-05-08 11:23:02,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +19: [2023-05-08 11:23:02,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +19: [2023-05-08 11:23:02,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +19: [2023-05-08 11:23:02,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +19: [2023-05-08 11:23:02,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 5: [2023-05-08 11:23:02,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 5: [2023-05-08 11:23:02,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +25: [2023-05-08 11:23:03,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +25: [2023-05-08 11:23:03,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 5: [2023-05-08 11:23:03,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:23:03,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +25: [2023-05-08 11:23:03,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 5: [2023-05-08 11:23:03,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 3: [2023-05-08 11:23:03,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 3: [2023-05-08 11:23:03,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:23:03,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 3: [2023-05-08 11:23:03,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +25: [2023-05-08 11:23:03,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:23:03,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:23:03,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:23:03,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +20: [2023-05-08 11:23:03,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +20: [2023-05-08 11:23:03,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +20: [2023-05-08 11:23:03,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 3: [2023-05-08 11:23:03,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +30: [2023-05-08 11:23:03,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +30: [2023-05-08 11:23:03,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +21: [2023-05-08 11:23:03,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +21: [2023-05-08 11:23:03,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +21: [2023-05-08 11:23:03,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 7: [2023-05-08 11:23:03,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +21: [2023-05-08 11:23:03,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 7: [2023-05-08 11:23:03,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 7: [2023-05-08 11:23:03,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +20: [2023-05-08 11:23:03,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +30: [2023-05-08 11:23:03,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +30: [2023-05-08 11:23:03,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +20: [2023-05-08 11:23:03,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:23:03,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:23:03,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +22: [2023-05-08 11:23:03,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +22: [2023-05-08 11:23:03,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +22: [2023-05-08 11:23:03,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +27: [2023-05-08 11:23:03,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +27: [2023-05-08 11:23:03,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 3: [2023-05-08 11:23:03,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +22: [2023-05-08 11:23:03,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +30: [2023-05-08 11:23:03,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:23:03,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:23:03,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +27: [2023-05-08 11:23:03,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +27: [2023-05-08 11:23:03,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +30: [2023-05-08 11:23:03,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:23:03,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +23: [2023-05-08 11:23:03,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 7: [2023-05-08 11:23:03,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:23:03,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 3: [2023-05-08 11:23:03,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +23: [2023-05-08 11:23:03,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +21: [2023-05-08 11:23:03,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +30: [2023-05-08 11:23:03,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +30: [2023-05-08 11:23:03,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +29: [2023-05-08 11:23:03,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +21: [2023-05-08 11:23:03,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:23:03,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +29: [2023-05-08 11:23:03,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +29: [2023-05-08 11:23:03,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +16: [2023-05-08 11:23:03,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +24: [2023-05-08 11:23:03,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +24: [2023-05-08 11:23:03,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +16: [2023-05-08 11:23:03,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +29: [2023-05-08 11:23:03,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +16: [2023-05-08 11:23:03,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +16: [2023-05-08 11:23:03,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. +24: [2023-05-08 11:23:03,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt. + 7: [2023-05-08 11:23:03,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:23:03,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +22: [2023-05-08 11:23:03,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +22: [2023-05-08 11:23:03,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +22: [2023-05-08 11:23:03,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:23:03,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +15: [2023-05-08 11:23:03,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +15: [2023-05-08 11:23:03,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +15: [2023-05-08 11:23:03,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +15: [2023-05-08 11:23:03,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +15: [2023-05-08 11:23:03,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +15: [2023-05-08 11:23:03,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +15: [2023-05-08 11:23:03,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:23:03,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:23:03,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:23:03,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +15: [2023-05-08 11:23:03,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +15: [2023-05-08 11:23:03,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +15: [2023-05-08 11:23:03,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +15: [2023-05-08 11:23:03,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +27: [2023-05-08 11:23:03,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:23:03,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +27: [2023-05-08 11:23:03,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +27: [2023-05-08 11:23:03,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:23:03,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +20: [2023-05-08 11:23:03,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +20: [2023-05-08 11:23:03,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +20: [2023-05-08 11:23:03,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +27: [2023-05-08 11:23:03,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:23:03,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:23:03,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:23:03,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:23:03,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:23:03,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:23:03,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +24: [2023-05-08 11:23:03,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:23:03,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:23:03,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +29: [2023-05-08 11:23:03,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +29: [2023-05-08 11:23:03,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +16: [2023-05-08 11:23:03,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +16: [2023-05-08 11:23:03,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +29: [2023-05-08 11:23:03,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +16: [2023-05-08 11:23:03,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:23:03,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +20: [2023-05-08 11:23:03,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +29: [2023-05-08 11:23:03,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:23:03,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +20: [2023-05-08 11:23:03,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +20: [2023-05-08 11:23:03,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +15: [2023-05-08 11:23:03,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +15: [2023-05-08 11:23:03,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +21: [2023-05-08 11:23:03,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +21: [2023-05-08 11:23:03,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:23:03,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +21: [2023-05-08 11:23:03,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +21: [2023-05-08 11:23:03,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +21: [2023-05-08 11:23:03,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +21: [2023-05-08 11:23:03,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +21: [2023-05-08 11:23:03,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:23:03,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +21: [2023-05-08 11:23:03,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +21: [2023-05-08 11:23:03,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:23:03,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:23:03,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +21: [2023-05-08 11:23:03,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +21: [2023-05-08 11:23:03,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +15: [2023-05-08 11:23:03,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:23:03,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 7: [2023-05-08 11:23:03,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +15: [2023-05-08 11:23:03,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:23:03,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:23:03,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:23:03,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:23:03,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:23:03,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:23:03,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:23:03,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:23:03,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +22: [2023-05-08 11:23:03,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:23:03,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:23:03,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +16: [2023-05-08 11:23:03,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:23:03,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:23:03,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +22: [2023-05-08 11:23:03,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +16: [2023-05-08 11:23:03,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:23:03,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +16: [2023-05-08 11:23:03,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +16: [2023-05-08 11:23:03,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:23:03,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:23:03,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +16: [2023-05-08 11:23:03,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +16: [2023-05-08 11:23:03,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:23:03,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:23:03,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +16: [2023-05-08 11:23:03,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +22: [2023-05-08 11:23:03,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +16: [2023-05-08 11:23:03,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:23:03,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +16: [2023-05-08 11:23:03,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +22: [2023-05-08 11:23:03,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +22: [2023-05-08 11:23:03,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:23:03,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:23:03,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:23:03,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:23:03,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:23:03,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:23:03,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:23:03,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +11: [2023-05-08 11:23:03,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +11: [2023-05-08 11:23:03,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:23:03,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +11: [2023-05-08 11:23:03,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:23:03,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:23:03,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:23:03,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:23:03,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:23:03,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:23:03,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +24: [2023-05-08 11:23:03,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:23:03,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:23:03,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:23:03,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:23:03,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:23:03,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:23:03,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:23:03,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:23:03,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:23:03,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:23:03,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:23:03,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +24: [2023-05-08 11:23:03,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +24: [2023-05-08 11:23:03,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +21: [2023-05-08 11:23:03,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:23:03,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +15: [2023-05-08 11:23:03,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +16: [2023-05-08 11:23:03,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 2: [2023-05-08 11:23:03,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 2: [2023-05-08 11:23:03,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:23:03,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 2: [2023-05-08 11:23:03,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +21: [2023-05-08 11:23:03,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:23:03,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 2: [2023-05-08 11:23:03,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 2: [2023-05-08 11:23:03,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +21: [2023-05-08 11:23:03,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +22: [2023-05-08 11:23:03,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +21: [2023-05-08 11:23:03,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:23:03,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:23:03,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:23:03,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:23:03,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:23:03,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:23:03,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +21: [2023-05-08 11:23:03,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:23:03,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:23:03,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:23:03,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:23:03,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:23:03,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:23:03,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:23:03,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:23:03,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:23:03,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +11: [2023-05-08 11:23:03,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:23:03,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +22: [2023-05-08 11:23:03,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:23:03,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +22: [2023-05-08 11:23:03,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:23:03,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +11: [2023-05-08 11:23:03,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:23:03,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:23:03,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:23:03,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +16: [2023-05-08 11:23:03,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +16: [2023-05-08 11:23:03,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +16: [2023-05-08 11:23:03,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:23:03,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +22: [2023-05-08 11:23:03,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 8: [2023-05-08 11:23:03,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 8: [2023-05-08 11:23:03,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 8: [2023-05-08 11:23:03,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:23:03,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +22: [2023-05-08 11:23:03,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:23:03,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:23:03,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:23:03,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:23:03,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:23:03,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:23:03,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:23:03,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:23:03,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +25: [2023-05-08 11:23:03,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:23:03,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +25: [2023-05-08 11:23:03,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +25: [2023-05-08 11:23:03,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:23:03,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:23:03,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:23:03,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +30: [2023-05-08 11:23:03,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +30: [2023-05-08 11:23:03,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +30: [2023-05-08 11:23:03,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +30: [2023-05-08 11:23:03,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +30: [2023-05-08 11:23:03,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +30: [2023-05-08 11:23:03,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +30: [2023-05-08 11:23:03,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +30: [2023-05-08 11:23:03,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +30: [2023-05-08 11:23:03,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +30: [2023-05-08 11:23:03,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +30: [2023-05-08 11:23:03,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +30: [2023-05-08 11:23:03,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +30: [2023-05-08 11:23:03,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +30: [2023-05-08 11:23:03,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +30: [2023-05-08 11:23:03,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +25: [2023-05-08 11:23:03,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 2: [2023-05-08 11:23:03,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:23:03,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +30: [2023-05-08 11:23:03,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 7: [2023-05-08 11:23:03,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:23:03,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +30: [2023-05-08 11:23:03,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:23:03,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:23:03,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:23:03,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:23:03,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +30: [2023-05-08 11:23:03,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +30: [2023-05-08 11:23:03,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +30: [2023-05-08 11:23:03,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:23:03,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:23:03,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:23:03,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +30: [2023-05-08 11:23:03,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +30: [2023-05-08 11:23:03,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +29: [2023-05-08 11:23:03,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 4: [2023-05-08 11:23:03,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +29: [2023-05-08 11:23:03,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:23:03,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 4: [2023-05-08 11:23:03,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +29: [2023-05-08 11:23:03,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:23:03,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:23:03,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:23:03,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:23:03,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:23:03,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +29: [2023-05-08 11:23:03,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +29: [2023-05-08 11:23:03,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +29: [2023-05-08 11:23:03,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +29: [2023-05-08 11:23:03,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:23:03,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +29: [2023-05-08 11:23:03,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:03,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +13: [2023-05-08 11:23:03,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +13: [2023-05-08 11:23:03,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +13: [2023-05-08 11:23:03,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +13: [2023-05-08 11:23:03,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +13: [2023-05-08 11:23:03,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:03,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +13: [2023-05-08 11:23:03,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +13: [2023-05-08 11:23:03,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +13: [2023-05-08 11:23:03,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +13: [2023-05-08 11:23:03,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:03,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:23:03,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:03,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:03,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +13: [2023-05-08 11:23:03,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:03,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +25: [2023-05-08 11:23:03,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:23:03,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +13: [2023-05-08 11:23:03,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:23:03,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +31: [2023-05-08 11:23:03,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +31: [2023-05-08 11:23:03,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +31: [2023-05-08 11:23:03,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +31: [2023-05-08 11:23:03,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +31: [2023-05-08 11:23:03,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 1: [2023-05-08 11:23:03,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +31: [2023-05-08 11:23:03,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +31: [2023-05-08 11:23:03,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +31: [2023-05-08 11:23:03,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +31: [2023-05-08 11:23:03,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +31: [2023-05-08 11:23:03,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +31: [2023-05-08 11:23:03,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:03,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +31: [2023-05-08 11:23:03,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +31: [2023-05-08 11:23:03,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 1: [2023-05-08 11:23:03,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +12: [2023-05-08 11:23:03,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +12: [2023-05-08 11:23:03,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 6: [2023-05-08 11:23:03,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +12: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +31: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +12: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 0: [2023-05-08 11:23:03,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 0: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +31: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 0: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +31: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +12: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +12: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +12: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +12: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 0: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +12: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 6: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 6: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 0: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 0: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 1: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +12: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +12: [2023-05-08 11:23:03,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:23:03,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:23:03,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +12: [2023-05-08 11:23:03,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +12: [2023-05-08 11:23:03,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 4: [2023-05-08 11:23:03,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:23:03,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +17: [2023-05-08 11:23:03,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +17: [2023-05-08 11:23:03,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +17: [2023-05-08 11:23:03,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:23:03,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +17: [2023-05-08 11:23:03,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +17: [2023-05-08 11:23:03,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +17: [2023-05-08 11:23:03,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:23:03,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +17: [2023-05-08 11:23:03,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:23:03,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:23:03,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +17: [2023-05-08 11:23:03,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 3: [2023-05-08 11:23:03,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +17: [2023-05-08 11:23:03,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:23:03,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +18: [2023-05-08 11:23:03,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:23:03,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:23:03,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +18: [2023-05-08 11:23:03,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:23:03,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:23:03,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:23:03,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:23:03,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +10: [2023-05-08 11:23:03,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +27: [2023-05-08 11:23:03,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +27: [2023-05-08 11:23:03,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +27: [2023-05-08 11:23:03,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +10: [2023-05-08 11:23:03,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +27: [2023-05-08 11:23:03,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +13: [2023-05-08 11:23:03,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:23:03,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:23:03,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +10: [2023-05-08 11:23:03,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +10: [2023-05-08 11:23:03,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +10: [2023-05-08 11:23:03,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +10: [2023-05-08 11:23:03,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +27: [2023-05-08 11:23:03,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +27: [2023-05-08 11:23:03,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +27: [2023-05-08 11:23:03,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +10: [2023-05-08 11:23:03,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +27: [2023-05-08 11:23:03,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +27: [2023-05-08 11:23:03,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +10: [2023-05-08 11:23:03,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +10: [2023-05-08 11:23:03,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +27: [2023-05-08 11:23:03,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:23:03,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:23:03,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +27: [2023-05-08 11:23:03,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:03,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:23:03,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +29: [2023-05-08 11:23:03,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:23:03,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +10: [2023-05-08 11:23:03,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:23:03,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:23:03,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +19: [2023-05-08 11:23:03,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +10: [2023-05-08 11:23:03,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +27: [2023-05-08 11:23:03,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +31: [2023-05-08 11:23:03,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +19: [2023-05-08 11:23:03,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +19: [2023-05-08 11:23:03,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +19: [2023-05-08 11:23:03,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +19: [2023-05-08 11:23:03,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +19: [2023-05-08 11:23:03,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +19: [2023-05-08 11:23:03,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +19: [2023-05-08 11:23:03,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +19: [2023-05-08 11:23:03,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +27: [2023-05-08 11:23:03,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +27: [2023-05-08 11:23:03,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +13: [2023-05-08 11:23:03,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:23:03,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +19: [2023-05-08 11:23:03,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +19: [2023-05-08 11:23:03,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +19: [2023-05-08 11:23:03,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +19: [2023-05-08 11:23:03,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:23:03,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +19: [2023-05-08 11:23:03,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 4: [2023-05-08 11:23:03,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +26: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +19: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +19: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +28: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +28: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +12: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +14: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +26: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +28: [2023-05-08 11:23:03,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +28: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +14: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +14: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 9: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +28: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:23:03,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +28: [2023-05-08 11:23:03,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +28: [2023-05-08 11:23:03,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:23:03,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:23:03,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +28: [2023-05-08 11:23:03,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +28: [2023-05-08 11:23:03,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +28: [2023-05-08 11:23:03,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 5: [2023-05-08 11:23:03,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 9: [2023-05-08 11:23:03,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 5: [2023-05-08 11:23:03,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 5: [2023-05-08 11:23:03,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +26: [2023-05-08 11:23:03,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 5: [2023-05-08 11:23:03,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +26: [2023-05-08 11:23:03,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +28: [2023-05-08 11:23:03,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +28: [2023-05-08 11:23:03,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +31: [2023-05-08 11:23:03,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:23:03,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +28: [2023-05-08 11:23:03,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:23:03,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:23:03,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 3: [2023-05-08 11:23:03,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +28: [2023-05-08 11:23:03,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +31: [2023-05-08 11:23:03,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +17: [2023-05-08 11:23:03,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +23: [2023-05-08 11:23:03,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +23: [2023-05-08 11:23:03,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +23: [2023-05-08 11:23:03,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +23: [2023-05-08 11:23:03,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +23: [2023-05-08 11:23:03,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +23: [2023-05-08 11:23:03,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +23: [2023-05-08 11:23:03,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +23: [2023-05-08 11:23:03,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:23:03,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:23:03,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:23:03,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +17: [2023-05-08 11:23:03,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +23: [2023-05-08 11:23:03,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:23:03,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +12: [2023-05-08 11:23:03,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +23: [2023-05-08 11:23:03,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... +23: [2023-05-08 11:23:03,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt... + 5: [2023-05-08 11:23:03,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +31: [2023-05-08 11:23:03,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +23: [2023-05-08 11:23:03,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:03,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +12: [2023-05-08 11:23:03,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +17: [2023-05-08 11:23:03,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:23:03,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:23:03,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:23:03,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:23:03,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +12: [2023-05-08 11:23:03,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:23:03,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +10: [2023-05-08 11:23:03,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +27: [2023-05-08 11:23:03,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +27: [2023-05-08 11:23:03,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +31: [2023-05-08 11:23:03,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +19: [2023-05-08 11:23:03,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:23:03,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +28: [2023-05-08 11:23:03,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +29: [2023-05-08 11:23:03,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:23:03,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +12: [2023-05-08 11:23:03,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +31: [2023-05-08 11:23:03,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +13: [2023-05-08 11:23:03,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +12: [2023-05-08 11:23:03,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:23:03,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +17: [2023-05-08 11:23:03,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +28: [2023-05-08 11:23:03,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +17: [2023-05-08 11:23:03,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +19: [2023-05-08 11:23:03,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:23:03,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:23:03,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:23:03,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +17: [2023-05-08 11:23:03,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:23:03,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:23:03,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:23:03,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +19: [2023-05-08 11:23:03,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +10: [2023-05-08 11:23:03,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +27: [2023-05-08 11:23:03,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:23:03,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +28: [2023-05-08 11:23:03,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +31: [2023-05-08 11:23:03,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +23: [2023-05-08 11:23:03,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +23: [2023-05-08 11:23:03,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:23:03,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:23:03,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +18: [2023-05-08 11:23:03,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +18: [2023-05-08 11:23:03,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +10: [2023-05-08 11:23:03,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:23:03,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 0: [2023-05-08 11:23:03,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:23:03,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:23:03,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:23:03,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +23: [2023-05-08 11:23:03,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +10: [2023-05-08 11:23:03,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +28: [2023-05-08 11:23:03,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +19: [2023-05-08 11:23:03,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +28: [2023-05-08 11:23:03,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:23:03,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:23:03,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:23:03,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +28: [2023-05-08 11:23:03,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +10: [2023-05-08 11:23:03,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:23:03,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:23:03,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +31: [2023-05-08 11:23:03,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:23:03,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +23: [2023-05-08 11:23:03,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:23:03,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +10: [2023-05-08 11:23:03,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +19: [2023-05-08 11:23:03,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:23:03,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:23:03,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +18: [2023-05-08 11:23:03,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +26: [2023-05-08 11:23:03,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:23:03,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +23: [2023-05-08 11:23:03,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:23:03,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +23: [2023-05-08 11:23:03,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +10: [2023-05-08 11:23:03,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +19: [2023-05-08 11:23:03,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +28: [2023-05-08 11:23:03,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:23:03,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +28: [2023-05-08 11:23:03,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +26: [2023-05-08 11:23:03,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:23:03,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:23:03,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt. +10: [2023-05-08 11:23:03,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +19: [2023-05-08 11:23:03,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +19: [2023-05-08 11:23:03,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +23: [2023-05-08 11:23:03,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:23:03,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:23:03,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 5: [2023-05-08 11:23:03,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +26: [2023-05-08 11:23:03,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +26: [2023-05-08 11:23:03,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:23:03,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:23:03,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 5: [2023-05-08 11:23:03,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:23:03,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:23:03,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 3: [2023-05-08 11:23:03,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 3: [2023-05-08 11:23:03,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 3: [2023-05-08 11:23:03,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 3: [2023-05-08 11:23:03,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +18: [2023-05-08 11:23:03,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:23:03,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 3: [2023-05-08 11:23:03,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +20: [2023-05-08 11:23:03,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:23:03,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +20: [2023-05-08 11:23:03,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:23:03,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:23:03,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +20: [2023-05-08 11:23:03,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +20: [2023-05-08 11:23:03,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:23:03,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 7: [2023-05-08 11:23:03,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 7: [2023-05-08 11:23:03,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 7: [2023-05-08 11:23:03,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 3: [2023-05-08 11:23:03,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +15: [2023-05-08 11:23:03,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:23:03,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:23:03,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:23:03,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:23:03,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:23:03,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:23:03,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:23:03,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +20: [2023-05-08 11:23:03,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:23:03,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:23:03,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:23:03,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +20: [2023-05-08 11:23:03,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:23:03,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:23:03,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +16: [2023-05-08 11:23:03,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +16: [2023-05-08 11:23:03,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +20: [2023-05-08 11:23:03,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:23:03,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +15: [2023-05-08 11:23:03,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:23:03,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 7: [2023-05-08 11:23:03,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:23:03,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +15: [2023-05-08 11:23:03,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:23:03,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:23:03,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +16: [2023-05-08 11:23:03,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:23:03,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:23:03,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +20: [2023-05-08 11:23:03,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:23:03,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:23:03,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:23:03,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +20: [2023-05-08 11:23:03,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +15: [2023-05-08 11:23:03,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:23:03,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:23:03,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 7: [2023-05-08 11:23:03,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +16: [2023-05-08 11:23:03,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +16: [2023-05-08 11:23:03,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +16: [2023-05-08 11:23:03,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +16: [2023-05-08 11:23:03,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +11: [2023-05-08 11:23:03,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +11: [2023-05-08 11:23:03,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +11: [2023-05-08 11:23:03,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +11: [2023-05-08 11:23:03,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +22: [2023-05-08 11:23:03,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:23:03,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:23:03,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:23:03,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +22: [2023-05-08 11:23:03,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +22: [2023-05-08 11:23:03,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +22: [2023-05-08 11:23:03,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:23:03,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +16: [2023-05-08 11:23:03,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:23:03,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:23:03,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:23:03,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:23:03,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:23:03,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:23:03,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:23:03,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:23:03,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +16: [2023-05-08 11:23:03,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:23:03,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +11: [2023-05-08 11:23:03,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +11: [2023-05-08 11:23:03,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +11: [2023-05-08 11:23:03,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +16: [2023-05-08 11:23:03,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:23:03,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +11: [2023-05-08 11:23:03,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +16: [2023-05-08 11:23:03,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:23:03,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +22: [2023-05-08 11:23:03,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:23:03,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:23:03,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:23:03,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:23:03,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:23:03,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:23:03,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:23:03,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:23:03,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:23:03,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:23:03,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +22: [2023-05-08 11:23:03,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:23:03,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:23:03,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:23:03,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:23:03,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +21: [2023-05-08 11:23:03,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +21: [2023-05-08 11:23:03,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +21: [2023-05-08 11:23:03,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +21: [2023-05-08 11:23:03,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:23:03,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:23:03,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:23:03,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:23:03,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:23:03,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:23:03,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 8: [2023-05-08 11:23:03,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 8: [2023-05-08 11:23:03,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 8: [2023-05-08 11:23:03,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +20: [2023-05-08 11:23:03,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +20: [2023-05-08 11:23:03,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +20: [2023-05-08 11:23:03,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +20: [2023-05-08 11:23:03,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 8: [2023-05-08 11:23:03,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +15: [2023-05-08 11:23:03,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +15: [2023-05-08 11:23:03,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +15: [2023-05-08 11:23:03,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +15: [2023-05-08 11:23:03,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +21: [2023-05-08 11:23:03,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:23:03,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:23:03,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:23:03,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +16: [2023-05-08 11:23:03,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:23:03,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +20: [2023-05-08 11:23:03,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +20: [2023-05-08 11:23:03,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:23:03,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +20: [2023-05-08 11:23:03,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:23:03,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:23:03,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:23:03,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:23:03,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:23:03,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:23:03,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:23:03,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:23:03,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +21: [2023-05-08 11:23:03,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +20: [2023-05-08 11:23:03,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:23:03,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +20: [2023-05-08 11:23:03,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:23:03,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +21: [2023-05-08 11:23:03,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:23:03,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:23:03,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:23:03,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +21: [2023-05-08 11:23:03,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +15: [2023-05-08 11:23:03,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:23:03,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:23:03,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:23:03,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:23:03,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +20: [2023-05-08 11:23:03,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +15: [2023-05-08 11:23:03,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:23:03,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +15: [2023-05-08 11:23:03,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +15: [2023-05-08 11:23:03,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +15: [2023-05-08 11:23:03,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +22: [2023-05-08 11:23:03,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +22: [2023-05-08 11:23:03,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +22: [2023-05-08 11:23:03,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +11: [2023-05-08 11:23:03,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:23:03,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +11: [2023-05-08 11:23:03,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +11: [2023-05-08 11:23:03,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +11: [2023-05-08 11:23:03,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +11: [2023-05-08 11:23:03,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +11: [2023-05-08 11:23:03,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +11: [2023-05-08 11:23:03,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +11: [2023-05-08 11:23:03,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +11: [2023-05-08 11:23:03,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +11: [2023-05-08 11:23:03,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +11: [2023-05-08 11:23:03,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +11: [2023-05-08 11:23:03,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +11: [2023-05-08 11:23:03,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +11: [2023-05-08 11:23:03,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +11: [2023-05-08 11:23:03,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 2: [2023-05-08 11:23:03,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +11: [2023-05-08 11:23:03,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 2: [2023-05-08 11:23:03,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 2: [2023-05-08 11:23:03,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 2: [2023-05-08 11:23:03,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +22: [2023-05-08 11:23:03,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:23:03,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +22: [2023-05-08 11:23:03,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:23:03,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +11: [2023-05-08 11:23:03,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +11: [2023-05-08 11:23:03,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +11: [2023-05-08 11:23:03,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:23:03,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:23:03,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +22: [2023-05-08 11:23:03,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:23:03,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 3: [2023-05-08 11:23:03,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:23:03,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:23:03,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +30: [2023-05-08 11:23:03,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +30: [2023-05-08 11:23:03,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +11: [2023-05-08 11:23:03,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:23:03,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +11: [2023-05-08 11:23:03,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:23:03,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:23:03,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:23:03,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +29: [2023-05-08 11:23:03,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +29: [2023-05-08 11:23:03,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +29: [2023-05-08 11:23:03,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +29: [2023-05-08 11:23:03,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 4: [2023-05-08 11:23:03,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 4: [2023-05-08 11:23:03,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 4: [2023-05-08 11:23:03,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:23:03,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +24: [2023-05-08 11:23:03,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +24: [2023-05-08 11:23:03,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 4: [2023-05-08 11:23:03,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +25: [2023-05-08 11:23:03,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +25: [2023-05-08 11:23:03,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +25: [2023-05-08 11:23:03,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +25: [2023-05-08 11:23:03,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +24: [2023-05-08 11:23:03,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 4: [2023-05-08 11:23:03,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +22: [2023-05-08 11:23:03,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:23:03,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +31: [2023-05-08 11:23:03,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +31: [2023-05-08 11:23:03,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +31: [2023-05-08 11:23:03,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +31: [2023-05-08 11:23:03,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +11: [2023-05-08 11:23:03,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:23:03,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +30: [2023-05-08 11:23:03,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:23:03,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +30: [2023-05-08 11:23:03,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +29: [2023-05-08 11:23:03,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +29: [2023-05-08 11:23:03,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:23:03,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:23:03,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:23:03,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:23:03,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +29: [2023-05-08 11:23:03,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +29: [2023-05-08 11:23:03,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:23:03,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:23:03,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:23:03,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +31: [2023-05-08 11:23:03,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:23:03,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +31: [2023-05-08 11:23:03,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +31: [2023-05-08 11:23:03,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 9: [2023-05-08 11:23:03,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 8: [2023-05-08 11:23:03,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 9: [2023-05-08 11:23:03,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 9: [2023-05-08 11:23:03,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 8: [2023-05-08 11:23:03,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 8: [2023-05-08 11:23:03,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 8: [2023-05-08 11:23:03,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:23:03,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:23:03,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +12: [2023-05-08 11:23:03,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +12: [2023-05-08 11:23:03,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +12: [2023-05-08 11:23:03,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 4: [2023-05-08 11:23:03,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:23:03,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:23:03,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +19: [2023-05-08 11:23:03,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +19: [2023-05-08 11:23:03,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +19: [2023-05-08 11:23:03,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +19: [2023-05-08 11:23:03,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +12: [2023-05-08 11:23:03,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +12: [2023-05-08 11:23:03,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:23:03,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 6: [2023-05-08 11:23:03,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 6: [2023-05-08 11:23:03,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +13: [2023-05-08 11:23:03,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +17: [2023-05-08 11:23:03,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +17: [2023-05-08 11:23:03,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 4: [2023-05-08 11:23:03,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 4: [2023-05-08 11:23:03,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 8: [2023-05-08 11:23:03,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +13: [2023-05-08 11:23:03,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 8: [2023-05-08 11:23:03,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +13: [2023-05-08 11:23:03,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 2: [2023-05-08 11:23:03,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 2: [2023-05-08 11:23:03,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 5: [2023-05-08 11:23:03,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:23:03,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 5: [2023-05-08 11:23:03,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 5: [2023-05-08 11:23:03,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:23:03,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 5: [2023-05-08 11:23:03,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +19: [2023-05-08 11:23:03,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 2: [2023-05-08 11:23:03,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +19: [2023-05-08 11:23:03,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 2: [2023-05-08 11:23:03,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 2: [2023-05-08 11:23:03,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 3: [2023-05-08 11:23:03,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +19: [2023-05-08 11:23:03,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 3: [2023-05-08 11:23:03,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 3: [2023-05-08 11:23:03,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 2: [2023-05-08 11:23:03,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +29: [2023-05-08 11:23:03,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +29: [2023-05-08 11:23:03,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +29: [2023-05-08 11:23:03,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +13: [2023-05-08 11:23:03,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +13: [2023-05-08 11:23:03,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +29: [2023-05-08 11:23:03,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +19: [2023-05-08 11:23:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +29: [2023-05-08 11:23:03,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:23:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +29: [2023-05-08 11:23:03,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:23:03,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:23:03,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:23:03,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:23:03,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:23:03,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +29: [2023-05-08 11:23:03,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +29: [2023-05-08 11:23:03,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:23:03,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +13: [2023-05-08 11:23:03,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +13: [2023-05-08 11:23:03,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:23:03,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +26: [2023-05-08 11:23:03,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +26: [2023-05-08 11:23:03,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +26: [2023-05-08 11:23:03,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +26: [2023-05-08 11:23:03,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +26: [2023-05-08 11:23:03,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +29: [2023-05-08 11:23:03,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +17: [2023-05-08 11:23:03,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +26: [2023-05-08 11:23:03,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:23:03,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:23:03,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +17: [2023-05-08 11:23:03,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:23:03,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +18: [2023-05-08 11:23:03,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +18: [2023-05-08 11:23:03,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 8: [2023-05-08 11:23:03,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +26: [2023-05-08 11:23:03,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +26: [2023-05-08 11:23:03,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:23:03,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:23:03,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:23:03,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:23:03,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +24: [2023-05-08 11:23:03,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:23:03,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:23:03,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:23:03,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +24: [2023-05-08 11:23:03,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +18: [2023-05-08 11:23:03,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:23:03,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +14: [2023-05-08 11:23:03,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +10: [2023-05-08 11:23:03,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +10: [2023-05-08 11:23:03,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 1: [2023-05-08 11:23:03,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 1: [2023-05-08 11:23:03,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +27: [2023-05-08 11:23:03,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 1: [2023-05-08 11:23:03,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +27: [2023-05-08 11:23:03,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +18: [2023-05-08 11:23:03,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:23:03,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +14: [2023-05-08 11:23:03,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 1: [2023-05-08 11:23:03,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +18: [2023-05-08 11:23:03,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +18: [2023-05-08 11:23:03,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:23:03,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +29: [2023-05-08 11:23:03,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:23:03,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:23:03,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:23:03,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:23:03,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:23:03,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:23:03,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 5: [2023-05-08 11:23:03,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:23:03,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:23:03,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +28: [2023-05-08 11:23:03,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +28: [2023-05-08 11:23:03,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 0: [2023-05-08 11:23:03,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 0: [2023-05-08 11:23:03,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +24: [2023-05-08 11:23:03,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 0: [2023-05-08 11:23:03,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 5: [2023-05-08 11:23:03,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 5: [2023-05-08 11:23:03,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 5: [2023-05-08 11:23:03,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +10: [2023-05-08 11:23:03,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 0: [2023-05-08 11:23:03,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 5: [2023-05-08 11:23:03,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +24: [2023-05-08 11:23:03,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +27: [2023-05-08 11:23:03,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 2: [2023-05-08 11:23:03,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:23:03,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:23:03,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +28: [2023-05-08 11:23:03,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 2: [2023-05-08 11:23:03,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:23:03,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +28: [2023-05-08 11:23:03,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +23: [2023-05-08 11:23:03,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 5: [2023-05-08 11:23:03,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +29: [2023-05-08 11:23:03,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:23:03,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +10: [2023-05-08 11:23:03,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +10: [2023-05-08 11:23:03,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +10: [2023-05-08 11:23:03,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:23:03,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:23:03,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:23:03,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:23:03,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:23:03,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +29: [2023-05-08 11:23:03,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +25: [2023-05-08 11:23:03,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:23:03,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:23:03,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:23:03,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 3: [2023-05-08 11:23:03,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 1: [2023-05-08 11:23:03,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:23:03,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:23:03,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:23:03,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +25: [2023-05-08 11:23:03,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:23:03,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +24: [2023-05-08 11:23:03,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:23:03,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:23:03,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:23:03,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:23:03,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:23:03,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:23:03,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +28: [2023-05-08 11:23:03,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +26: [2023-05-08 11:23:03,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +26: [2023-05-08 11:23:03,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:23:03,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:23:03,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +26: [2023-05-08 11:23:03,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +23: [2023-05-08 11:23:03,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +26: [2023-05-08 11:23:03,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +18: [2023-05-08 11:23:03,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:23:03,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +26: [2023-05-08 11:23:03,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. +18: [2023-05-08 11:23:03,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +26: [2023-05-08 11:23:03,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt. + 2: [2023-05-08 11:23:03,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +25: [2023-05-08 11:23:03,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 0: [2023-05-08 11:23:03,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:23:03,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:23:03,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +29: [2023-05-08 11:23:03,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +28: [2023-05-08 11:23:03,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +23: [2023-05-08 11:23:03,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:23:03,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:23:03,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +18: [2023-05-08 11:23:03,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +18: [2023-05-08 11:23:03,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +18: [2023-05-08 11:23:03,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +18: [2023-05-08 11:23:03,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +18: [2023-05-08 11:23:03,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:23:03,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:23:03,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +28: [2023-05-08 11:23:03,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +18: [2023-05-08 11:23:03,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +18: [2023-05-08 11:23:03,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 5: [2023-05-08 11:23:03,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 5: [2023-05-08 11:23:03,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +18: [2023-05-08 11:23:03,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +18: [2023-05-08 11:23:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 5: [2023-05-08 11:23:03,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:23:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:23:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:23:03,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 6: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +30: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +30: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +12: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +13: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +13: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +13: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +13: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +13: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +30: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +13: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +13: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +13: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +13: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +12: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +13: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +13: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +12: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +12: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +13: [2023-05-08 11:23:03,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 6: [2023-05-08 11:23:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +13: [2023-05-08 11:23:03,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 6: [2023-05-08 11:23:03,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +13: [2023-05-08 11:23:03,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 6: [2023-05-08 11:23:03,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +13: [2023-05-08 11:23:03,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 6: [2023-05-08 11:23:03,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +12: [2023-05-08 11:23:03,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:23:03,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +13: [2023-05-08 11:23:03,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +12: [2023-05-08 11:23:03,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +29: [2023-05-08 11:23:03,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:23:03,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +30: [2023-05-08 11:23:03,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +30: [2023-05-08 11:23:03,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +12: [2023-05-08 11:23:03,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +12: [2023-05-08 11:23:03,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:23:03,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +18: [2023-05-08 11:23:03,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 7: [2023-05-08 11:23:03,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 7: [2023-05-08 11:23:03,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +30: [2023-05-08 11:23:03,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +12: [2023-05-08 11:23:03,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +31: [2023-05-08 11:23:03,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +31: [2023-05-08 11:23:03,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +31: [2023-05-08 11:23:03,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +31: [2023-05-08 11:23:03,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +31: [2023-05-08 11:23:03,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +31: [2023-05-08 11:23:03,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +31: [2023-05-08 11:23:03,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +31: [2023-05-08 11:23:03,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +18: [2023-05-08 11:23:03,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:23:03,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +31: [2023-05-08 11:23:03,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +31: [2023-05-08 11:23:03,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +26: [2023-05-08 11:23:03,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +26: [2023-05-08 11:23:03,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:23:03,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +18: [2023-05-08 11:23:03,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +26: [2023-05-08 11:23:03,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +26: [2023-05-08 11:23:03,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +31: [2023-05-08 11:23:03,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +31: [2023-05-08 11:23:03,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +18: [2023-05-08 11:23:03,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:23:03,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:23:03,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +18: [2023-05-08 11:23:03,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 0: [2023-05-08 11:23:03,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +31: [2023-05-08 11:23:03,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 5: [2023-05-08 11:23:03,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +26: [2023-05-08 11:23:03,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:23:03,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:23:03,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +26: [2023-05-08 11:23:03,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:23:03,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +31: [2023-05-08 11:23:03,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +26: [2023-05-08 11:23:03,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +26: [2023-05-08 11:23:03,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:23:03,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +12: [2023-05-08 11:23:03,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:23:03,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +13: [2023-05-08 11:23:03,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:23:03,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:23:03,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 0: [2023-05-08 11:23:03,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +24: [2023-05-08 11:23:03,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:23:03,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:23:03,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +18: [2023-05-08 11:23:03,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:23:03,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +13: [2023-05-08 11:23:03,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:23:03,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:23:03,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +25: [2023-05-08 11:23:03,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +31: [2023-05-08 11:23:03,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:23:03,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:23:03,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:23:03,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +26: [2023-05-08 11:23:03,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +31: [2023-05-08 11:23:03,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:23:03,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +26: [2023-05-08 11:23:03,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:23:03,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +26: [2023-05-08 11:23:03,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:23:03,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +12: [2023-05-08 11:23:03,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +26: [2023-05-08 11:23:03,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 0: [2023-05-08 11:23:03,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:23:03,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +13: [2023-05-08 11:23:03,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +12: [2023-05-08 11:23:03,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +31: [2023-05-08 11:23:03,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 0: [2023-05-08 11:23:03,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 0: [2023-05-08 11:23:03,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +13: [2023-05-08 11:23:03,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +31: [2023-05-08 11:23:03,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +13: [2023-05-08 11:23:03,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +26: [2023-05-08 11:23:03,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +13: [2023-05-08 11:23:03,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +26: [2023-05-08 11:23:03,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 5: [2023-05-08 11:23:03,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:23:03,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:23:03,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 0: [2023-05-08 11:23:03,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +26: [2023-05-08 11:23:03,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +31: [2023-05-08 11:23:03,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +31: [2023-05-08 11:23:03,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +26: [2023-05-08 11:23:03,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 0: [2023-05-08 11:23:03,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +12: [2023-05-08 11:23:03,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +13: [2023-05-08 11:23:03,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:23:03,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 7: [2023-05-08 11:23:03,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 7: [2023-05-08 11:23:03,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +26: [2023-05-08 11:23:03,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +26: [2023-05-08 11:23:03,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 7: [2023-05-08 11:23:03,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 0: [2023-05-08 11:23:03,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 9: [2023-05-08 11:23:03,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 9: [2023-05-08 11:23:03,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 9: [2023-05-08 11:23:03,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +12: [2023-05-08 11:23:03,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +31: [2023-05-08 11:23:03,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:23:03,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:23:03,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +17: [2023-05-08 11:23:03,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:23:03,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:23:03,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:23:03,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:23:03,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:23:03,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:23:03,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:23:03,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:23:03,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:23:03,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:23:03,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:23:03,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:23:03,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +17: [2023-05-08 11:23:03,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +17: [2023-05-08 11:23:03,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +10: [2023-05-08 11:23:03,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +10: [2023-05-08 11:23:03,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +10: [2023-05-08 11:23:03,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +10: [2023-05-08 11:23:03,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +10: [2023-05-08 11:23:03,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +10: [2023-05-08 11:23:03,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +10: [2023-05-08 11:23:03,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +10: [2023-05-08 11:23:03,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +10: [2023-05-08 11:23:03,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +10: [2023-05-08 11:23:03,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +10: [2023-05-08 11:23:03,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +10: [2023-05-08 11:23:03,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +10: [2023-05-08 11:23:03,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +10: [2023-05-08 11:23:03,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +10: [2023-05-08 11:23:03,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +28: [2023-05-08 11:23:03,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +28: [2023-05-08 11:23:03,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:23:03,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +28: [2023-05-08 11:23:03,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +28: [2023-05-08 11:23:03,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +28: [2023-05-08 11:23:03,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +10: [2023-05-08 11:23:03,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +28: [2023-05-08 11:23:03,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +28: [2023-05-08 11:23:03,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +28: [2023-05-08 11:23:03,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +28: [2023-05-08 11:23:03,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +28: [2023-05-08 11:23:03,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +28: [2023-05-08 11:23:03,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +28: [2023-05-08 11:23:03,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +28: [2023-05-08 11:23:03,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +28: [2023-05-08 11:23:03,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +28: [2023-05-08 11:23:03,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +28: [2023-05-08 11:23:03,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:23:03,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +14: [2023-05-08 11:23:03,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:23:03,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +14: [2023-05-08 11:23:03,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +14: [2023-05-08 11:23:03,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:23:03,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +14: [2023-05-08 11:23:03,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +14: [2023-05-08 11:23:03,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +14: [2023-05-08 11:23:03,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +14: [2023-05-08 11:23:03,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +14: [2023-05-08 11:23:03,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:23:03,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:23:03,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +14: [2023-05-08 11:23:03,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +14: [2023-05-08 11:23:03,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 9: [2023-05-08 11:23:03,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:23:03,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 9: [2023-05-08 11:23:03,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:23:03,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:23:03,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +23: [2023-05-08 11:23:03,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:23:03,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:23:03,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:23:03,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:23:03,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:23:03,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:23:03,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +23: [2023-05-08 11:23:03,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +23: [2023-05-08 11:23:03,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +23: [2023-05-08 11:23:03,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +23: [2023-05-08 11:23:03,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +23: [2023-05-08 11:23:03,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +23: [2023-05-08 11:23:03,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:23:03,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 7: [2023-05-08 11:23:03,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:23:03,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:23:03,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:23:03,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:23:03,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:23:03,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:23:03,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:23:03,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:23:03,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +27: [2023-05-08 11:23:03,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:23:03,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +27: [2023-05-08 11:23:03,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:23:03,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:23:03,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:23:03,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:23:03,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +27: [2023-05-08 11:23:03,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 1: [2023-05-08 11:23:03,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 1: [2023-05-08 11:23:03,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... + 1: [2023-05-08 11:23:03,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +10: [2023-05-08 11:23:03,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +19: [2023-05-08 11:23:03,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +19: [2023-05-08 11:23:03,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +19: [2023-05-08 11:23:03,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +19: [2023-05-08 11:23:03,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +19: [2023-05-08 11:23:03,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:23:03,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +19: [2023-05-08 11:23:03,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +19: [2023-05-08 11:23:03,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +19: [2023-05-08 11:23:03,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +28: [2023-05-08 11:23:03,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +19: [2023-05-08 11:23:03,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +19: [2023-05-08 11:23:03,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:23:03,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +19: [2023-05-08 11:23:03,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +28: [2023-05-08 11:23:03,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +19: [2023-05-08 11:23:03,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +19: [2023-05-08 11:23:03,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... +19: [2023-05-08 11:23:03,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +19: [2023-05-08 11:23:03,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +14: [2023-05-08 11:23:03,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:23:03,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt... +17: [2023-05-08 11:23:03,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:23:03,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +10: [2023-05-08 11:23:03,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +28: [2023-05-08 11:23:03,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +14: [2023-05-08 11:23:03,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +14: [2023-05-08 11:23:03,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:23:03,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:23:03,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +28: [2023-05-08 11:23:03,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:23:03,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:23:03,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:23:03,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:23:03,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:23:03,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:23:03,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:23:03,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:23:03,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:23:03,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:23:03,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:23:03,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:23:03,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:23:03,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +15: [2023-05-08 11:23:03,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +15: [2023-05-08 11:23:03,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +28: [2023-05-08 11:23:03,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:23:03,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +28: [2023-05-08 11:23:03,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +15: [2023-05-08 11:23:03,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:23:03,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +17: [2023-05-08 11:23:03,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +19: [2023-05-08 11:23:03,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:23:03,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:23:03,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +10: [2023-05-08 11:23:03,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +17: [2023-05-08 11:23:03,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +28: [2023-05-08 11:23:03,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:23:03,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:23:03,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:23:03,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:23:03,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:23:03,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +28: [2023-05-08 11:23:03,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:23:03,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +19: [2023-05-08 11:23:03,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +19: [2023-05-08 11:23:03,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:23:03,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +19: [2023-05-08 11:23:03,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:23:03,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:23:03,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:23:03,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:23:03,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:23:03,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:23:03,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:23:03,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:23:03,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:23:03,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:23:03,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:23:03,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:23:03,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:23:03,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:23:03,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:23:03,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:23:03,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +19: [2023-05-08 11:23:03,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:23:03,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:23:03,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:23:03,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +19: [2023-05-08 11:23:03,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +15: [2023-05-08 11:23:03,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +15: [2023-05-08 11:23:03,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +20: [2023-05-08 11:23:03,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:23:03,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +19: [2023-05-08 11:23:03,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +15: [2023-05-08 11:23:03,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:23:03,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:23:03,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:23:03,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:23:03,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:23:03,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +20: [2023-05-08 11:23:03,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +20: [2023-05-08 11:23:03,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:23:03,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:23:03,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +15: [2023-05-08 11:23:03,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +15: [2023-05-08 11:23:03,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +15: [2023-05-08 11:23:03,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 2: [2023-05-08 11:23:03,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 2: [2023-05-08 11:23:03,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 2: [2023-05-08 11:23:03,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 2: [2023-05-08 11:23:03,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +20: [2023-05-08 11:23:03,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +20: [2023-05-08 11:23:03,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +20: [2023-05-08 11:23:03,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +20: [2023-05-08 11:23:03,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +21: [2023-05-08 11:23:03,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +21: [2023-05-08 11:23:03,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +21: [2023-05-08 11:23:03,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +21: [2023-05-08 11:23:03,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +15: [2023-05-08 11:23:03,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +15: [2023-05-08 11:23:03,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 8: [2023-05-08 11:23:03,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 8: [2023-05-08 11:23:03,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 8: [2023-05-08 11:23:03,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 2: [2023-05-08 11:23:03,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +15: [2023-05-08 11:23:03,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +15: [2023-05-08 11:23:03,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:23:03,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:23:03,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:23:03,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:23:03,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:23:03,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:23:03,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:23:03,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:23:03,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:23:03,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +16: [2023-05-08 11:23:03,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +16: [2023-05-08 11:23:03,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +16: [2023-05-08 11:23:03,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 8: [2023-05-08 11:23:03,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +15: [2023-05-08 11:23:03,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:23:03,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +20: [2023-05-08 11:23:03,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:23:03,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:23:03,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +20: [2023-05-08 11:23:03,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +15: [2023-05-08 11:23:03,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:23:03,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +16: [2023-05-08 11:23:03,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:23:03,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:23:03,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:23:03,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +16: [2023-05-08 11:23:03,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:23:03,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 6: [2023-05-08 11:23:03,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 6: [2023-05-08 11:23:03,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 6: [2023-05-08 11:23:03,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +15: [2023-05-08 11:23:03,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:23:03,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:23:03,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +15: [2023-05-08 11:23:03,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +20: [2023-05-08 11:23:03,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +20: [2023-05-08 11:23:03,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 6: [2023-05-08 11:23:03,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:23:03,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +12: [2023-05-08 11:23:03,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +12: [2023-05-08 11:23:03,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +12: [2023-05-08 11:23:03,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 3: [2023-05-08 11:23:03,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:23:03,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +18: [2023-05-08 11:23:03,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +18: [2023-05-08 11:23:03,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +18: [2023-05-08 11:23:03,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +18: [2023-05-08 11:23:03,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +18: [2023-05-08 11:23:03,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +18: [2023-05-08 11:23:03,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +18: [2023-05-08 11:23:03,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:23:03,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +12: [2023-05-08 11:23:03,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:23:03,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +12: [2023-05-08 11:23:03,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +29: [2023-05-08 11:23:03,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +29: [2023-05-08 11:23:03,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +29: [2023-05-08 11:23:03,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +29: [2023-05-08 11:23:03,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 3: [2023-05-08 11:23:03,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:23:03,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +18: [2023-05-08 11:23:03,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:23:03,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +18: [2023-05-08 11:23:03,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:23:03,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +16: [2023-05-08 11:23:03,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +16: [2023-05-08 11:23:03,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +16: [2023-05-08 11:23:03,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +18: [2023-05-08 11:23:03,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:23:03,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +16: [2023-05-08 11:23:03,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +16: [2023-05-08 11:23:03,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +16: [2023-05-08 11:23:03,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +16: [2023-05-08 11:23:03,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:23:03,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:23:03,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +16: [2023-05-08 11:23:03,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +29: [2023-05-08 11:23:03,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:23:03,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:23:03,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +16: [2023-05-08 11:23:03,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +29: [2023-05-08 11:23:03,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +29: [2023-05-08 11:23:03,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +29: [2023-05-08 11:23:03,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:23:03,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:23:03,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:23:03,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:23:03,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:23:03,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:23:03,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:23:03,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:23:03,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +18: [2023-05-08 11:23:03,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:23:03,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:23:03,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:23:03,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 3: [2023-05-08 11:23:03,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 3: [2023-05-08 11:23:03,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 4: [2023-05-08 11:23:03,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 7: [2023-05-08 11:23:03,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:23:03,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 7: [2023-05-08 11:23:03,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:23:03,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +22: [2023-05-08 11:23:03,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +22: [2023-05-08 11:23:03,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 4: [2023-05-08 11:23:03,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:23:03,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:23:03,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:23:03,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +16: [2023-05-08 11:23:03,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +16: [2023-05-08 11:23:03,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +16: [2023-05-08 11:23:03,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 4: [2023-05-08 11:23:03,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 4: [2023-05-08 11:23:03,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:23:03,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 3: [2023-05-08 11:23:03,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +11: [2023-05-08 11:23:03,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +11: [2023-05-08 11:23:03,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +16: [2023-05-08 11:23:03,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:23:03,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +11: [2023-05-08 11:23:03,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:23:03,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:23:03,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +26: [2023-05-08 11:23:03,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +26: [2023-05-08 11:23:03,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +22: [2023-05-08 11:23:03,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:23:03,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:23:03,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +11: [2023-05-08 11:23:03,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:23:03,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +26: [2023-05-08 11:23:03,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +26: [2023-05-08 11:23:03,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 7: [2023-05-08 11:23:03,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 7: [2023-05-08 11:23:03,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 4: [2023-05-08 11:23:03,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +18: [2023-05-08 11:23:03,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +18: [2023-05-08 11:23:03,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +18: [2023-05-08 11:23:03,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +11: [2023-05-08 11:23:03,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +18: [2023-05-08 11:23:03,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 7: [2023-05-08 11:23:03,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +26: [2023-05-08 11:23:03,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:23:03,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +24: [2023-05-08 11:23:03,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +24: [2023-05-08 11:23:03,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 7: [2023-05-08 11:23:03,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:23:03,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +26: [2023-05-08 11:23:03,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:23:03,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +11: [2023-05-08 11:23:03,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +11: [2023-05-08 11:23:03,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +11: [2023-05-08 11:23:03,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +26: [2023-05-08 11:23:03,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +26: [2023-05-08 11:23:03,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +13: [2023-05-08 11:23:03,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +13: [2023-05-08 11:23:03,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +13: [2023-05-08 11:23:03,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +13: [2023-05-08 11:23:03,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 3: [2023-05-08 11:23:03,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 3: [2023-05-08 11:23:03,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 4: [2023-05-08 11:23:03,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 7: [2023-05-08 11:23:03,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +18: [2023-05-08 11:23:03,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +18: [2023-05-08 11:23:03,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:23:03,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +31: [2023-05-08 11:23:03,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +18: [2023-05-08 11:23:03,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +18: [2023-05-08 11:23:03,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:23:03,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +31: [2023-05-08 11:23:03,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 7: [2023-05-08 11:23:03,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:23:03,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 4: [2023-05-08 11:23:03,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +31: [2023-05-08 11:23:03,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 3: [2023-05-08 11:23:03,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +11: [2023-05-08 11:23:03,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +24: [2023-05-08 11:23:03,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +24: [2023-05-08 11:23:03,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:23:03,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:23:03,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +22: [2023-05-08 11:23:03,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:23:03,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:23:03,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 3: [2023-05-08 11:23:03,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +11: [2023-05-08 11:23:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:23:03,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:23:03,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:23:03,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:23:03,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:23:03,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:23:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:23:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +13: [2023-05-08 11:23:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:23:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +13: [2023-05-08 11:23:03,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:23:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +22: [2023-05-08 11:23:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +22: [2023-05-08 11:23:03,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:23:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:23:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +31: [2023-05-08 11:23:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +13: [2023-05-08 11:23:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +31: [2023-05-08 11:23:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:23:03,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 4: [2023-05-08 11:23:03,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +18: [2023-05-08 11:23:03,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +18: [2023-05-08 11:23:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +31: [2023-05-08 11:23:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +18: [2023-05-08 11:23:03,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +18: [2023-05-08 11:23:03,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 4: [2023-05-08 11:23:03,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +31: [2023-05-08 11:23:03,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 7: [2023-05-08 11:23:03,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 7: [2023-05-08 11:23:03,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +18: [2023-05-08 11:23:03,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +11: [2023-05-08 11:23:03,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +18: [2023-05-08 11:23:03,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +18: [2023-05-08 11:23:03,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +18: [2023-05-08 11:23:03,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +11: [2023-05-08 11:23:03,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +11: [2023-05-08 11:23:03,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:23:03,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:23:03,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:23:03,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +11: [2023-05-08 11:23:03,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +22: [2023-05-08 11:23:03,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +22: [2023-05-08 11:23:03,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:23:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +20: [2023-05-08 11:23:03,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:23:03,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:23:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +20: [2023-05-08 11:23:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:23:03,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:23:03,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:23:03,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:23:03,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:23:03,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:23:03,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:23:03,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:23:03,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:23:03,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:23:03,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:23:03,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:23:03,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:23:03,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:23:03,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +11: [2023-05-08 11:23:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 5: [2023-05-08 11:23:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 5: [2023-05-08 11:23:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 5: [2023-05-08 11:23:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +11: [2023-05-08 11:23:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +24: [2023-05-08 11:23:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:23:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:23:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:23:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:23:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:23:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:23:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:23:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +24: [2023-05-08 11:23:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +24: [2023-05-08 11:23:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +24: [2023-05-08 11:23:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +24: [2023-05-08 11:23:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +24: [2023-05-08 11:23:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +24: [2023-05-08 11:23:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:23:03,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:23:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:23:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +22: [2023-05-08 11:23:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +30: [2023-05-08 11:23:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:23:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:23:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:23:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:23:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:23:03,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:23:03,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:23:03,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:23:03,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:23:03,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +19: [2023-05-08 11:23:03,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +19: [2023-05-08 11:23:03,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +19: [2023-05-08 11:23:03,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 5: [2023-05-08 11:23:03,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +15: [2023-05-08 11:23:03,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:23:03,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +15: [2023-05-08 11:23:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:23:03,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:23:03,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 0: [2023-05-08 11:23:03,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +30: [2023-05-08 11:23:03,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +30: [2023-05-08 11:23:03,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +30: [2023-05-08 11:23:03,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +20: [2023-05-08 11:23:03,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:23:03,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +17: [2023-05-08 11:23:03,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +17: [2023-05-08 11:23:03,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +17: [2023-05-08 11:23:03,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +24: [2023-05-08 11:23:03,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 9: [2023-05-08 11:23:03,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 9: [2023-05-08 11:23:03,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 9: [2023-05-08 11:23:03,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +20: [2023-05-08 11:23:03,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:23:03,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:23:03,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +10: [2023-05-08 11:23:03,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +10: [2023-05-08 11:23:03,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +10: [2023-05-08 11:23:03,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 0: [2023-05-08 11:23:03,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +10: [2023-05-08 11:23:03,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +24: [2023-05-08 11:23:03,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +19: [2023-05-08 11:23:03,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:23:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:23:03,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:23:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:23:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:23:03,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +19: [2023-05-08 11:23:03,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +19: [2023-05-08 11:23:03,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:23:03,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +15: [2023-05-08 11:23:03,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +15: [2023-05-08 11:23:03,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +15: [2023-05-08 11:23:03,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +24: [2023-05-08 11:23:03,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +30: [2023-05-08 11:23:03,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:23:03,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:23:03,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:23:03,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +10: [2023-05-08 11:23:03,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +17: [2023-05-08 11:23:03,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +24: [2023-05-08 11:23:03,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:23:03,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +17: [2023-05-08 11:23:03,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:23:03,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:23:03,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:23:03,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:23:03,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +30: [2023-05-08 11:23:03,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:23:03,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:23:03,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:23:03,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +20: [2023-05-08 11:23:03,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +20: [2023-05-08 11:23:03,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +24: [2023-05-08 11:23:03,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +20: [2023-05-08 11:23:03,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +24: [2023-05-08 11:23:03,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +17: [2023-05-08 11:23:03,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +17: [2023-05-08 11:23:03,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:23:03,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:23:03,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +30: [2023-05-08 11:23:03,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +20: [2023-05-08 11:23:03,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +20: [2023-05-08 11:23:03,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +30: [2023-05-08 11:23:03,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +20: [2023-05-08 11:23:03,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:23:03,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +14: [2023-05-08 11:23:03,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +14: [2023-05-08 11:23:03,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +30: [2023-05-08 11:23:03,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:23:03,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:23:03,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +15: [2023-05-08 11:23:03,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +15: [2023-05-08 11:23:03,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +30: [2023-05-08 11:23:03,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +30: [2023-05-08 11:23:03,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +15: [2023-05-08 11:23:03,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +15: [2023-05-08 11:23:03,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +30: [2023-05-08 11:23:03,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 9: [2023-05-08 11:23:03,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:23:03,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:23:03,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +30: [2023-05-08 11:23:03,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:23:03,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +28: [2023-05-08 11:23:03,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +28: [2023-05-08 11:23:03,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +28: [2023-05-08 11:23:03,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +20: [2023-05-08 11:23:03,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:23:03,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +27: [2023-05-08 11:23:03,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +27: [2023-05-08 11:23:03,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +30: [2023-05-08 11:23:03,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 9: [2023-05-08 11:23:03,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +27: [2023-05-08 11:23:03,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +15: [2023-05-08 11:23:03,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +27: [2023-05-08 11:23:03,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +15: [2023-05-08 11:23:03,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:23:03,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +15: [2023-05-08 11:23:03,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 1: [2023-05-08 11:23:03,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +23: [2023-05-08 11:23:03,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 1: [2023-05-08 11:23:03,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 1: [2023-05-08 11:23:03,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +23: [2023-05-08 11:23:03,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +23: [2023-05-08 11:23:03,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +23: [2023-05-08 11:23:03,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 9: [2023-05-08 11:23:03,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +14: [2023-05-08 11:23:03,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:23:03,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:23:03,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:23:03,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +20: [2023-05-08 11:23:03,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +20: [2023-05-08 11:23:03,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:23:03,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +14: [2023-05-08 11:23:03,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +28: [2023-05-08 11:23:03,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:23:03,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:23:03,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:23:03,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:23:03,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +28: [2023-05-08 11:23:03,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +27: [2023-05-08 11:23:03,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:23:03,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 5: [2023-05-08 11:23:03,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 5: [2023-05-08 11:23:03,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 5: [2023-05-08 11:23:03,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:23:03,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:23:03,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +23: [2023-05-08 11:23:03,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:23:03,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +25: [2023-05-08 11:23:03,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +25: [2023-05-08 11:23:03,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. + 1: [2023-05-08 11:23:03,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +25: [2023-05-08 11:23:03,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +23: [2023-05-08 11:23:03,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +25: [2023-05-08 11:23:03,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt. +15: [2023-05-08 11:23:03,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:23:03,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +15: [2023-05-08 11:23:03,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:23:03,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +15: [2023-05-08 11:23:03,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:23:03,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +15: [2023-05-08 11:23:03,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 9: [2023-05-08 11:23:03,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:23:03,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:23:03,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:23:03,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:23:03,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:23:03,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:23:03,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:23:03,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 9: [2023-05-08 11:23:03,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:23:03,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +25: [2023-05-08 11:23:03,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +25: [2023-05-08 11:23:03,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +25: [2023-05-08 11:23:03,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +25: [2023-05-08 11:23:03,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:23:03,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:23:03,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +14: [2023-05-08 11:23:03,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 5: [2023-05-08 11:23:03,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:23:03,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 0: [2023-05-08 11:23:03,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:23:03,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:23:03,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:23:03,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:23:03,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:23:03,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:23:03,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 8: [2023-05-08 11:23:03,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 0: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 8: [2023-05-08 11:23:03,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 2: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 2: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 8: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 2: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 8: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 8: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +27: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +27: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +27: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:23:03,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:23:03,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:23:03,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +23: [2023-05-08 11:23:03,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:23:03,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +27: [2023-05-08 11:23:03,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +27: [2023-05-08 11:23:03,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:23:03,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:23:03,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:23:03,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:23:03,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +27: [2023-05-08 11:23:03,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:23:03,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:23:03,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +21: [2023-05-08 11:23:03,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:23:03,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +27: [2023-05-08 11:23:03,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +27: [2023-05-08 11:23:03,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +27: [2023-05-08 11:23:03,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 0: [2023-05-08 11:23:03,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 0: [2023-05-08 11:23:03,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:23:03,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:23:03,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 8: [2023-05-08 11:23:03,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +21: [2023-05-08 11:23:03,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +21: [2023-05-08 11:23:03,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:23:03,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:23:03,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:23:03,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +27: [2023-05-08 11:23:03,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +19: [2023-05-08 11:23:03,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:23:03,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:23:03,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:23:03,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:23:03,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:23:03,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +19: [2023-05-08 11:23:03,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:23:03,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:23:03,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +19: [2023-05-08 11:23:03,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:23:03,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:23:03,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +19: [2023-05-08 11:23:03,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +19: [2023-05-08 11:23:03,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:23:03,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +19: [2023-05-08 11:23:03,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:23:03,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:23:03,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:23:03,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +19: [2023-05-08 11:23:03,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:23:03,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:23:03,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +25: [2023-05-08 11:23:03,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +25: [2023-05-08 11:23:03,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +25: [2023-05-08 11:23:03,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +25: [2023-05-08 11:23:03,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +25: [2023-05-08 11:23:03,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:23:03,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +29: [2023-05-08 11:23:03,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +29: [2023-05-08 11:23:03,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +29: [2023-05-08 11:23:03,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +25: [2023-05-08 11:23:03,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +29: [2023-05-08 11:23:03,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +29: [2023-05-08 11:23:03,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +29: [2023-05-08 11:23:03,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +29: [2023-05-08 11:23:03,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +29: [2023-05-08 11:23:03,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +29: [2023-05-08 11:23:03,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:23:03,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +11: [2023-05-08 11:23:03,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +29: [2023-05-08 11:23:03,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +29: [2023-05-08 11:23:03,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +29: [2023-05-08 11:23:03,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +29: [2023-05-08 11:23:03,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +29: [2023-05-08 11:23:03,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 5: [2023-05-08 11:23:03,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:23:03,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +29: [2023-05-08 11:23:03,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +25: [2023-05-08 11:23:03,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +25: [2023-05-08 11:23:03,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +29: [2023-05-08 11:23:03,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:23:03,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:23:03,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +25: [2023-05-08 11:23:03,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +25: [2023-05-08 11:23:03,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +25: [2023-05-08 11:23:03,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +12: [2023-05-08 11:23:03,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:23:03,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:23:03,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +12: [2023-05-08 11:23:03,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +11: [2023-05-08 11:23:03,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 8: [2023-05-08 11:23:03,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:23:03,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +12: [2023-05-08 11:23:03,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:23:03,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +12: [2023-05-08 11:23:03,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:23:03,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +12: [2023-05-08 11:23:03,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:23:03,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:23:03,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:23:03,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +12: [2023-05-08 11:23:03,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +12: [2023-05-08 11:23:03,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:23:03,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +14: [2023-05-08 11:23:03,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:23:03,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +11: [2023-05-08 11:23:03,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 0: [2023-05-08 11:23:03,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:23:03,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:23:03,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +26: [2023-05-08 11:23:03,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +27: [2023-05-08 11:23:03,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +27: [2023-05-08 11:23:03,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +26: [2023-05-08 11:23:03,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +26: [2023-05-08 11:23:03,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +26: [2023-05-08 11:23:03,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +26: [2023-05-08 11:23:03,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +26: [2023-05-08 11:23:03,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +26: [2023-05-08 11:23:03,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +27: [2023-05-08 11:23:03,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +27: [2023-05-08 11:23:03,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +26: [2023-05-08 11:23:03,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +13: [2023-05-08 11:23:03,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +26: [2023-05-08 11:23:03,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +26: [2023-05-08 11:23:03,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +26: [2023-05-08 11:23:03,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +13: [2023-05-08 11:23:03,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +13: [2023-05-08 11:23:03,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +13: [2023-05-08 11:23:03,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +13: [2023-05-08 11:23:03,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +13: [2023-05-08 11:23:03,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +13: [2023-05-08 11:23:03,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +13: [2023-05-08 11:23:03,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +13: [2023-05-08 11:23:03,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +13: [2023-05-08 11:23:03,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +13: [2023-05-08 11:23:03,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +13: [2023-05-08 11:23:03,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +13: [2023-05-08 11:23:03,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +27: [2023-05-08 11:23:03,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:23:03,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:23:03,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:23:03,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +26: [2023-05-08 11:23:03,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +26: [2023-05-08 11:23:03,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:23:03,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:23:03,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +26: [2023-05-08 11:23:03,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 1: [2023-05-08 11:23:03,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 2: [2023-05-08 11:23:03,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 1: [2023-05-08 11:23:03,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 4: [2023-05-08 11:23:03,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 4: [2023-05-08 11:23:03,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +13: [2023-05-08 11:23:03,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:23:03,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +14: [2023-05-08 11:23:03,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +13: [2023-05-08 11:23:03,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 2: [2023-05-08 11:23:03,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +26: [2023-05-08 11:23:03,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:23:03,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:23:03,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +13: [2023-05-08 11:23:03,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +26: [2023-05-08 11:23:03,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:23:03,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +27: [2023-05-08 11:23:03,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 1: [2023-05-08 11:23:03,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:23:03,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:23:03,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +25: [2023-05-08 11:23:03,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 1: [2023-05-08 11:23:03,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 1: [2023-05-08 11:23:03,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +21: [2023-05-08 11:23:03,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:23:03,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:23:03,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +19: [2023-05-08 11:23:03,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:23:03,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:23:03,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +25: [2023-05-08 11:23:03,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +25: [2023-05-08 11:23:03,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +11: [2023-05-08 11:23:03,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +29: [2023-05-08 11:23:03,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +25: [2023-05-08 11:23:03,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +23: [2023-05-08 11:23:03,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +29: [2023-05-08 11:23:03,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +11: [2023-05-08 11:23:03,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:23:03,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:23:03,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:23:03,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:23:03,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 2: [2023-05-08 11:23:03,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:23:03,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:23:03,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +23: [2023-05-08 11:23:03,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:23:03,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +25: [2023-05-08 11:23:03,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:23:03,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:23:03,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:23:03,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +13: [2023-05-08 11:23:03,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +27: [2023-05-08 11:23:03,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +19: [2023-05-08 11:23:03,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 5: [2023-05-08 11:23:03,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 8: [2023-05-08 11:23:03,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +13: [2023-05-08 11:23:03,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:23:03,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +19: [2023-05-08 11:23:03,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:23:03,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:23:03,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:23:03,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +23: [2023-05-08 11:23:03,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:23:03,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:23:03,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +22: [2023-05-08 11:23:03,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +29: [2023-05-08 11:23:03,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:23:03,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +29: [2023-05-08 11:23:03,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 2: [2023-05-08 11:23:03,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:23:03,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:23:03,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +25: [2023-05-08 11:23:03,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +21: [2023-05-08 11:23:03,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:23:03,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +29: [2023-05-08 11:23:03,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:23:03,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +19: [2023-05-08 11:23:03,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:23:03,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 1: [2023-05-08 11:23:03,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +29: [2023-05-08 11:23:03,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +13: [2023-05-08 11:23:03,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:23:03,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +12: [2023-05-08 11:23:03,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:23:03,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +26: [2023-05-08 11:23:03,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +26: [2023-05-08 11:23:03,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:23:03,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:23:03,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +23: [2023-05-08 11:23:03,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:23:03,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +23: [2023-05-08 11:23:03,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:23:03,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +19: [2023-05-08 11:23:03,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:23:03,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:23:03,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:23:03,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:23:03,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:23:03,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:23:03,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:23:03,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:23:03,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:23:03,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:23:03,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +31: [2023-05-08 11:23:03,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:23:03,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:23:03,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:23:03,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:23:03,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +31: [2023-05-08 11:23:03,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +31: [2023-05-08 11:23:03,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +31: [2023-05-08 11:23:03,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 4: [2023-05-08 11:23:03,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:23:03,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 4: [2023-05-08 11:23:03,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +28: [2023-05-08 11:23:03,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +13: [2023-05-08 11:23:03,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:23:03,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:23:03,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:23:03,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:23:03,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:23:03,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +28: [2023-05-08 11:23:03,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +28: [2023-05-08 11:23:03,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:23:03,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +28: [2023-05-08 11:23:03,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +29: [2023-05-08 11:23:03,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:23:03,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 4: [2023-05-08 11:23:03,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:23:03,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:23:03,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:23:03,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:23:03,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:23:03,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 4: [2023-05-08 11:23:03,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +28: [2023-05-08 11:23:03,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +29: [2023-05-08 11:23:03,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 4: [2023-05-08 11:23:03,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +25: [2023-05-08 11:23:03,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:23:03,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +26: [2023-05-08 11:23:03,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +13: [2023-05-08 11:23:03,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +13: [2023-05-08 11:23:03,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:23:03,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +25: [2023-05-08 11:23:03,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:23:03,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +26: [2023-05-08 11:23:03,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +26: [2023-05-08 11:23:03,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 6: [2023-05-08 11:23:03,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +11: [2023-05-08 11:23:03,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:23:03,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +11: [2023-05-08 11:23:03,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +11: [2023-05-08 11:23:03,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +11: [2023-05-08 11:23:03,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:23:03,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:23:03,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +11: [2023-05-08 11:23:03,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +10: [2023-05-08 11:23:03,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +10: [2023-05-08 11:23:03,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:23:03,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +10: [2023-05-08 11:23:03,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:23:03,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:23:03,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:23:03,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +11: [2023-05-08 11:23:03,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +11: [2023-05-08 11:23:03,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +11: [2023-05-08 11:23:03,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +10: [2023-05-08 11:23:03,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:23:03,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +11: [2023-05-08 11:23:03,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +11: [2023-05-08 11:23:03,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:23:03,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +10: [2023-05-08 11:23:03,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:23:03,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +10: [2023-05-08 11:23:03,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +10: [2023-05-08 11:23:03,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... + 4: [2023-05-08 11:23:03,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:23:03,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:23:03,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +17: [2023-05-08 11:23:03,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +17: [2023-05-08 11:23:03,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +17: [2023-05-08 11:23:03,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +17: [2023-05-08 11:23:03,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +17: [2023-05-08 11:23:03,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +17: [2023-05-08 11:23:03,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +13: [2023-05-08 11:23:03,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:23:03,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +17: [2023-05-08 11:23:03,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +31: [2023-05-08 11:23:03,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +10: [2023-05-08 11:23:03,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 4: [2023-05-08 11:23:03,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:23:03,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +28: [2023-05-08 11:23:03,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +10: [2023-05-08 11:23:03,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +17: [2023-05-08 11:23:03,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +17: [2023-05-08 11:23:03,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +17: [2023-05-08 11:23:03,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +17: [2023-05-08 11:23:03,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +17: [2023-05-08 11:23:03,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +17: [2023-05-08 11:23:03,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +17: [2023-05-08 11:23:03,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +17: [2023-05-08 11:23:03,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +10: [2023-05-08 11:23:03,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt... +17: [2023-05-08 11:23:03,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt... +31: [2023-05-08 11:23:03,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +26: [2023-05-08 11:23:03,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +26: [2023-05-08 11:23:03,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +28: [2023-05-08 11:23:03,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:23:03,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:23:03,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:23:03,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:23:03,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:23:03,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +17: [2023-05-08 11:23:03,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:23:03,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +17: [2023-05-08 11:23:03,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 4: [2023-05-08 11:23:03,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +11: [2023-05-08 11:23:03,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +11: [2023-05-08 11:23:03,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:23:03,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +11: [2023-05-08 11:23:03,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +11: [2023-05-08 11:23:03,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:23:03,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +10: [2023-05-08 11:23:03,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:23:03,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:23:03,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:23:03,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:23:03,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:23:03,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:23:03,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:23:03,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:23:03,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:23:03,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:23:03,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +16: [2023-05-08 11:23:03,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 6: [2023-05-08 11:23:03,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +28: [2023-05-08 11:23:03,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +28: [2023-05-08 11:23:03,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:23:03,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +16: [2023-05-08 11:23:03,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +16: [2023-05-08 11:23:03,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +17: [2023-05-08 11:23:03,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +11: [2023-05-08 11:23:03,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:23:03,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +17: [2023-05-08 11:23:03,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:23:03,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +11: [2023-05-08 11:23:03,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +11: [2023-05-08 11:23:03,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +10: [2023-05-08 11:23:03,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:23:03,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +17: [2023-05-08 11:23:03,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +16: [2023-05-08 11:23:03,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:23:03,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:23:03,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +17: [2023-05-08 11:23:03,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +16: [2023-05-08 11:23:03,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:23:03,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:23:03,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:23:03,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:23:03,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:23:03,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:23:03,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:23:03,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:23:03,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:23:03,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:23:03,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:23:03,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +17: [2023-05-08 11:23:03,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:23:03,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +10: [2023-05-08 11:23:03,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:23:03,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:23:03,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +22: [2023-05-08 11:23:03,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:23:03,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:23:03,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:23:03,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:23:03,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:23:03,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:23:03,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:23:03,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:23:03,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:23:03,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +22: [2023-05-08 11:23:03,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:23:03,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +22: [2023-05-08 11:23:03,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +22: [2023-05-08 11:23:03,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:23:03,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +22: [2023-05-08 11:23:03,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:23:03,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +24: [2023-05-08 11:23:03,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +24: [2023-05-08 11:23:03,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +24: [2023-05-08 11:23:03,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 3: [2023-05-08 11:23:03,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:23:03,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:23:03,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:23:03,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:23:03,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:23:03,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:23:03,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +20: [2023-05-08 11:23:03,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:23:03,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:23:03,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:23:03,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:23:03,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:23:03,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:23:03,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:23:03,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:23:03,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +22: [2023-05-08 11:23:03,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:23:03,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 9: [2023-05-08 11:23:03,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 9: [2023-05-08 11:23:03,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 9: [2023-05-08 11:23:03,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +22: [2023-05-08 11:23:03,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 7: [2023-05-08 11:23:03,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:23:03,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:23:03,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:23:03,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:23:03,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +22: [2023-05-08 11:23:03,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:23:03,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +30: [2023-05-08 11:23:03,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +22: [2023-05-08 11:23:03,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:23:03,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:23:03,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:23:03,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +20: [2023-05-08 11:23:03,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:23:03,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +30: [2023-05-08 11:23:04,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +30: [2023-05-08 11:23:04,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +20: [2023-05-08 11:23:04,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:23:04,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:23:04,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +20: [2023-05-08 11:23:04,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:23:04,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:23:04,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:23:04,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:23:04,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:23:04,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +18: [2023-05-08 11:23:04,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:23:04,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:23:04,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:23:04,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:23:04,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +30: [2023-05-08 11:23:04,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:23:04,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +30: [2023-05-08 11:23:04,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +20: [2023-05-08 11:23:04,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:23:04,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:23:04,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +15: [2023-05-08 11:23:04,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:23:04,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +15: [2023-05-08 11:23:04,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:23:04,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:23:04,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:23:04,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +15: [2023-05-08 11:23:04,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +15: [2023-05-08 11:23:04,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +20: [2023-05-08 11:23:04,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:23:04,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:23:04,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:23:04,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:23:04,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:23:04,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 5: [2023-05-08 11:23:04,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 5: [2023-05-08 11:23:04,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 5: [2023-05-08 11:23:04,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +15: [2023-05-08 11:23:04,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:23:04,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:23:04,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:23:04,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:23:04,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +15: [2023-05-08 11:23:04,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:23:04,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:23:04,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:23:04,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:23:04,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:23:04,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +15: [2023-05-08 11:23:04,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +15: [2023-05-08 11:23:04,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +29: [2023-05-08 11:23:04,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +29: [2023-05-08 11:23:04,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +29: [2023-05-08 11:23:04,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +29: [2023-05-08 11:23:04,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +11: [2023-05-08 11:23:04,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:23:04,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:23:04,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:23:04,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +11: [2023-05-08 11:23:04,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +11: [2023-05-08 11:23:04,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +11: [2023-05-08 11:23:04,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:23:04,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:23:04,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +14: [2023-05-08 11:23:04,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +14: [2023-05-08 11:23:04,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +14: [2023-05-08 11:23:04,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +29: [2023-05-08 11:23:04,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +29: [2023-05-08 11:23:04,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:23:04,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:23:04,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:23:04,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:23:04,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:23:04,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:23:04,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:23:04,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:23:04,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:23:04,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:23:04,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 8: [2023-05-08 11:23:04,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 8: [2023-05-08 11:23:04,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 8: [2023-05-08 11:23:04,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +14: [2023-05-08 11:23:04,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:23:04,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:23:04,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +25: [2023-05-08 11:23:04,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +25: [2023-05-08 11:23:04,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +16: [2023-05-08 11:23:04,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:23:04,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:23:04,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:23:04,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:23:04,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +22: [2023-05-08 11:23:04,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +22: [2023-05-08 11:23:04,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:23:04,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:23:04,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:23:04,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +16: [2023-05-08 11:23:04,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:23:04,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +22: [2023-05-08 11:23:04,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +22: [2023-05-08 11:23:04,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +22: [2023-05-08 11:23:04,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +22: [2023-05-08 11:23:04,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +16: [2023-05-08 11:23:04,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:23:04,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 2: [2023-05-08 11:23:04,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 2: [2023-05-08 11:23:04,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +11: [2023-05-08 11:23:04,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:23:04,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:23:04,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +22: [2023-05-08 11:23:04,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 2: [2023-05-08 11:23:04,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +14: [2023-05-08 11:23:04,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:23:04,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:23:04,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:23:04,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:23:04,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:23:04,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:23:04,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:23:04,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +24: [2023-05-08 11:23:04,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:23:04,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:23:04,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:23:04,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +11: [2023-05-08 11:23:04,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:23:04,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +11: [2023-05-08 11:23:04,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +21: [2023-05-08 11:23:04,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:23:04,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 8: [2023-05-08 11:23:04,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:23:04,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +21: [2023-05-08 11:23:04,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +21: [2023-05-08 11:23:04,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +21: [2023-05-08 11:23:04,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +21: [2023-05-08 11:23:04,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +21: [2023-05-08 11:23:04,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +21: [2023-05-08 11:23:04,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +21: [2023-05-08 11:23:04,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 7: [2023-05-08 11:23:04,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +25: [2023-05-08 11:23:04,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:23:04,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +11: [2023-05-08 11:23:04,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +26: [2023-05-08 11:23:04,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 2: [2023-05-08 11:23:04,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:23:04,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:23:04,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +26: [2023-05-08 11:23:04,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +26: [2023-05-08 11:23:04,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 2: [2023-05-08 11:23:04,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +26: [2023-05-08 11:23:04,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +27: [2023-05-08 11:23:04,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 2: [2023-05-08 11:23:04,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:23:04,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +16: [2023-05-08 11:23:04,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:23:04,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +22: [2023-05-08 11:23:04,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:23:04,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:23:04,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:23:04,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:23:04,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:23:04,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:23:04,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +27: [2023-05-08 11:23:04,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +26: [2023-05-08 11:23:04,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:23:04,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +26: [2023-05-08 11:23:04,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +26: [2023-05-08 11:23:04,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:23:04,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:23:04,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:23:04,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:23:04,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +21: [2023-05-08 11:23:04,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +22: [2023-05-08 11:23:04,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +22: [2023-05-08 11:23:04,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:23:04,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +21: [2023-05-08 11:23:04,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +11: [2023-05-08 11:23:04,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +21: [2023-05-08 11:23:04,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:23:04,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +16: [2023-05-08 11:23:04,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +15: [2023-05-08 11:23:04,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +15: [2023-05-08 11:23:04,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +15: [2023-05-08 11:23:04,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +15: [2023-05-08 11:23:04,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +24: [2023-05-08 11:23:04,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:23:04,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +21: [2023-05-08 11:23:04,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:23:04,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:23:04,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +16: [2023-05-08 11:23:04,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +21: [2023-05-08 11:23:04,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 7: [2023-05-08 11:23:04,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 7: [2023-05-08 11:23:04,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +15: [2023-05-08 11:23:04,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +15: [2023-05-08 11:23:04,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:23:04,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:23:04,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 9: [2023-05-08 11:23:04,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 9: [2023-05-08 11:23:04,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 9: [2023-05-08 11:23:04,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +16: [2023-05-08 11:23:04,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:23:04,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +23: [2023-05-08 11:23:04,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:23:04,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +23: [2023-05-08 11:23:04,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:23:04,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:23:04,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +23: [2023-05-08 11:23:04,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +23: [2023-05-08 11:23:04,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:23:04,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:23:04,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:23:04,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 7: [2023-05-08 11:23:04,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +15: [2023-05-08 11:23:04,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:23:04,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 7: [2023-05-08 11:23:04,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +16: [2023-05-08 11:23:04,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 2: [2023-05-08 11:23:04,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 2: [2023-05-08 11:23:04,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 2: [2023-05-08 11:23:04,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:23:04,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:23:04,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:23:04,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:23:04,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:23:04,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +27: [2023-05-08 11:23:04,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:23:04,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:23:04,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:23:04,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +27: [2023-05-08 11:23:04,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +27: [2023-05-08 11:23:04,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +27: [2023-05-08 11:23:04,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:23:04,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +27: [2023-05-08 11:23:04,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:23:04,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +30: [2023-05-08 11:23:04,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 5: [2023-05-08 11:23:04,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 5: [2023-05-08 11:23:04,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 5: [2023-05-08 11:23:04,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 5: [2023-05-08 11:23:04,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:23:04,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +30: [2023-05-08 11:23:04,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +30: [2023-05-08 11:23:04,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +30: [2023-05-08 11:23:04,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +30: [2023-05-08 11:23:04,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +30: [2023-05-08 11:23:04,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +30: [2023-05-08 11:23:04,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +30: [2023-05-08 11:23:04,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:23:04,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +27: [2023-05-08 11:23:04,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +30: [2023-05-08 11:23:04,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:23:04,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +30: [2023-05-08 11:23:04,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +13: [2023-05-08 11:23:04,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +30: [2023-05-08 11:23:04,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 8: [2023-05-08 11:23:04,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +13: [2023-05-08 11:23:04,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +13: [2023-05-08 11:23:04,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +13: [2023-05-08 11:23:04,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:23:04,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +13: [2023-05-08 11:23:04,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +13: [2023-05-08 11:23:04,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 3: [2023-05-08 11:23:04,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 3: [2023-05-08 11:23:04,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +30: [2023-05-08 11:23:04,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +30: [2023-05-08 11:23:04,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 0: [2023-05-08 11:23:04,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +12: [2023-05-08 11:23:04,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +12: [2023-05-08 11:23:04,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +12: [2023-05-08 11:23:04,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 3: [2023-05-08 11:23:04,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 3: [2023-05-08 11:23:04,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +12: [2023-05-08 11:23:04,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +19: [2023-05-08 11:23:04,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:23:04,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:23:04,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:23:04,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:23:04,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:23:04,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +19: [2023-05-08 11:23:04,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +19: [2023-05-08 11:23:04,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +30: [2023-05-08 11:23:04,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:23:04,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:23:04,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:23:04,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:23:04,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:23:04,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +19: [2023-05-08 11:23:04,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:23:04,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:23:04,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +23: [2023-05-08 11:23:04,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:23:04,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +28: [2023-05-08 11:23:04,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:23:04,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:23:04,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +28: [2023-05-08 11:23:04,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:23:04,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +28: [2023-05-08 11:23:04,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +15: [2023-05-08 11:23:04,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 0: [2023-05-08 11:23:04,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:23:04,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +29: [2023-05-08 11:23:04,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +29: [2023-05-08 11:23:04,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +29: [2023-05-08 11:23:04,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +29: [2023-05-08 11:23:04,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +29: [2023-05-08 11:23:04,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +29: [2023-05-08 11:23:04,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +29: [2023-05-08 11:23:04,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +29: [2023-05-08 11:23:04,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +29: [2023-05-08 11:23:04,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +29: [2023-05-08 11:23:04,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +29: [2023-05-08 11:23:04,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +29: [2023-05-08 11:23:04,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 0: [2023-05-08 11:23:04,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +29: [2023-05-08 11:23:04,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 0: [2023-05-08 11:23:04,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 2: [2023-05-08 11:23:04,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +29: [2023-05-08 11:23:04,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +29: [2023-05-08 11:23:04,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +29: [2023-05-08 11:23:04,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +13: [2023-05-08 11:23:04,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +13: [2023-05-08 11:23:04,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +13: [2023-05-08 11:23:04,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +31: [2023-05-08 11:23:04,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +31: [2023-05-08 11:23:04,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +31: [2023-05-08 11:23:04,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +31: [2023-05-08 11:23:04,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +13: [2023-05-08 11:23:04,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 0: [2023-05-08 11:23:04,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:23:04,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:23:04,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:23:04,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +23: [2023-05-08 11:23:04,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +23: [2023-05-08 11:23:04,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +12: [2023-05-08 11:23:04,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +28: [2023-05-08 11:23:04,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +27: [2023-05-08 11:23:04,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:23:04,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:23:04,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:23:04,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +28: [2023-05-08 11:23:04,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +13: [2023-05-08 11:23:04,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:23:04,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:23:04,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:23:04,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:23:04,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:23:04,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:23:04,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +30: [2023-05-08 11:23:04,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:23:04,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +30: [2023-05-08 11:23:04,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:23:04,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +28: [2023-05-08 11:23:04,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:23:04,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:23:04,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +13: [2023-05-08 11:23:04,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:23:04,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +29: [2023-05-08 11:23:04,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:23:04,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:23:04,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:23:04,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:23:04,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +13: [2023-05-08 11:23:04,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:23:04,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +29: [2023-05-08 11:23:04,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:23:04,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:23:04,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:23:04,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +26: [2023-05-08 11:23:04,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:23:04,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:23:04,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:23:04,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +26: [2023-05-08 11:23:04,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +26: [2023-05-08 11:23:04,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +28: [2023-05-08 11:23:04,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:23:04,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:23:04,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:23:04,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:23:04,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:23:04,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:23:04,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:23:04,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:23:04,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:23:04,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:23:04,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:23:04,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:23:04,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:23:04,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:23:04,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:23:04,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 0: [2023-05-08 11:23:04,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +26: [2023-05-08 11:23:04,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 6: [2023-05-08 11:23:04,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +26: [2023-05-08 11:23:04,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 8: [2023-05-08 11:23:04,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:23:04,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +19: [2023-05-08 11:23:04,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:23:04,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:23:04,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:23:04,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 3: [2023-05-08 11:23:04,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +29: [2023-05-08 11:23:04,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:23:04,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +19: [2023-05-08 11:23:04,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +29: [2023-05-08 11:23:04,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:23:04,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:23:04,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:23:04,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:23:04,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:23:04,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:23:04,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +29: [2023-05-08 11:23:04,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:23:04,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:23:04,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +13: [2023-05-08 11:23:04,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +13: [2023-05-08 11:23:04,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +13: [2023-05-08 11:23:04,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +29: [2023-05-08 11:23:04,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +13: [2023-05-08 11:23:04,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +30: [2023-05-08 11:23:04,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 0: [2023-05-08 11:23:04,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +13: [2023-05-08 11:23:04,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +13: [2023-05-08 11:23:04,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +20: [2023-05-08 11:23:04,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +20: [2023-05-08 11:23:04,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +20: [2023-05-08 11:23:04,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +20: [2023-05-08 11:23:04,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +13: [2023-05-08 11:23:04,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +13: [2023-05-08 11:23:04,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +29: [2023-05-08 11:23:04,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +29: [2023-05-08 11:23:04,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:23:04,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:23:04,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +26: [2023-05-08 11:23:04,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:23:04,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:23:04,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:23:04,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:23:04,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:23:04,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:23:04,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:23:04,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +26: [2023-05-08 11:23:04,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:23:04,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:23:04,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +10: [2023-05-08 11:23:04,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +10: [2023-05-08 11:23:04,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +10: [2023-05-08 11:23:04,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +10: [2023-05-08 11:23:04,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +28: [2023-05-08 11:23:04,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:23:04,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +17: [2023-05-08 11:23:04,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +23: [2023-05-08 11:23:04,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +23: [2023-05-08 11:23:04,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +28: [2023-05-08 11:23:04,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:23:04,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +17: [2023-05-08 11:23:04,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +19: [2023-05-08 11:23:04,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +19: [2023-05-08 11:23:04,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +23: [2023-05-08 11:23:04,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +19: [2023-05-08 11:23:04,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +21: [2023-05-08 11:23:04,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +21: [2023-05-08 11:23:04,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +21: [2023-05-08 11:23:04,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 1: [2023-05-08 11:23:04,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +23: [2023-05-08 11:23:04,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 4: [2023-05-08 11:23:04,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +21: [2023-05-08 11:23:04,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 4: [2023-05-08 11:23:04,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 1: [2023-05-08 11:23:04,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 1: [2023-05-08 11:23:04,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +20: [2023-05-08 11:23:04,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 4: [2023-05-08 11:23:04,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 1: [2023-05-08 11:23:04,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +17: [2023-05-08 11:23:04,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +20: [2023-05-08 11:23:04,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +19: [2023-05-08 11:23:04,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. +26: [2023-05-08 11:23:04,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:23:04,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt. + 6: [2023-05-08 11:23:04,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +14: [2023-05-08 11:23:04,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +14: [2023-05-08 11:23:04,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:23:04,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +14: [2023-05-08 11:23:04,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +14: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +14: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +14: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +14: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +14: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 6: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 6: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 6: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +14: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +12: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +12: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +12: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +25: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +14: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +14: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +14: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +12: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +25: [2023-05-08 11:23:04,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +25: [2023-05-08 11:23:04,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:23:04,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:23:04,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +26: [2023-05-08 11:23:04,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:23:04,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +10: [2023-05-08 11:23:04,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:23:04,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 6: [2023-05-08 11:23:04,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:23:04,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +20: [2023-05-08 11:23:04,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:23:04,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:23:04,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +20: [2023-05-08 11:23:04,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +10: [2023-05-08 11:23:04,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +23: [2023-05-08 11:23:04,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:23:04,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +21: [2023-05-08 11:23:04,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +19: [2023-05-08 11:23:04,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +23: [2023-05-08 11:23:04,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +19: [2023-05-08 11:23:04,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +19: [2023-05-08 11:23:04,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +21: [2023-05-08 11:23:04,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +21: [2023-05-08 11:23:04,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +23: [2023-05-08 11:23:04,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +17: [2023-05-08 11:23:04,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +23: [2023-05-08 11:23:04,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:23:04,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 1: [2023-05-08 11:23:04,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:23:04,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +22: [2023-05-08 11:23:04,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +22: [2023-05-08 11:23:04,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 1: [2023-05-08 11:23:04,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +17: [2023-05-08 11:23:04,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:23:04,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +17: [2023-05-08 11:23:04,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +26: [2023-05-08 11:23:04,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:23:04,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +24: [2023-05-08 11:23:04,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +17: [2023-05-08 11:23:04,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:23:04,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +17: [2023-05-08 11:23:04,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:23:04,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:23:04,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +17: [2023-05-08 11:23:04,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:23:04,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:23:04,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:23:04,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:23:04,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:23:04,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +31: [2023-05-08 11:23:04,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +24: [2023-05-08 11:23:04,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +10: [2023-05-08 11:23:04,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:23:04,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:23:04,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:23:04,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:23:04,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:23:04,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:23:04,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:23:04,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +26: [2023-05-08 11:23:04,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +31: [2023-05-08 11:23:04,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:23:04,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:23:04,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:23:04,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:23:04,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +31: [2023-05-08 11:23:04,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:23:04,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:23:04,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:23:04,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +31: [2023-05-08 11:23:04,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +14: [2023-05-08 11:23:04,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:23:04,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:23:04,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:23:04,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +14: [2023-05-08 11:23:04,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:23:04,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:23:04,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:23:04,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:23:04,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +23: [2023-05-08 11:23:04,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:23:04,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:23:04,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:23:04,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 4: [2023-05-08 11:23:04,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +12: [2023-05-08 11:23:04,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:23:04,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:23:04,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +22: [2023-05-08 11:23:04,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +23: [2023-05-08 11:23:04,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:23:04,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +24: [2023-05-08 11:23:04,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +22: [2023-05-08 11:23:04,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:23:04,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:23:04,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:23:04,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:23:04,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:23:04,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:23:04,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 1: [2023-05-08 11:23:04,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +19: [2023-05-08 11:23:04,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:23:04,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:23:04,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:23:04,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +21: [2023-05-08 11:23:04,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:23:04,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:23:04,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:23:04,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:23:04,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +23: [2023-05-08 11:23:04,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +21: [2023-05-08 11:23:04,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +25: [2023-05-08 11:23:04,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:23:04,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +12: [2023-05-08 11:23:04,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:23:04,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:23:04,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +19: [2023-05-08 11:23:04,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +19: [2023-05-08 11:23:04,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +19: [2023-05-08 11:23:04,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 6: [2023-05-08 11:23:04,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +10: [2023-05-08 11:23:04,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:23:04,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:23:04,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +10: [2023-05-08 11:23:04,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 4: [2023-05-08 11:23:04,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 4: [2023-05-08 11:23:04,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:23:04,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:23:04,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +31: [2023-05-08 11:23:04,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +10: [2023-05-08 11:23:04,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:23:04,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:23:04,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:23:04,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:23:04,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +21: [2023-05-08 11:23:04,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +21: [2023-05-08 11:23:04,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +10: [2023-05-08 11:23:04,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 7: [2023-05-08 11:23:04,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:23:04,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +21: [2023-05-08 11:23:04,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 1: [2023-05-08 11:23:04,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +14: [2023-05-08 11:23:04,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +25: [2023-05-08 11:23:04,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +25: [2023-05-08 11:23:04,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:23:04,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:23:04,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:23:04,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:23:04,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:23:04,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:23:04,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:23:04,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:23:04,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:23:04,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +15: [2023-05-08 11:23:04,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +17: [2023-05-08 11:23:04,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +10: [2023-05-08 11:23:04,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:23:04,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:23:04,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:23:04,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:23:04,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:23:04,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +17: [2023-05-08 11:23:04,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 6: [2023-05-08 11:23:04,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:23:04,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:23:04,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:23:04,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +18: [2023-05-08 11:23:04,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +18: [2023-05-08 11:23:04,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:23:04,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:23:04,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +18: [2023-05-08 11:23:04,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +18: [2023-05-08 11:23:04,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +18: [2023-05-08 11:23:04,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +18: [2023-05-08 11:23:04,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +17: [2023-05-08 11:23:04,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:23:04,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +18: [2023-05-08 11:23:04,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:23:04,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:23:04,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:23:04,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:23:04,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:23:04,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +16: [2023-05-08 11:23:04,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:23:04,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +17: [2023-05-08 11:23:04,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +22: [2023-05-08 11:23:04,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:23:04,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:23:04,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +22: [2023-05-08 11:23:04,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +31: [2023-05-08 11:23:04,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:23:04,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +10: [2023-05-08 11:23:04,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:23:04,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... + 7: [2023-05-08 11:23:04,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +16: [2023-05-08 11:23:04,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:23:04,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +16: [2023-05-08 11:23:04,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +10: [2023-05-08 11:23:04,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +31: [2023-05-08 11:23:04,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +24: [2023-05-08 11:23:04,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:23:04,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 5: [2023-05-08 11:23:04,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +22: [2023-05-08 11:23:04,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +17: [2023-05-08 11:23:04,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +22: [2023-05-08 11:23:04,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 5: [2023-05-08 11:23:04,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +10: [2023-05-08 11:23:04,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +31: [2023-05-08 11:23:04,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +10: [2023-05-08 11:23:04,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +17: [2023-05-08 11:23:04,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:23:04,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +17: [2023-05-08 11:23:04,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +17: [2023-05-08 11:23:04,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt... +10: [2023-05-08 11:23:04,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +15: [2023-05-08 11:23:04,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:23:04,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +20: [2023-05-08 11:23:04,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:23:04,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +20: [2023-05-08 11:23:04,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:23:04,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +20: [2023-05-08 11:23:04,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +20: [2023-05-08 11:23:04,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:23:04,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:23:04,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:23:04,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:23:04,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:23:04,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:23:04,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:23:04,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +22: [2023-05-08 11:23:04,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:23:04,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +22: [2023-05-08 11:23:04,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +22: [2023-05-08 11:23:04,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +31: [2023-05-08 11:23:04,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:23:04,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:23:04,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +24: [2023-05-08 11:23:04,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +16: [2023-05-08 11:23:04,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:23:04,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:23:04,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:23:04,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:23:04,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:23:04,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +18: [2023-05-08 11:23:04,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +15: [2023-05-08 11:23:04,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:23:04,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +18: [2023-05-08 11:23:04,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +18: [2023-05-08 11:23:04,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +31: [2023-05-08 11:23:04,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +16: [2023-05-08 11:23:04,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:23:04,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:23:04,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt. +11: [2023-05-08 11:23:04,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +11: [2023-05-08 11:23:04,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +11: [2023-05-08 11:23:04,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +11: [2023-05-08 11:23:04,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 5: [2023-05-08 11:23:04,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +15: [2023-05-08 11:23:04,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:23:04,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:23:04,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:23:04,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +16: [2023-05-08 11:23:04,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:23:04,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:23:04,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:23:04,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:23:04,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:23:04,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +11: [2023-05-08 11:23:04,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +16: [2023-05-08 11:23:04,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:23:04,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +11: [2023-05-08 11:23:04,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:23:04,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +16: [2023-05-08 11:23:04,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:23:04,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:23:04,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +11: [2023-05-08 11:23:04,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:23:04,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +11: [2023-05-08 11:23:04,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:23:04,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +11: [2023-05-08 11:23:04,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:23:04,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +11: [2023-05-08 11:23:04,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +11: [2023-05-08 11:23:04,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +11: [2023-05-08 11:23:04,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +11: [2023-05-08 11:23:04,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:23:04,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:23:04,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:23:04,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +11: [2023-05-08 11:23:04,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:23:04,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +21: [2023-05-08 11:23:04,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:23:04,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +21: [2023-05-08 11:23:04,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:23:04,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +21: [2023-05-08 11:23:04,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +21: [2023-05-08 11:23:04,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:23:04,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:23:04,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +11: [2023-05-08 11:23:04,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +11: [2023-05-08 11:23:04,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:23:04,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +21: [2023-05-08 11:23:04,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:23:04,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +22: [2023-05-08 11:23:04,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +22: [2023-05-08 11:23:04,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:23:04,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:23:04,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:23:04,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +22: [2023-05-08 11:23:04,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:23:04,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:23:04,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:23:04,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:23:04,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:23:04,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:23:04,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +24: [2023-05-08 11:23:04,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +24: [2023-05-08 11:23:04,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +24: [2023-05-08 11:23:04,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +24: [2023-05-08 11:23:04,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:23:04,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:23:04,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:23:04,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +24: [2023-05-08 11:23:04,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +21: [2023-05-08 11:23:04,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:23:04,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:23:04,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:23:04,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:23:04,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +24: [2023-05-08 11:23:04,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +24: [2023-05-08 11:23:04,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:23:04,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +24: [2023-05-08 11:23:04,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +24: [2023-05-08 11:23:04,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +23: [2023-05-08 11:23:04,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +23: [2023-05-08 11:23:04,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +23: [2023-05-08 11:23:04,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +23: [2023-05-08 11:23:04,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:23:04,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +23: [2023-05-08 11:23:04,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +23: [2023-05-08 11:23:04,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:23:04,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +18: [2023-05-08 11:23:04,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +18: [2023-05-08 11:23:04,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +22: [2023-05-08 11:23:04,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +23: [2023-05-08 11:23:04,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +23: [2023-05-08 11:23:04,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 5: [2023-05-08 11:23:04,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 5: [2023-05-08 11:23:04,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 5: [2023-05-08 11:23:04,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +22: [2023-05-08 11:23:04,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:23:04,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:23:04,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +13: [2023-05-08 11:23:04,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:23:04,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +22: [2023-05-08 11:23:04,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +24: [2023-05-08 11:23:04,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:23:04,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:23:04,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +27: [2023-05-08 11:23:04,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 2: [2023-05-08 11:23:04,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 2: [2023-05-08 11:23:04,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +27: [2023-05-08 11:23:04,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +13: [2023-05-08 11:23:04,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +13: [2023-05-08 11:23:04,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +13: [2023-05-08 11:23:04,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:23:04,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:23:04,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 2: [2023-05-08 11:23:04,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +27: [2023-05-08 11:23:04,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +24: [2023-05-08 11:23:04,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +13: [2023-05-08 11:23:04,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +29: [2023-05-08 11:23:04,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:23:04,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:23:04,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +29: [2023-05-08 11:23:04,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +24: [2023-05-08 11:23:04,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +18: [2023-05-08 11:23:04,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:23:04,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +16: [2023-05-08 11:23:04,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +16: [2023-05-08 11:23:04,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +19: [2023-05-08 11:23:04,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:23:04,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:23:04,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:23:04,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:23:04,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:23:04,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:23:04,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:23:04,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:23:04,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:23:04,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:23:04,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:23:04,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:23:04,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:23:04,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:23:04,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +22: [2023-05-08 11:23:04,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +18: [2023-05-08 11:23:04,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +16: [2023-05-08 11:23:04,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +18: [2023-05-08 11:23:04,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +24: [2023-05-08 11:23:04,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:23:04,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:23:04,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:23:04,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +29: [2023-05-08 11:23:04,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:23:04,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +22: [2023-05-08 11:23:04,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:23:04,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:23:04,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +23: [2023-05-08 11:23:04,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:23:04,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:23:04,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:23:04,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +23: [2023-05-08 11:23:04,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:23:04,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:23:04,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +26: [2023-05-08 11:23:04,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +26: [2023-05-08 11:23:04,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:23:04,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +26: [2023-05-08 11:23:04,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:23:04,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:23:04,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:23:04,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:23:04,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:23:04,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +13: [2023-05-08 11:23:04,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:23:04,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:23:04,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:23:04,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +16: [2023-05-08 11:23:04,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +23: [2023-05-08 11:23:04,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:23:04,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:23:04,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:23:04,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +29: [2023-05-08 11:23:04,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +13: [2023-05-08 11:23:04,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:23:04,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:23:04,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 5: [2023-05-08 11:23:04,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +23: [2023-05-08 11:23:04,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +18: [2023-05-08 11:23:04,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:23:04,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +26: [2023-05-08 11:23:04,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:23:04,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 5: [2023-05-08 11:23:04,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +13: [2023-05-08 11:23:04,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +29: [2023-05-08 11:23:04,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:23:04,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:23:04,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +23: [2023-05-08 11:23:04,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:23:04,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:23:04,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:23:04,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:23:04,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:23:04,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:23:04,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:23:04,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +23: [2023-05-08 11:23:04,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:23:04,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +29: [2023-05-08 11:23:04,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:23:04,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +13: [2023-05-08 11:23:04,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:23:04,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 5: [2023-05-08 11:23:04,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:23:04,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +23: [2023-05-08 11:23:04,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +16: [2023-05-08 11:23:04,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:23:04,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 7: [2023-05-08 11:23:04,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 7: [2023-05-08 11:23:04,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 7: [2023-05-08 11:23:04,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +16: [2023-05-08 11:23:04,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +19: [2023-05-08 11:23:04,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:23:04,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:23:04,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:23:04,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +26: [2023-05-08 11:23:04,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:23:04,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:23:04,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +23: [2023-05-08 11:23:04,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:23:04,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:23:04,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +16: [2023-05-08 11:23:04,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:23:04,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +30: [2023-05-08 11:23:04,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +15: [2023-05-08 11:23:04,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +15: [2023-05-08 11:23:04,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +15: [2023-05-08 11:23:04,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +15: [2023-05-08 11:23:04,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +30: [2023-05-08 11:23:04,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:23:04,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:23:04,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:23:04,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 7: [2023-05-08 11:23:04,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:23:04,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +16: [2023-05-08 11:23:04,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:23:04,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:23:04,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:23:04,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:23:04,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:23:04,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:23:04,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +16: [2023-05-08 11:23:04,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 7: [2023-05-08 11:23:04,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +15: [2023-05-08 11:23:04,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:23:04,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +15: [2023-05-08 11:23:04,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +15: [2023-05-08 11:23:04,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:23:04,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:23:04,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:23:04,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +29: [2023-05-08 11:23:04,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +30: [2023-05-08 11:23:04,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:23:04,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:23:04,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:23:04,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 4: [2023-05-08 11:23:04,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 4: [2023-05-08 11:23:04,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 4: [2023-05-08 11:23:04,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +26: [2023-05-08 11:23:04,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:23:04,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 7: [2023-05-08 11:23:04,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +15: [2023-05-08 11:23:04,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:23:04,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:23:04,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +15: [2023-05-08 11:23:04,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 7: [2023-05-08 11:23:04,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +15: [2023-05-08 11:23:04,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:23:04,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 7: [2023-05-08 11:23:04,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 4: [2023-05-08 11:23:04,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 3: [2023-05-08 11:23:04,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 3: [2023-05-08 11:23:04,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 3: [2023-05-08 11:23:04,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +15: [2023-05-08 11:23:04,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:23:04,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 4: [2023-05-08 11:23:04,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:23:04,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +27: [2023-05-08 11:23:04,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:23:04,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:23:04,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:23:04,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:23:04,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:23:04,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:23:04,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:23:04,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +27: [2023-05-08 11:23:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:23:04,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 7: [2023-05-08 11:23:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:23:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:23:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:23:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:23:04,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 7: [2023-05-08 11:23:04,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +24: [2023-05-08 11:23:04,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +24: [2023-05-08 11:23:04,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +24: [2023-05-08 11:23:04,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +24: [2023-05-08 11:23:04,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 3: [2023-05-08 11:23:04,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +27: [2023-05-08 11:23:04,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:23:04,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +27: [2023-05-08 11:23:04,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +16: [2023-05-08 11:23:04,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +16: [2023-05-08 11:23:04,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:23:04,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +16: [2023-05-08 11:23:04,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +16: [2023-05-08 11:23:04,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:23:04,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +22: [2023-05-08 11:23:04,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +16: [2023-05-08 11:23:04,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +16: [2023-05-08 11:23:04,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +22: [2023-05-08 11:23:04,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 4: [2023-05-08 11:23:04,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 5: [2023-05-08 11:23:04,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:23:04,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +22: [2023-05-08 11:23:04,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +24: [2023-05-08 11:23:04,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:23:04,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:23:04,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +21: [2023-05-08 11:23:04,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:23:04,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:23:04,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:23:04,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +21: [2023-05-08 11:23:04,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +21: [2023-05-08 11:23:04,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +21: [2023-05-08 11:23:04,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:23:04,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:23:04,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +28: [2023-05-08 11:23:04,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:23:04,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:23:04,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:23:04,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:23:04,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:23:04,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:23:04,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:23:04,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:23:04,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:23:04,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:23:04,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +16: [2023-05-08 11:23:04,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:23:04,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:23:04,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +20: [2023-05-08 11:23:04,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +20: [2023-05-08 11:23:04,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +16: [2023-05-08 11:23:04,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +22: [2023-05-08 11:23:04,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:23:04,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +20: [2023-05-08 11:23:04,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +14: [2023-05-08 11:23:04,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:23:04,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +14: [2023-05-08 11:23:04,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +14: [2023-05-08 11:23:04,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:23:04,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:23:04,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:23:04,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +14: [2023-05-08 11:23:04,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 3: [2023-05-08 11:23:04,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +10: [2023-05-08 11:23:04,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +10: [2023-05-08 11:23:04,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:23:04,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +10: [2023-05-08 11:23:04,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +10: [2023-05-08 11:23:04,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +10: [2023-05-08 11:23:04,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:23:04,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 8: [2023-05-08 11:23:04,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +24: [2023-05-08 11:23:04,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:23:04,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:23:04,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +10: [2023-05-08 11:23:04,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:23:04,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 2: [2023-05-08 11:23:04,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +12: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 2: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 2: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 9: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +27: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +25: [2023-05-08 11:23:04,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:23:04,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:23:04,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +12: [2023-05-08 11:23:04,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:23:04,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +20: [2023-05-08 11:23:04,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:23:04,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +17: [2023-05-08 11:23:04,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +17: [2023-05-08 11:23:04,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +17: [2023-05-08 11:23:04,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +17: [2023-05-08 11:23:04,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:23:04,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:23:04,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +31: [2023-05-08 11:23:04,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +31: [2023-05-08 11:23:04,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +31: [2023-05-08 11:23:04,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +31: [2023-05-08 11:23:04,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +31: [2023-05-08 11:23:04,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +31: [2023-05-08 11:23:04,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +31: [2023-05-08 11:23:04,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:23:04,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:23:04,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +12: [2023-05-08 11:23:04,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +17: [2023-05-08 11:23:04,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:23:04,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:23:04,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:23:04,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:23:04,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:23:04,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +24: [2023-05-08 11:23:04,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +20: [2023-05-08 11:23:04,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +20: [2023-05-08 11:23:04,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:23:04,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:23:04,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:23:04,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 6: [2023-05-08 11:23:04,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +17: [2023-05-08 11:23:04,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:23:04,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:23:04,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +13: [2023-05-08 11:23:04,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +13: [2023-05-08 11:23:04,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:23:04,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +13: [2023-05-08 11:23:04,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:23:04,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:23:04,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +22: [2023-05-08 11:23:04,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +13: [2023-05-08 11:23:04,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +13: [2023-05-08 11:23:04,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +13: [2023-05-08 11:23:04,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:23:04,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:23:04,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +22: [2023-05-08 11:23:04,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +22: [2023-05-08 11:23:04,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +22: [2023-05-08 11:23:04,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +22: [2023-05-08 11:23:04,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 5: [2023-05-08 11:23:04,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +22: [2023-05-08 11:23:04,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 3: [2023-05-08 11:23:04,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:23:04,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:23:04,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 9: [2023-05-08 11:23:04,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:23:04,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +14: [2023-05-08 11:23:04,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:23:04,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:23:04,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +21: [2023-05-08 11:23:04,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:23:04,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +24: [2023-05-08 11:23:04,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +10: [2023-05-08 11:23:04,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:23:04,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:23:04,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +31: [2023-05-08 11:23:04,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:23:04,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:23:04,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +17: [2023-05-08 11:23:04,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +21: [2023-05-08 11:23:04,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:23:04,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:23:04,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +10: [2023-05-08 11:23:04,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:23:04,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:23:04,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:23:04,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:23:04,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:23:04,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +28: [2023-05-08 11:23:04,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:23:04,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:23:04,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +13: [2023-05-08 11:23:04,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:23:04,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +13: [2023-05-08 11:23:04,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:23:04,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 7: [2023-05-08 11:23:04,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +25: [2023-05-08 11:23:04,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:23:04,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:23:04,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +28: [2023-05-08 11:23:04,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:23:04,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +31: [2023-05-08 11:23:04,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:23:04,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:23:04,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +28: [2023-05-08 11:23:04,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +21: [2023-05-08 11:23:04,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:23:04,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:23:04,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +10: [2023-05-08 11:23:04,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:23:04,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +20: [2023-05-08 11:23:04,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:23:04,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +11: [2023-05-08 11:23:04,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +13: [2023-05-08 11:23:04,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +13: [2023-05-08 11:23:04,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +11: [2023-05-08 11:23:04,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +31: [2023-05-08 11:23:04,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:23:04,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:23:04,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +29: [2023-05-08 11:23:04,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:23:04,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:23:04,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +14: [2023-05-08 11:23:04,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +29: [2023-05-08 11:23:04,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:23:04,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +28: [2023-05-08 11:23:04,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +10: [2023-05-08 11:23:04,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +16: [2023-05-08 11:23:04,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:23:04,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:23:04,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +13: [2023-05-08 11:23:04,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:23:04,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:23:04,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:23:04,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +17: [2023-05-08 11:23:04,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:23:04,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +17: [2023-05-08 11:23:04,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +17: [2023-05-08 11:23:04,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:23:04,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:23:04,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:23:04,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:23:04,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +13: [2023-05-08 11:23:04,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:23:04,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 5: [2023-05-08 11:23:04,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 5: [2023-05-08 11:23:04,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 6: [2023-05-08 11:23:04,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:23:04,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +10: [2023-05-08 11:23:04,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +10: [2023-05-08 11:23:04,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:23:04,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +31: [2023-05-08 11:23:04,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:23:04,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:23:04,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:23:04,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +14: [2023-05-08 11:23:04,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +17: [2023-05-08 11:23:04,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +31: [2023-05-08 11:23:04,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:23:04,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:23:04,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +17: [2023-05-08 11:23:04,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:23:04,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +29: [2023-05-08 11:23:04,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 9: [2023-05-08 11:23:04,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 9: [2023-05-08 11:23:04,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 1: [2023-05-08 11:23:04,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +10: [2023-05-08 11:23:04,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +10: [2023-05-08 11:23:04,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +29: [2023-05-08 11:23:04,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +18: [2023-05-08 11:23:04,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +18: [2023-05-08 11:23:04,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +18: [2023-05-08 11:23:04,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +18: [2023-05-08 11:23:04,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +23: [2023-05-08 11:23:04,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +23: [2023-05-08 11:23:04,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:23:04,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +23: [2023-05-08 11:23:04,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +23: [2023-05-08 11:23:04,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +23: [2023-05-08 11:23:04,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +23: [2023-05-08 11:23:04,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +23: [2023-05-08 11:23:04,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +23: [2023-05-08 11:23:04,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:23:04,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +25: [2023-05-08 11:23:04,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:23:04,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +26: [2023-05-08 11:23:04,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +11: [2023-05-08 11:23:04,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +11: [2023-05-08 11:23:04,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +19: [2023-05-08 11:23:04,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +19: [2023-05-08 11:23:04,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:23:04,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +19: [2023-05-08 11:23:04,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +19: [2023-05-08 11:23:04,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +19: [2023-05-08 11:23:04,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:23:04,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:23:04,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +11: [2023-05-08 11:23:04,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +29: [2023-05-08 11:23:04,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +19: [2023-05-08 11:23:04,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:23:04,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +29: [2023-05-08 11:23:04,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:23:04,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:23:04,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +16: [2023-05-08 11:23:04,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +11: [2023-05-08 11:23:04,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 1: [2023-05-08 11:23:04,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:23:04,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +18: [2023-05-08 11:23:04,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:23:04,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +29: [2023-05-08 11:23:04,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:23:04,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 9: [2023-05-08 11:23:04,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +16: [2023-05-08 11:23:04,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +26: [2023-05-08 11:23:04,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +26: [2023-05-08 11:23:04,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:23:04,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:23:04,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +17: [2023-05-08 11:23:04,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +17: [2023-05-08 11:23:04,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +17: [2023-05-08 11:23:04,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 5: [2023-05-08 11:23:04,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:23:04,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 5: [2023-05-08 11:23:04,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 5: [2023-05-08 11:23:04,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +19: [2023-05-08 11:23:04,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +19: [2023-05-08 11:23:04,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +12: [2023-05-08 11:23:04,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +12: [2023-05-08 11:23:04,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +30: [2023-05-08 11:23:04,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:23:04,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +30: [2023-05-08 11:23:04,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:23:04,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +30: [2023-05-08 11:23:04,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +19: [2023-05-08 11:23:04,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +30: [2023-05-08 11:23:04,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +30: [2023-05-08 11:23:04,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +30: [2023-05-08 11:23:04,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +30: [2023-05-08 11:23:04,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +30: [2023-05-08 11:23:04,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:23:04,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +29: [2023-05-08 11:23:04,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:23:04,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +20: [2023-05-08 11:23:04,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:23:04,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:23:04,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +20: [2023-05-08 11:23:04,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:23:04,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:23:04,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:23:04,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +20: [2023-05-08 11:23:04,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:23:04,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +30: [2023-05-08 11:23:04,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +30: [2023-05-08 11:23:04,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 8: [2023-05-08 11:23:04,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +11: [2023-05-08 11:23:04,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +30: [2023-05-08 11:23:04,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 8: [2023-05-08 11:23:04,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +31: [2023-05-08 11:23:04,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +31: [2023-05-08 11:23:04,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +31: [2023-05-08 11:23:04,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +30: [2023-05-08 11:23:04,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 8: [2023-05-08 11:23:04,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +31: [2023-05-08 11:23:04,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +23: [2023-05-08 11:23:04,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +23: [2023-05-08 11:23:04,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:23:04,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:23:04,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +23: [2023-05-08 11:23:04,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 0: [2023-05-08 11:23:04,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +13: [2023-05-08 11:23:04,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +23: [2023-05-08 11:23:04,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +13: [2023-05-08 11:23:04,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +13: [2023-05-08 11:23:04,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +28: [2023-05-08 11:23:04,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +16: [2023-05-08 11:23:04,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:23:04,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 0: [2023-05-08 11:23:04,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +10: [2023-05-08 11:23:04,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +10: [2023-05-08 11:23:04,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 0: [2023-05-08 11:23:04,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 1: [2023-05-08 11:23:04,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 1: [2023-05-08 11:23:04,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +21: [2023-05-08 11:23:04,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +21: [2023-05-08 11:23:04,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 0: [2023-05-08 11:23:04,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 1: [2023-05-08 11:23:04,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 1: [2023-05-08 11:23:04,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +23: [2023-05-08 11:23:04,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +23: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +23: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +21: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +14: [2023-05-08 11:23:04,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +28: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +28: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 6: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +23: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +28: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +21: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 6: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +14: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +14: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 3: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 3: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +29: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 6: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +29: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +25: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. + 3: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +25: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +10: [2023-05-08 11:23:04,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +25: [2023-05-08 11:23:04,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +10: [2023-05-08 11:23:04,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +29: [2023-05-08 11:23:04,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +25: [2023-05-08 11:23:04,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt. +16: [2023-05-08 11:23:04,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:23:04,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:23:04,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:23:04,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:23:04,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +18: [2023-05-08 11:23:04,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:23:04,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:23:04,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:23:04,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +18: [2023-05-08 11:23:04,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +18: [2023-05-08 11:23:04,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +18: [2023-05-08 11:23:04,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:23:04,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:23:04,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:23:04,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:23:04,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:23:04,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:23:04,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:23:04,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +12: [2023-05-08 11:23:04,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:23:04,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +17: [2023-05-08 11:23:04,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:23:04,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:23:04,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +11: [2023-05-08 11:23:04,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +17: [2023-05-08 11:23:04,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +19: [2023-05-08 11:23:04,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +19: [2023-05-08 11:23:04,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:23:04,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +29: [2023-05-08 11:23:04,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +31: [2023-05-08 11:23:04,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +31: [2023-05-08 11:23:04,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +31: [2023-05-08 11:23:04,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +23: [2023-05-08 11:23:04,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:23:04,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:23:04,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +17: [2023-05-08 11:23:04,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:23:04,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:23:04,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +23: [2023-05-08 11:23:04,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +26: [2023-05-08 11:23:04,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +13: [2023-05-08 11:23:04,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:23:04,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +13: [2023-05-08 11:23:04,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:23:04,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:23:04,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +23: [2023-05-08 11:23:04,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:23:04,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:23:04,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +28: [2023-05-08 11:23:04,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +23: [2023-05-08 11:23:04,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +31: [2023-05-08 11:23:04,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:23:04,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:23:04,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:23:04,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +29: [2023-05-08 11:23:04,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +23: [2023-05-08 11:23:04,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:23:04,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:23:04,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +21: [2023-05-08 11:23:04,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:23:04,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:23:04,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +26: [2023-05-08 11:23:04,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +26: [2023-05-08 11:23:04,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +26: [2023-05-08 11:23:04,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +29: [2023-05-08 11:23:04,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:23:04,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:23:04,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +16: [2023-05-08 11:23:04,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +23: [2023-05-08 11:23:04,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +29: [2023-05-08 11:23:04,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:23:04,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +29: [2023-05-08 11:23:04,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:23:04,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +16: [2023-05-08 11:23:04,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +13: [2023-05-08 11:23:04,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:23:04,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:23:04,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +26: [2023-05-08 11:23:04,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +28: [2023-05-08 11:23:04,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +10: [2023-05-08 11:23:04,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +25: [2023-05-08 11:23:04,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +25: [2023-05-08 11:23:04,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +25: [2023-05-08 11:23:04,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +25: [2023-05-08 11:23:04,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +10: [2023-05-08 11:23:04,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +10: [2023-05-08 11:23:04,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:23:04,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +30: [2023-05-08 11:23:04,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +30: [2023-05-08 11:23:04,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +23: [2023-05-08 11:23:04,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:23:04,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:23:04,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:23:04,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:23:04,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 0: [2023-05-08 11:23:04,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:23:04,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +20: [2023-05-08 11:23:04,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:23:04,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +18: [2023-05-08 11:23:04,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +30: [2023-05-08 11:23:04,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt... +11: [2023-05-08 11:23:04,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:23:04,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +19: [2023-05-08 11:23:04,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:23:04,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:23:04,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +19: [2023-05-08 11:23:04,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +19: [2023-05-08 11:23:04,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +18: [2023-05-08 11:23:04,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:23:04,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:23:04,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:23:04,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +18: [2023-05-08 11:23:04,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +18: [2023-05-08 11:23:04,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:23:04,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +12: [2023-05-08 11:23:04,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 9: [2023-05-08 11:23:04,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +16: [2023-05-08 11:23:04,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:23:04,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +16: [2023-05-08 11:23:04,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +12: [2023-05-08 11:23:04,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:23:04,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +16: [2023-05-08 11:23:04,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:23:04,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +20: [2023-05-08 11:23:04,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:23:04,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +30: [2023-05-08 11:23:04,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +30: [2023-05-08 11:23:04,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:23:04,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +23: [2023-05-08 11:23:04,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:23:04,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:23:04,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:23:04,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +12: [2023-05-08 11:23:04,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 1: [2023-05-08 11:23:04,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +31: [2023-05-08 11:23:04,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +23: [2023-05-08 11:23:04,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 8: [2023-05-08 11:23:04,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 8: [2023-05-08 11:23:04,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +28: [2023-05-08 11:23:04,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 1: [2023-05-08 11:23:04,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 3: [2023-05-08 11:23:04,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +31: [2023-05-08 11:23:04,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +28: [2023-05-08 11:23:04,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 6: [2023-05-08 11:23:04,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:23:04,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +13: [2023-05-08 11:23:04,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +13: [2023-05-08 11:23:04,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +30: [2023-05-08 11:23:04,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:23:04,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +18: [2023-05-08 11:23:04,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:23:04,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 3: [2023-05-08 11:23:04,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +20: [2023-05-08 11:23:04,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +13: [2023-05-08 11:23:04,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:23:04,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:23:04,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:23:04,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +31: [2023-05-08 11:23:04,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +31: [2023-05-08 11:23:04,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:23:04,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +31: [2023-05-08 11:23:04,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +31: [2023-05-08 11:23:04,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +11: [2023-05-08 11:23:04,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +21: [2023-05-08 11:23:04,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 1: [2023-05-08 11:23:04,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +19: [2023-05-08 11:23:04,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:23:04,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +13: [2023-05-08 11:23:04,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +21: [2023-05-08 11:23:04,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +21: [2023-05-08 11:23:04,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +21: [2023-05-08 11:23:04,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +21: [2023-05-08 11:23:04,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +21: [2023-05-08 11:23:04,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:23:04,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +17: [2023-05-08 11:23:04,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +31: [2023-05-08 11:23:04,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:23:04,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +17: [2023-05-08 11:23:04,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +31: [2023-05-08 11:23:04,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +25: [2023-05-08 11:23:04,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:23:04,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +14: [2023-05-08 11:23:04,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +23: [2023-05-08 11:23:04,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 6: [2023-05-08 11:23:04,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +19: [2023-05-08 11:23:04,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:23:04,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +25: [2023-05-08 11:23:04,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +25: [2023-05-08 11:23:04,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:23:04,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +21: [2023-05-08 11:23:04,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +19: [2023-05-08 11:23:04,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 4: [2023-05-08 11:23:04,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:23:04,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 4: [2023-05-08 11:23:04,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:23:04,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +28: [2023-05-08 11:23:04,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:23:04,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +14: [2023-05-08 11:23:04,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:23:04,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +14: [2023-05-08 11:23:04,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +14: [2023-05-08 11:23:04,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +28: [2023-05-08 11:23:04,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +13: [2023-05-08 11:23:04,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +13: [2023-05-08 11:23:04,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +11: [2023-05-08 11:23:04,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +30: [2023-05-08 11:23:04,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:23:04,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:23:04,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +10: [2023-05-08 11:23:04,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:23:04,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:23:04,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +19: [2023-05-08 11:23:04,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +25: [2023-05-08 11:23:04,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:23:04,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +30: [2023-05-08 11:23:04,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:23:04,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +23: [2023-05-08 11:23:04,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +30: [2023-05-08 11:23:04,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:23:04,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +19: [2023-05-08 11:23:04,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +10: [2023-05-08 11:23:04,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +26: [2023-05-08 11:23:04,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:23:04,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:23:04,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:23:04,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +10: [2023-05-08 11:23:04,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +10: [2023-05-08 11:23:04,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +29: [2023-05-08 11:23:04,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +29: [2023-05-08 11:23:04,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +29: [2023-05-08 11:23:04,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +19: [2023-05-08 11:23:04,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +17: [2023-05-08 11:23:04,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +30: [2023-05-08 11:23:04,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:23:04,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:23:04,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +19: [2023-05-08 11:23:04,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +19: [2023-05-08 11:23:04,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 0: [2023-05-08 11:23:04,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:23:04,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:23:04,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:23:04,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:23:04,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +30: [2023-05-08 11:23:04,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:23:04,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 0: [2023-05-08 11:23:04,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 1: [2023-05-08 11:23:04,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:23:04,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:23:04,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:23:04,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +27: [2023-05-08 11:23:04,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:23:04,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:23:04,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:23:04,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +27: [2023-05-08 11:23:04,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +27: [2023-05-08 11:23:04,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:23:04,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 1: [2023-05-08 11:23:04,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +25: [2023-05-08 11:23:04,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:23:04,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +17: [2023-05-08 11:23:04,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +25: [2023-05-08 11:23:04,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 6: [2023-05-08 11:23:04,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +19: [2023-05-08 11:23:04,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +15: [2023-05-08 11:23:04,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:23:04,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +15: [2023-05-08 11:23:04,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:23:04,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:23:04,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:23:04,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +15: [2023-05-08 11:23:04,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:23:04,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +28: [2023-05-08 11:23:04,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:23:04,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:23:04,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 4: [2023-05-08 11:23:04,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:23:04,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:23:04,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:23:04,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +14: [2023-05-08 11:23:04,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:23:04,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... + 0: [2023-05-08 11:23:04,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +27: [2023-05-08 11:23:04,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +30: [2023-05-08 11:23:04,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:23:04,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 4: [2023-05-08 11:23:04,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:23:04,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:23:04,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:23:04,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +24: [2023-05-08 11:23:04,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:23:04,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:23:04,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +14: [2023-05-08 11:23:04,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +27: [2023-05-08 11:23:04,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +24: [2023-05-08 11:23:04,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:23:04,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:23:04,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +24: [2023-05-08 11:23:04,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:23:04,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 2: [2023-05-08 11:23:04,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 2: [2023-05-08 11:23:04,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +24: [2023-05-08 11:23:04,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:23:04,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +26: [2023-05-08 11:23:04,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +30: [2023-05-08 11:23:04,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 4: [2023-05-08 11:23:04,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +30: [2023-05-08 11:23:04,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt... +15: [2023-05-08 11:23:04,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +27: [2023-05-08 11:23:04,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:23:04,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:23:04,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +15: [2023-05-08 11:23:04,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:23:04,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +27: [2023-05-08 11:23:04,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +27: [2023-05-08 11:23:04,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:23:04,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +27: [2023-05-08 11:23:04,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +26: [2023-05-08 11:23:04,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:23:04,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:23:04,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:23:04,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +15: [2023-05-08 11:23:04,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +15: [2023-05-08 11:23:04,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +15: [2023-05-08 11:23:04,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:23:04,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +26: [2023-05-08 11:23:04,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +24: [2023-05-08 11:23:04,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +27: [2023-05-08 11:23:04,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:23:04,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:23:04,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:23:04,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 4: [2023-05-08 11:23:04,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +24: [2023-05-08 11:23:04,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +24: [2023-05-08 11:23:04,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:23:04,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +27: [2023-05-08 11:23:04,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:23:04,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +27: [2023-05-08 11:23:04,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:23:04,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:23:04,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +24: [2023-05-08 11:23:04,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +27: [2023-05-08 11:23:04,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +24: [2023-05-08 11:23:04,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +22: [2023-05-08 11:23:04,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:23:04,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:23:04,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:23:04,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:23:04,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:23:04,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:23:04,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:23:04,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:23:04,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +14: [2023-05-08 11:23:04,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:23:04,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +14: [2023-05-08 11:23:04,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:23:04,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:23:04,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:23:04,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:23:04,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:23:04,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +27: [2023-05-08 11:23:04,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:23:04,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +24: [2023-05-08 11:23:04,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +27: [2023-05-08 11:23:04,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:23:04,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:23:04,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:23:04,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:23:04,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:23:04,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:23:04,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:23:04,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +22: [2023-05-08 11:23:04,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +14: [2023-05-08 11:23:04,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:23:04,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:23:04,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:23:04,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:23:04,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:23:04,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:23:04,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +22: [2023-05-08 11:23:04,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +25: [2023-05-08 11:23:04,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +25: [2023-05-08 11:23:04,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:23:04,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +25: [2023-05-08 11:23:04,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +25: [2023-05-08 11:23:04,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:23:04,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +25: [2023-05-08 11:23:04,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +25: [2023-05-08 11:23:04,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:23:04,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +31: [2023-05-08 11:23:04,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:23:04,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:23:04,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +10: [2023-05-08 11:23:04,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:23:04,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:23:04,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:23:04,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +10: [2023-05-08 11:23:04,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +10: [2023-05-08 11:23:04,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +31: [2023-05-08 11:23:04,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +31: [2023-05-08 11:23:04,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:23:04,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +31: [2023-05-08 11:23:04,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +31: [2023-05-08 11:23:04,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:23:04,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:23:04,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:23:04,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +28: [2023-05-08 11:23:04,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +28: [2023-05-08 11:23:04,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:23:04,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +28: [2023-05-08 11:23:04,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:23:04,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 2: [2023-05-08 11:23:04,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 2: [2023-05-08 11:23:04,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:23:04,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 1: [2023-05-08 11:23:04,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +31: [2023-05-08 11:23:04,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +10: [2023-05-08 11:23:04,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +16: [2023-05-08 11:23:04,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:23:04,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +16: [2023-05-08 11:23:04,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +31: [2023-05-08 11:23:04,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:23:04,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +16: [2023-05-08 11:23:04,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:23:04,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:23:04,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +25: [2023-05-08 11:23:04,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +25: [2023-05-08 11:23:04,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +25: [2023-05-08 11:23:04,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +31: [2023-05-08 11:23:04,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:23:04,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +25: [2023-05-08 11:23:04,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:23:04,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:23:04,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:23:04,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:23:04,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +28: [2023-05-08 11:23:04,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:23:04,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:23:04,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:23:04,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +31: [2023-05-08 11:23:04,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:23:04,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:23:04,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:23:04,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:23:04,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +25: [2023-05-08 11:23:04,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +25: [2023-05-08 11:23:04,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +28: [2023-05-08 11:23:04,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:23:04,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +28: [2023-05-08 11:23:04,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +28: [2023-05-08 11:23:04,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +28: [2023-05-08 11:23:04,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +31: [2023-05-08 11:23:04,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:23:04,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:23:04,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +25: [2023-05-08 11:23:04,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:23:04,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:23:04,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +12: [2023-05-08 11:23:04,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +12: [2023-05-08 11:23:04,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +12: [2023-05-08 11:23:04,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +29: [2023-05-08 11:23:04,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:23:04,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:23:04,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:23:04,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:23:04,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:23:04,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:23:04,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:23:04,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +31: [2023-05-08 11:23:04,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +17: [2023-05-08 11:23:04,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:23:04,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +17: [2023-05-08 11:23:04,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:23:04,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 6: [2023-05-08 11:23:04,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:23:04,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +17: [2023-05-08 11:23:04,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:23:04,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:23:04,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +17: [2023-05-08 11:23:04,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 7: [2023-05-08 11:23:04,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 1: [2023-05-08 11:23:04,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:23:04,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 5: [2023-05-08 11:23:04,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +16: [2023-05-08 11:23:04,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 6: [2023-05-08 11:23:04,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:23:04,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:23:04,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +12: [2023-05-08 11:23:04,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:23:04,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:23:04,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:23:04,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:23:04,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:23:04,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:23:04,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +12: [2023-05-08 11:23:04,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:23:04,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:23:04,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +31: [2023-05-08 11:23:04,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:23:04,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:23:04,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:23:04,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:23:04,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:23:04,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:23:04,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:23:04,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:23:04,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:23:04,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +29: [2023-05-08 11:23:04,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +29: [2023-05-08 11:23:04,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +30: [2023-05-08 11:23:04,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:23:04,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:23:04,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:23:04,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:23:04,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:23:04,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:23:04,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:23:04,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:23:04,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:23:04,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +12: [2023-05-08 11:23:04,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +17: [2023-05-08 11:23:04,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:23:04,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:23:04,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:23:04,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:23:04,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:23:04,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:23:04,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:23:04,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +12: [2023-05-08 11:23:04,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:23:04,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:23:04,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +17: [2023-05-08 11:23:04,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:23:04,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:23:04,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:23:04,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:23:04,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:23:04,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:23:04,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:23:04,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 6: [2023-05-08 11:23:04,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:23:04,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +23: [2023-05-08 11:23:04,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:23:04,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +27: [2023-05-08 11:23:04,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:23:04,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +27: [2023-05-08 11:23:04,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +27: [2023-05-08 11:23:04,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:23:04,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:23:04,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 7: [2023-05-08 11:23:04,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +17: [2023-05-08 11:23:04,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:23:04,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:23:04,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:23:04,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:23:04,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:23:04,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:23:04,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:23:04,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:23:04,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +30: [2023-05-08 11:23:04,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:23:04,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +23: [2023-05-08 11:23:04,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:23:04,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:23:04,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +21: [2023-05-08 11:23:04,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +21: [2023-05-08 11:23:04,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:23:04,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +21: [2023-05-08 11:23:04,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:23:04,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +21: [2023-05-08 11:23:04,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +27: [2023-05-08 11:23:04,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +21: [2023-05-08 11:23:04,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:23:04,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +13: [2023-05-08 11:23:04,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +13: [2023-05-08 11:23:04,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +13: [2023-05-08 11:23:04,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +13: [2023-05-08 11:23:04,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +13: [2023-05-08 11:23:04,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +13: [2023-05-08 11:23:04,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +13: [2023-05-08 11:23:04,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 8: [2023-05-08 11:23:04,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 8: [2023-05-08 11:23:04,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +27: [2023-05-08 11:23:04,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:23:04,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +17: [2023-05-08 11:23:04,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +13: [2023-05-08 11:23:04,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +19: [2023-05-08 11:23:04,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +19: [2023-05-08 11:23:04,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +19: [2023-05-08 11:23:04,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +19: [2023-05-08 11:23:04,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:23:04,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +23: [2023-05-08 11:23:04,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +23: [2023-05-08 11:23:04,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +17: [2023-05-08 11:23:04,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 3: [2023-05-08 11:23:04,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +30: [2023-05-08 11:23:04,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +19: [2023-05-08 11:23:04,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:23:04,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:23:04,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:23:04,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:23:04,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:23:04,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +19: [2023-05-08 11:23:04,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 3: [2023-05-08 11:23:04,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +26: [2023-05-08 11:23:04,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +19: [2023-05-08 11:23:04,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:23:04,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 9: [2023-05-08 11:23:04,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:23:04,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 9: [2023-05-08 11:23:04,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +19: [2023-05-08 11:23:04,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:23:04,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +30: [2023-05-08 11:23:04,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +26: [2023-05-08 11:23:04,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:23:04,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +27: [2023-05-08 11:23:04,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:23:04,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +27: [2023-05-08 11:23:04,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +27: [2023-05-08 11:23:04,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +21: [2023-05-08 11:23:04,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +17: [2023-05-08 11:23:04,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:23:04,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:23:04,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:23:04,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +18: [2023-05-08 11:23:04,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +18: [2023-05-08 11:23:04,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +18: [2023-05-08 11:23:04,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +18: [2023-05-08 11:23:04,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +18: [2023-05-08 11:23:04,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +18: [2023-05-08 11:23:04,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +18: [2023-05-08 11:23:04,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:23:04,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:23:04,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +30: [2023-05-08 11:23:04,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +27: [2023-05-08 11:23:04,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +11: [2023-05-08 11:23:04,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +13: [2023-05-08 11:23:04,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +13: [2023-05-08 11:23:04,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +11: [2023-05-08 11:23:04,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +11: [2023-05-08 11:23:04,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +11: [2023-05-08 11:23:04,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +11: [2023-05-08 11:23:04,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +20: [2023-05-08 11:23:04,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +20: [2023-05-08 11:23:04,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +20: [2023-05-08 11:23:04,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +20: [2023-05-08 11:23:04,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +20: [2023-05-08 11:23:04,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +11: [2023-05-08 11:23:04,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +20: [2023-05-08 11:23:04,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +20: [2023-05-08 11:23:04,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +11: [2023-05-08 11:23:04,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +11: [2023-05-08 11:23:04,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +21: [2023-05-08 11:23:04,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +23: [2023-05-08 11:23:04,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +20: [2023-05-08 11:23:04,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:23:04,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 9: [2023-05-08 11:23:04,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:23:04,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +22: [2023-05-08 11:23:04,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +11: [2023-05-08 11:23:04,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +15: [2023-05-08 11:23:04,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +15: [2023-05-08 11:23:04,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +22: [2023-05-08 11:23:04,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +22: [2023-05-08 11:23:04,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +15: [2023-05-08 11:23:04,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +22: [2023-05-08 11:23:04,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +15: [2023-05-08 11:23:04,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +21: [2023-05-08 11:23:04,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +27: [2023-05-08 11:23:04,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:23:04,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +26: [2023-05-08 11:23:04,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +26: [2023-05-08 11:23:04,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +13: [2023-05-08 11:23:04,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +11: [2023-05-08 11:23:04,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +24: [2023-05-08 11:23:04,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +24: [2023-05-08 11:23:04,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +24: [2023-05-08 11:23:04,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +21: [2023-05-08 11:23:04,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +24: [2023-05-08 11:23:04,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 4: [2023-05-08 11:23:04,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +19: [2023-05-08 11:23:04,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +19: [2023-05-08 11:23:04,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +19: [2023-05-08 11:23:04,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +20: [2023-05-08 11:23:04,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 4: [2023-05-08 11:23:04,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +20: [2023-05-08 11:23:04,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +20: [2023-05-08 11:23:04,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +18: [2023-05-08 11:23:04,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +20: [2023-05-08 11:23:04,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 0: [2023-05-08 11:23:04,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:23:04,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +13: [2023-05-08 11:23:04,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +23: [2023-05-08 11:23:04,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +23: [2023-05-08 11:23:04,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 0: [2023-05-08 11:23:04,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +22: [2023-05-08 11:23:04,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +18: [2023-05-08 11:23:04,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +18: [2023-05-08 11:23:04,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +27: [2023-05-08 11:23:04,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +13: [2023-05-08 11:23:04,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +13: [2023-05-08 11:23:04,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:23:04,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +26: [2023-05-08 11:23:04,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:23:04,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:23:04,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:23:04,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:23:04,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +26: [2023-05-08 11:23:04,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:23:04,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 9: [2023-05-08 11:23:04,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +17: [2023-05-08 11:23:04,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:23:04,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:23:04,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:23:04,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +21: [2023-05-08 11:23:04,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +22: [2023-05-08 11:23:04,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:23:04,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +15: [2023-05-08 11:23:04,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +15: [2023-05-08 11:23:04,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +19: [2023-05-08 11:23:04,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +11: [2023-05-08 11:23:04,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:23:04,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:23:04,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +17: [2023-05-08 11:23:04,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +19: [2023-05-08 11:23:04,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +19: [2023-05-08 11:23:04,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +16: [2023-05-08 11:23:04,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +24: [2023-05-08 11:23:04,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +24: [2023-05-08 11:23:04,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:23:04,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +19: [2023-05-08 11:23:04,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:23:04,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +26: [2023-05-08 11:23:04,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:23:04,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +16: [2023-05-08 11:23:04,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +15: [2023-05-08 11:23:04,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +15: [2023-05-08 11:23:04,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:23:04,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:23:04,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:23:04,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +11: [2023-05-08 11:23:04,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +23: [2023-05-08 11:23:04,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:23:04,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +24: [2023-05-08 11:23:04,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:23:04,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +11: [2023-05-08 11:23:04,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:23:04,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +17: [2023-05-08 11:23:04,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +17: [2023-05-08 11:23:04,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 8: [2023-05-08 11:23:04,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +18: [2023-05-08 11:23:04,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:23:04,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 7: [2023-05-08 11:23:04,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +11: [2023-05-08 11:23:04,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +13: [2023-05-08 11:23:04,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +18: [2023-05-08 11:23:04,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +17: [2023-05-08 11:23:04,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +23: [2023-05-08 11:23:04,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +17: [2023-05-08 11:23:04,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 0: [2023-05-08 11:23:04,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +13: [2023-05-08 11:23:04,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +13: [2023-05-08 11:23:04,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +23: [2023-05-08 11:23:04,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +23: [2023-05-08 11:23:04,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:23:04,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 3: [2023-05-08 11:23:04,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:23:04,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 3: [2023-05-08 11:23:04,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 3: [2023-05-08 11:23:04,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +21: [2023-05-08 11:23:04,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:23:04,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 7: [2023-05-08 11:23:04,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +26: [2023-05-08 11:23:04,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +13: [2023-05-08 11:23:04,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +26: [2023-05-08 11:23:04,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +20: [2023-05-08 11:23:04,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +26: [2023-05-08 11:23:04,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +13: [2023-05-08 11:23:04,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +11: [2023-05-08 11:23:04,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +19: [2023-05-08 11:23:04,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +16: [2023-05-08 11:23:04,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:23:04,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:23:04,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:23:04,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:23:04,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:23:04,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:23:04,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 3: [2023-05-08 11:23:04,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 3: [2023-05-08 11:23:04,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:23:04,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:23:04,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +20: [2023-05-08 11:23:04,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +15: [2023-05-08 11:23:04,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +15: [2023-05-08 11:23:04,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +20: [2023-05-08 11:23:04,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 8: [2023-05-08 11:23:04,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:23:04,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +18: [2023-05-08 11:23:04,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:23:04,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +23: [2023-05-08 11:23:04,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +11: [2023-05-08 11:23:04,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +11: [2023-05-08 11:23:04,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +23: [2023-05-08 11:23:04,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 3: [2023-05-08 11:23:04,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:23:04,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +20: [2023-05-08 11:23:04,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:23:04,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:23:04,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +15: [2023-05-08 11:23:04,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 8: [2023-05-08 11:23:04,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +26: [2023-05-08 11:23:04,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:23:04,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +13: [2023-05-08 11:23:04,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:23:04,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:23:04,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:23:04,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:23:04,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +24: [2023-05-08 11:23:04,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +24: [2023-05-08 11:23:04,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +25: [2023-05-08 11:23:04,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +25: [2023-05-08 11:23:04,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +25: [2023-05-08 11:23:04,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +25: [2023-05-08 11:23:04,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +25: [2023-05-08 11:23:04,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +25: [2023-05-08 11:23:04,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +25: [2023-05-08 11:23:04,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +24: [2023-05-08 11:23:04,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:23:04,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +13: [2023-05-08 11:23:04,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:23:04,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +23: [2023-05-08 11:23:04,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +23: [2023-05-08 11:23:04,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +13: [2023-05-08 11:23:04,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +25: [2023-05-08 11:23:04,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +26: [2023-05-08 11:23:04,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:23:04,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:23:04,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 4: [2023-05-08 11:23:04,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +20: [2023-05-08 11:23:04,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +20: [2023-05-08 11:23:04,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +20: [2023-05-08 11:23:04,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +20: [2023-05-08 11:23:04,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +20: [2023-05-08 11:23:04,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +23: [2023-05-08 11:23:04,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +23: [2023-05-08 11:23:04,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +16: [2023-05-08 11:23:04,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:23:04,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:23:04,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +26: [2023-05-08 11:23:04,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:23:04,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +20: [2023-05-08 11:23:04,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:23:04,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:23:04,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +14: [2023-05-08 11:23:04,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:23:04,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:23:04,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +14: [2023-05-08 11:23:04,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:23:04,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:23:04,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:23:04,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:23:04,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:23:04,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +15: [2023-05-08 11:23:04,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:23:04,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:23:04,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:23:04,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:23:04,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 8: [2023-05-08 11:23:04,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:23:04,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +23: [2023-05-08 11:23:04,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +26: [2023-05-08 11:23:04,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +22: [2023-05-08 11:23:04,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +26: [2023-05-08 11:23:04,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +22: [2023-05-08 11:23:04,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 8: [2023-05-08 11:23:04,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:23:04,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:23:04,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 5: [2023-05-08 11:23:04,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 5: [2023-05-08 11:23:04,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +25: [2023-05-08 11:23:04,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +13: [2023-05-08 11:23:04,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +13: [2023-05-08 11:23:04,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +13: [2023-05-08 11:23:04,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +13: [2023-05-08 11:23:04,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +13: [2023-05-08 11:23:04,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +13: [2023-05-08 11:23:04,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +13: [2023-05-08 11:23:04,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +13: [2023-05-08 11:23:04,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 5: [2023-05-08 11:23:04,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +14: [2023-05-08 11:23:04,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +14: [2023-05-08 11:23:04,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 5: [2023-05-08 11:23:04,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 4: [2023-05-08 11:23:04,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:23:04,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +18: [2023-05-08 11:23:04,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +18: [2023-05-08 11:23:04,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +14: [2023-05-08 11:23:04,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +18: [2023-05-08 11:23:04,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 2: [2023-05-08 11:23:04,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 2: [2023-05-08 11:23:04,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +20: [2023-05-08 11:23:04,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +11: [2023-05-08 11:23:04,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +28: [2023-05-08 11:23:04,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +28: [2023-05-08 11:23:04,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +28: [2023-05-08 11:23:04,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +25: [2023-05-08 11:23:04,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +25: [2023-05-08 11:23:04,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:23:04,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +11: [2023-05-08 11:23:04,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:23:04,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +11: [2023-05-08 11:23:04,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +11: [2023-05-08 11:23:04,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +11: [2023-05-08 11:23:04,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +27: [2023-05-08 11:23:04,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +27: [2023-05-08 11:23:04,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +27: [2023-05-08 11:23:04,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +27: [2023-05-08 11:23:04,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 9: [2023-05-08 11:23:04,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 9: [2023-05-08 11:23:04,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 9: [2023-05-08 11:23:04,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +11: [2023-05-08 11:23:04,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 9: [2023-05-08 11:23:04,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +20: [2023-05-08 11:23:04,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:23:04,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 2: [2023-05-08 11:23:04,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +14: [2023-05-08 11:23:04,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +28: [2023-05-08 11:23:04,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +29: [2023-05-08 11:23:04,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +29: [2023-05-08 11:23:04,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +29: [2023-05-08 11:23:04,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +29: [2023-05-08 11:23:04,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +16: [2023-05-08 11:23:04,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:23:04,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +25: [2023-05-08 11:23:04,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 6: [2023-05-08 11:23:04,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 6: [2023-05-08 11:23:04,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 6: [2023-05-08 11:23:04,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 5: [2023-05-08 11:23:04,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +24: [2023-05-08 11:23:04,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +18: [2023-05-08 11:23:04,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +24: [2023-05-08 11:23:04,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +24: [2023-05-08 11:23:04,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +24: [2023-05-08 11:23:04,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +24: [2023-05-08 11:23:04,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +24: [2023-05-08 11:23:04,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +24: [2023-05-08 11:23:04,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +24: [2023-05-08 11:23:04,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +18: [2023-05-08 11:23:04,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +28: [2023-05-08 11:23:04,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:23:04,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +28: [2023-05-08 11:23:04,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +28: [2023-05-08 11:23:04,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:23:04,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +14: [2023-05-08 11:23:04,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:23:04,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:23:04,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:23:04,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +14: [2023-05-08 11:23:04,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +25: [2023-05-08 11:23:04,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:23:04,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:23:04,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:23:04,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +28: [2023-05-08 11:23:04,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +29: [2023-05-08 11:23:04,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +29: [2023-05-08 11:23:04,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +29: [2023-05-08 11:23:04,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +29: [2023-05-08 11:23:04,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +14: [2023-05-08 11:23:04,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:23:04,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:23:04,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:23:04,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:23:04,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:23:04,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +18: [2023-05-08 11:23:04,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:23:04,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +24: [2023-05-08 11:23:04,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +18: [2023-05-08 11:23:04,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 8: [2023-05-08 11:23:04,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +18: [2023-05-08 11:23:04,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +18: [2023-05-08 11:23:04,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +28: [2023-05-08 11:23:04,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +28: [2023-05-08 11:23:04,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +16: [2023-05-08 11:23:04,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +16: [2023-05-08 11:23:04,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +16: [2023-05-08 11:23:04,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +16: [2023-05-08 11:23:04,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +16: [2023-05-08 11:23:04,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +16: [2023-05-08 11:23:04,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +24: [2023-05-08 11:23:04,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +24: [2023-05-08 11:23:04,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +18: [2023-05-08 11:23:04,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:23:04,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 8: [2023-05-08 11:23:04,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:23:04,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +16: [2023-05-08 11:23:04,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +16: [2023-05-08 11:23:04,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +14: [2023-05-08 11:23:04,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:23:04,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +27: [2023-05-08 11:23:04,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 9: [2023-05-08 11:23:04,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:23:04,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 9: [2023-05-08 11:23:04,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +25: [2023-05-08 11:23:04,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 9: [2023-05-08 11:23:04,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +27: [2023-05-08 11:23:04,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 9: [2023-05-08 11:23:04,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +27: [2023-05-08 11:23:04,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 5: [2023-05-08 11:23:04,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +14: [2023-05-08 11:23:04,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +14: [2023-05-08 11:23:04,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:23:04,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:23:04,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +28: [2023-05-08 11:23:04,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:23:04,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 6: [2023-05-08 11:23:04,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +12: [2023-05-08 11:23:04,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +12: [2023-05-08 11:23:04,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +28: [2023-05-08 11:23:04,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +24: [2023-05-08 11:23:04,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:23:04,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +24: [2023-05-08 11:23:04,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +14: [2023-05-08 11:23:04,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +27: [2023-05-08 11:23:04,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +27: [2023-05-08 11:23:04,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 9: [2023-05-08 11:23:04,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:23:04,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 5: [2023-05-08 11:23:04,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 8: [2023-05-08 11:23:04,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +18: [2023-05-08 11:23:04,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +28: [2023-05-08 11:23:04,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:23:04,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +24: [2023-05-08 11:23:04,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +24: [2023-05-08 11:23:04,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:23:04,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +27: [2023-05-08 11:23:04,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 5: [2023-05-08 11:23:04,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +27: [2023-05-08 11:23:04,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 5: [2023-05-08 11:23:04,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:23:04,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 6: [2023-05-08 11:23:04,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +21: [2023-05-08 11:23:04,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +21: [2023-05-08 11:23:04,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +29: [2023-05-08 11:23:04,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:23:04,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 6: [2023-05-08 11:23:04,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 6: [2023-05-08 11:23:04,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 1: [2023-05-08 11:23:04,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +29: [2023-05-08 11:23:04,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +12: [2023-05-08 11:23:04,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:23:04,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:23:04,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:23:04,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +14: [2023-05-08 11:23:04,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 1: [2023-05-08 11:23:04,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 1: [2023-05-08 11:23:04,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +16: [2023-05-08 11:23:04,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +16: [2023-05-08 11:23:04,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +12: [2023-05-08 11:23:04,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:23:04,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:23:04,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:23:04,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +17: [2023-05-08 11:23:04,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +14: [2023-05-08 11:23:04,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:23:04,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +29: [2023-05-08 11:23:04,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:23:04,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:23:04,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +19: [2023-05-08 11:23:04,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +19: [2023-05-08 11:23:04,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +19: [2023-05-08 11:23:04,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +19: [2023-05-08 11:23:04,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 5: [2023-05-08 11:23:04,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +17: [2023-05-08 11:23:04,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +29: [2023-05-08 11:23:04,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +29: [2023-05-08 11:23:04,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +29: [2023-05-08 11:23:04,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 1: [2023-05-08 11:23:04,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:23:04,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +17: [2023-05-08 11:23:04,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +21: [2023-05-08 11:23:04,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +21: [2023-05-08 11:23:04,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +16: [2023-05-08 11:23:04,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:23:04,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +21: [2023-05-08 11:23:04,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +21: [2023-05-08 11:23:04,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:23:04,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:23:04,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:23:04,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +17: [2023-05-08 11:23:04,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:23:04,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +19: [2023-05-08 11:23:04,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +17: [2023-05-08 11:23:04,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 5: [2023-05-08 11:23:04,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:23:04,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +17: [2023-05-08 11:23:04,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:23:04,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 8: [2023-05-08 11:23:04,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 8: [2023-05-08 11:23:04,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 1: [2023-05-08 11:23:04,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +10: [2023-05-08 11:23:04,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 1: [2023-05-08 11:23:04,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +10: [2023-05-08 11:23:04,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:23:04,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:23:04,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:23:04,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:23:04,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:23:04,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:23:04,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:23:04,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:23:04,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:23:04,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +21: [2023-05-08 11:23:04,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +21: [2023-05-08 11:23:04,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +21: [2023-05-08 11:23:04,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 4: [2023-05-08 11:23:04,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +22: [2023-05-08 11:23:04,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +22: [2023-05-08 11:23:04,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +22: [2023-05-08 11:23:04,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +22: [2023-05-08 11:23:04,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +22: [2023-05-08 11:23:04,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +22: [2023-05-08 11:23:04,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +22: [2023-05-08 11:23:04,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +22: [2023-05-08 11:23:04,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +31: [2023-05-08 11:23:04,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +19: [2023-05-08 11:23:04,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +31: [2023-05-08 11:23:04,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:23:04,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:23:04,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +31: [2023-05-08 11:23:04,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +31: [2023-05-08 11:23:04,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +28: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +28: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +19: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +28: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +31: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +31: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +31: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +28: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +19: [2023-05-08 11:23:04,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +19: [2023-05-08 11:23:04,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +12: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:23:04,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 2: [2023-05-08 11:23:04,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 2: [2023-05-08 11:23:04,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +12: [2023-05-08 11:23:04,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:23:04,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +12: [2023-05-08 11:23:04,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +12: [2023-05-08 11:23:04,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +12: [2023-05-08 11:23:04,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 2: [2023-05-08 11:23:04,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:23:04,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +28: [2023-05-08 11:23:04,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +28: [2023-05-08 11:23:04,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:23:04,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +23: [2023-05-08 11:23:04,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +23: [2023-05-08 11:23:04,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +12: [2023-05-08 11:23:04,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +13: [2023-05-08 11:23:04,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 2: [2023-05-08 11:23:04,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +19: [2023-05-08 11:23:04,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +23: [2023-05-08 11:23:04,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +13: [2023-05-08 11:23:04,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +13: [2023-05-08 11:23:04,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +19: [2023-05-08 11:23:04,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +13: [2023-05-08 11:23:04,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 1: [2023-05-08 11:23:04,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:23:04,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +28: [2023-05-08 11:23:04,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +19: [2023-05-08 11:23:04,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +19: [2023-05-08 11:23:04,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 1: [2023-05-08 11:23:04,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:23:04,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:23:04,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:23:04,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +30: [2023-05-08 11:23:04,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +21: [2023-05-08 11:23:04,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +30: [2023-05-08 11:23:04,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +10: [2023-05-08 11:23:04,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +30: [2023-05-08 11:23:04,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +10: [2023-05-08 11:23:04,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 1: [2023-05-08 11:23:04,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 4: [2023-05-08 11:23:04,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +10: [2023-05-08 11:23:04,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:23:04,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:23:04,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 4: [2023-05-08 11:23:04,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:23:04,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +31: [2023-05-08 11:23:04,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +28: [2023-05-08 11:23:04,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:23:04,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +22: [2023-05-08 11:23:04,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +10: [2023-05-08 11:23:04,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:23:04,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:23:04,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:23:04,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:23:04,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:23:04,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:23:04,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:23:04,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:23:04,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:23:04,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +13: [2023-05-08 11:23:04,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:23:04,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:23:04,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:23:04,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +17: [2023-05-08 11:23:04,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:23:04,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +17: [2023-05-08 11:23:04,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +17: [2023-05-08 11:23:04,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +17: [2023-05-08 11:23:04,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 8: [2023-05-08 11:23:04,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +12: [2023-05-08 11:23:04,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +23: [2023-05-08 11:23:04,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:23:04,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:23:04,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +13: [2023-05-08 11:23:04,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:23:04,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:23:04,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +17: [2023-05-08 11:23:04,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +31: [2023-05-08 11:23:04,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +28: [2023-05-08 11:23:04,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:23:04,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:23:04,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:23:04,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 4: [2023-05-08 11:23:04,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +13: [2023-05-08 11:23:04,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:23:04,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:23:04,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +31: [2023-05-08 11:23:04,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +30: [2023-05-08 11:23:04,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:23:04,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:23:04,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:23:04,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +10: [2023-05-08 11:23:04,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +31: [2023-05-08 11:23:04,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +31: [2023-05-08 11:23:04,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +31: [2023-05-08 11:23:04,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 0: [2023-05-08 11:23:04,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +12: [2023-05-08 11:23:04,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +28: [2023-05-08 11:23:04,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +10: [2023-05-08 11:23:04,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:23:04,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:23:04,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +10: [2023-05-08 11:23:04,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:23:04,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:23:04,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:23:04,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +22: [2023-05-08 11:23:04,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +25: [2023-05-08 11:23:04,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +25: [2023-05-08 11:23:04,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 0: [2023-05-08 11:23:04,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +12: [2023-05-08 11:23:04,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +12: [2023-05-08 11:23:04,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +25: [2023-05-08 11:23:04,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +10: [2023-05-08 11:23:04,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:23:04,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. +31: [2023-05-08 11:23:04,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +28: [2023-05-08 11:23:04,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +25: [2023-05-08 11:23:04,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt. + 6: [2023-05-08 11:23:04,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +28: [2023-05-08 11:23:04,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +13: [2023-05-08 11:23:04,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:23:04,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:23:04,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:23:04,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 0: [2023-05-08 11:23:04,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:23:04,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:23:04,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 2: [2023-05-08 11:23:04,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +17: [2023-05-08 11:23:04,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:23:04,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:23:04,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:23:04,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:23:04,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +15: [2023-05-08 11:23:04,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +15: [2023-05-08 11:23:04,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +30: [2023-05-08 11:23:04,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +15: [2023-05-08 11:23:04,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +15: [2023-05-08 11:23:04,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +22: [2023-05-08 11:23:04,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +15: [2023-05-08 11:23:04,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +15: [2023-05-08 11:23:04,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +31: [2023-05-08 11:23:04,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +31: [2023-05-08 11:23:04,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +15: [2023-05-08 11:23:04,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +13: [2023-05-08 11:23:04,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +13: [2023-05-08 11:23:04,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 0: [2023-05-08 11:23:04,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +28: [2023-05-08 11:23:04,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +31: [2023-05-08 11:23:04,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:23:04,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:23:04,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +25: [2023-05-08 11:23:04,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:23:04,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +28: [2023-05-08 11:23:04,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:23:04,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +30: [2023-05-08 11:23:04,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +10: [2023-05-08 11:23:04,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +25: [2023-05-08 11:23:04,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:23:04,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +25: [2023-05-08 11:23:04,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:23:04,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 6: [2023-05-08 11:23:04,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +12: [2023-05-08 11:23:04,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +23: [2023-05-08 11:23:04,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:23:04,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +30: [2023-05-08 11:23:04,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +23: [2023-05-08 11:23:04,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +31: [2023-05-08 11:23:04,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt... + 1: [2023-05-08 11:23:04,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +17: [2023-05-08 11:23:04,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +17: [2023-05-08 11:23:04,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +17: [2023-05-08 11:23:04,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +23: [2023-05-08 11:23:04,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:23:04,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +17: [2023-05-08 11:23:04,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 2: [2023-05-08 11:23:04,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +28: [2023-05-08 11:23:04,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +10: [2023-05-08 11:23:04,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:23:04,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +10: [2023-05-08 11:23:04,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +10: [2023-05-08 11:23:04,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:23:04,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +13: [2023-05-08 11:23:04,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:23:04,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:23:04,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +15: [2023-05-08 11:23:04,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +23: [2023-05-08 11:23:04,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:23:04,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +23: [2023-05-08 11:23:04,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +13: [2023-05-08 11:23:04,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +25: [2023-05-08 11:23:04,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:23:04,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +25: [2023-05-08 11:23:04,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +30: [2023-05-08 11:23:04,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +31: [2023-05-08 11:23:04,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +31: [2023-05-08 11:23:04,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +25: [2023-05-08 11:23:04,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +31: [2023-05-08 11:23:04,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +25: [2023-05-08 11:23:04,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +31: [2023-05-08 11:23:04,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +30: [2023-05-08 11:23:04,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 0: [2023-05-08 11:23:04,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +25: [2023-05-08 11:23:04,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +25: [2023-05-08 11:23:04,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +10: [2023-05-08 11:23:04,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:23:04,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:23:04,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:23:04,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:23:04,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +15: [2023-05-08 11:23:04,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... + 1: [2023-05-08 11:23:04,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 0: [2023-05-08 11:23:04,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. + 0: [2023-05-08 11:23:04,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +26: [2023-05-08 11:23:04,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +26: [2023-05-08 11:23:04,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +26: [2023-05-08 11:23:04,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +26: [2023-05-08 11:23:04,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +26: [2023-05-08 11:23:04,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +26: [2023-05-08 11:23:04,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +26: [2023-05-08 11:23:04,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +26: [2023-05-08 11:23:04,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +14: [2023-05-08 11:23:04,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +14: [2023-05-08 11:23:04,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +14: [2023-05-08 11:23:04,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +31: [2023-05-08 11:23:04,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +25: [2023-05-08 11:23:04,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +31: [2023-05-08 11:23:04,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt. +31: [2023-05-08 11:23:04,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +14: [2023-05-08 11:23:04,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +29: [2023-05-08 11:23:04,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +29: [2023-05-08 11:23:04,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +29: [2023-05-08 11:23:04,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +29: [2023-05-08 11:23:04,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +29: [2023-05-08 11:23:04,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +29: [2023-05-08 11:23:04,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +29: [2023-05-08 11:23:04,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +29: [2023-05-08 11:23:04,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +25: [2023-05-08 11:23:04,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +31: [2023-05-08 11:23:04,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt... +15: [2023-05-08 11:23:04,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 6: [2023-05-08 11:23:04,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 6: [2023-05-08 11:23:04,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 6: [2023-05-08 11:23:04,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +28: [2023-05-08 11:23:04,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +28: [2023-05-08 11:23:04,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +28: [2023-05-08 11:23:04,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +28: [2023-05-08 11:23:04,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +14: [2023-05-08 11:23:04,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:23:04,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:23:04,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:23:04,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +28: [2023-05-08 11:23:04,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +28: [2023-05-08 11:23:04,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +28: [2023-05-08 11:23:04,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:23:04,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +26: [2023-05-08 11:23:04,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +26: [2023-05-08 11:23:04,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +28: [2023-05-08 11:23:04,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:23:04,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +29: [2023-05-08 11:23:04,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +29: [2023-05-08 11:23:04,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +29: [2023-05-08 11:23:04,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +26: [2023-05-08 11:23:04,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +14: [2023-05-08 11:23:04,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:23:04,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +29: [2023-05-08 11:23:04,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +29: [2023-05-08 11:23:04,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +14: [2023-05-08 11:23:04,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:23:04,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +28: [2023-05-08 11:23:04,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +28: [2023-05-08 11:23:04,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +29: [2023-05-08 11:23:04,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +26: [2023-05-08 11:23:04,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +26: [2023-05-08 11:23:04,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +26: [2023-05-08 11:23:04,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:23:04,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +23: [2023-05-08 11:23:04,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +23: [2023-05-08 11:23:04,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +23: [2023-05-08 11:23:04,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +23: [2023-05-08 11:23:04,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +23: [2023-05-08 11:23:04,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +23: [2023-05-08 11:23:04,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +29: [2023-05-08 11:23:04,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +29: [2023-05-08 11:23:04,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 6: [2023-05-08 11:23:04,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:23:04,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 6: [2023-05-08 11:23:04,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +29: [2023-05-08 11:23:04,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:23:04,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +26: [2023-05-08 11:23:04,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +29: [2023-05-08 11:23:04,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 6: [2023-05-08 11:23:04,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:23:04,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +14: [2023-05-08 11:23:04,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 6: [2023-05-08 11:23:04,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +28: [2023-05-08 11:23:04,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +28: [2023-05-08 11:23:04,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 6: [2023-05-08 11:23:04,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:23:04,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +29: [2023-05-08 11:23:04,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +28: [2023-05-08 11:23:04,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +28: [2023-05-08 11:23:04,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +29: [2023-05-08 11:23:04,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:23:04,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:23:04,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:23:04,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +28: [2023-05-08 11:23:04,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:23:04,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +23: [2023-05-08 11:23:04,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +28: [2023-05-08 11:23:04,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +19: [2023-05-08 11:23:04,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +19: [2023-05-08 11:23:04,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +19: [2023-05-08 11:23:04,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +19: [2023-05-08 11:23:04,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +23: [2023-05-08 11:23:04,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +23: [2023-05-08 11:23:04,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +29: [2023-05-08 11:23:04,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:23:04,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 1: [2023-05-08 11:23:04,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +29: [2023-05-08 11:23:04,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 1: [2023-05-08 11:23:04,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 1: [2023-05-08 11:23:04,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +21: [2023-05-08 11:23:04,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +21: [2023-05-08 11:23:04,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +23: [2023-05-08 11:23:04,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +21: [2023-05-08 11:23:04,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +23: [2023-05-08 11:23:04,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +21: [2023-05-08 11:23:05,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +23: [2023-05-08 11:23:05,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:23:05,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +19: [2023-05-08 11:23:05,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +19: [2023-05-08 11:23:05,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +19: [2023-05-08 11:23:05,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +19: [2023-05-08 11:23:05,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:23:05,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +29: [2023-05-08 11:23:05,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:23:05,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:23:05,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +29: [2023-05-08 11:23:05,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +10: [2023-05-08 11:23:05,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +10: [2023-05-08 11:23:05,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 1: [2023-05-08 11:23:05,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:23:05,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:23:05,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:23:05,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:23:05,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +30: [2023-05-08 11:23:05,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +30: [2023-05-08 11:23:05,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +30: [2023-05-08 11:23:05,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +21: [2023-05-08 11:23:05,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:23:05,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +29: [2023-05-08 11:23:05,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:23:05,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +10: [2023-05-08 11:23:05,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 7: [2023-05-08 11:23:05,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:23:05,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +19: [2023-05-08 11:23:05,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +19: [2023-05-08 11:23:05,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 7: [2023-05-08 11:23:05,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +31: [2023-05-08 11:23:05,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +31: [2023-05-08 11:23:05,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 5: [2023-05-08 11:23:05,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:23:05,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:23:05,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +19: [2023-05-08 11:23:05,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 5: [2023-05-08 11:23:05,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +25: [2023-05-08 11:23:05,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +25: [2023-05-08 11:23:05,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 1: [2023-05-08 11:23:05,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:23:05,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:23:05,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:23:05,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:23:05,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:23:05,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:23:05,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +20: [2023-05-08 11:23:05,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +19: [2023-05-08 11:23:05,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +20: [2023-05-08 11:23:05,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +20: [2023-05-08 11:23:05,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +20: [2023-05-08 11:23:05,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:23:05,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:23:05,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +25: [2023-05-08 11:23:05,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +10: [2023-05-08 11:23:05,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +25: [2023-05-08 11:23:05,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +30: [2023-05-08 11:23:05,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:23:05,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 1: [2023-05-08 11:23:05,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +31: [2023-05-08 11:23:05,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +21: [2023-05-08 11:23:05,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:23:05,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:23:05,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +21: [2023-05-08 11:23:05,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +31: [2023-05-08 11:23:05,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +21: [2023-05-08 11:23:05,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +21: [2023-05-08 11:23:05,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 5: [2023-05-08 11:23:05,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +31: [2023-05-08 11:23:05,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +31: [2023-05-08 11:23:05,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +19: [2023-05-08 11:23:05,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +19: [2023-05-08 11:23:05,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +25: [2023-05-08 11:23:05,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +11: [2023-05-08 11:23:05,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:23:05,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:23:05,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +11: [2023-05-08 11:23:05,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +20: [2023-05-08 11:23:05,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:23:05,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:23:05,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +31: [2023-05-08 11:23:05,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +25: [2023-05-08 11:23:05,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +25: [2023-05-08 11:23:05,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +11: [2023-05-08 11:23:05,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +11: [2023-05-08 11:23:05,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:23:05,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:23:05,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +31: [2023-05-08 11:23:05,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:23:05,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +20: [2023-05-08 11:23:05,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:23:05,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:23:05,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +21: [2023-05-08 11:23:05,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 0: [2023-05-08 11:23:05,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 0: [2023-05-08 11:23:05,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 0: [2023-05-08 11:23:05,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. +10: [2023-05-08 11:23:05,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 0: [2023-05-08 11:23:05,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt. + 7: [2023-05-08 11:23:05,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +10: [2023-05-08 11:23:05,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:23:05,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 7: [2023-05-08 11:23:05,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +20: [2023-05-08 11:23:05,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +20: [2023-05-08 11:23:05,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:23:05,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:23:05,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 0: [2023-05-08 11:23:05,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:23:05,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:23:05,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:23:05,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +24: [2023-05-08 11:23:05,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +24: [2023-05-08 11:23:05,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +24: [2023-05-08 11:23:05,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +24: [2023-05-08 11:23:05,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +24: [2023-05-08 11:23:05,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +24: [2023-05-08 11:23:05,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +11: [2023-05-08 11:23:05,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:23:05,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +25: [2023-05-08 11:23:05,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:23:05,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:23:05,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +31: [2023-05-08 11:23:05,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +25: [2023-05-08 11:23:05,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +20: [2023-05-08 11:23:05,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +31: [2023-05-08 11:23:05,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +31: [2023-05-08 11:23:05,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +25: [2023-05-08 11:23:05,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +25: [2023-05-08 11:23:05,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +10: [2023-05-08 11:23:05,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:23:05,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +30: [2023-05-08 11:23:05,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +18: [2023-05-08 11:23:05,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +18: [2023-05-08 11:23:05,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +16: [2023-05-08 11:23:05,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:23:05,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:23:05,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:23:05,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:23:05,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +25: [2023-05-08 11:23:05,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +18: [2023-05-08 11:23:05,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:23:05,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:23:05,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:23:05,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +18: [2023-05-08 11:23:05,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:23:05,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +16: [2023-05-08 11:23:05,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +16: [2023-05-08 11:23:05,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:23:05,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:23:05,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:23:05,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +31: [2023-05-08 11:23:05,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +31: [2023-05-08 11:23:05,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 0: [2023-05-08 11:23:05,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:23:05,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:23:05,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +20: [2023-05-08 11:23:05,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +20: [2023-05-08 11:23:05,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +31: [2023-05-08 11:23:05,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +31: [2023-05-08 11:23:05,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 9: [2023-05-08 11:23:05,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +11: [2023-05-08 11:23:05,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:23:05,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +24: [2023-05-08 11:23:05,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +11: [2023-05-08 11:23:05,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:23:05,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +16: [2023-05-08 11:23:05,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +18: [2023-05-08 11:23:05,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:23:05,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +18: [2023-05-08 11:23:05,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +24: [2023-05-08 11:23:05,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +24: [2023-05-08 11:23:05,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:23:05,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:23:05,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +25: [2023-05-08 11:23:05,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +24: [2023-05-08 11:23:05,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 3: [2023-05-08 11:23:05,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 3: [2023-05-08 11:23:05,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 3: [2023-05-08 11:23:05,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +24: [2023-05-08 11:23:05,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 3: [2023-05-08 11:23:05,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +24: [2023-05-08 11:23:05,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +24: [2023-05-08 11:23:05,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 4: [2023-05-08 11:23:05,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 4: [2023-05-08 11:23:05,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 4: [2023-05-08 11:23:05,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +16: [2023-05-08 11:23:05,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +16: [2023-05-08 11:23:05,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +18: [2023-05-08 11:23:05,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +18: [2023-05-08 11:23:05,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:23:05,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +11: [2023-05-08 11:23:05,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:23:05,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +16: [2023-05-08 11:23:05,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +18: [2023-05-08 11:23:05,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +24: [2023-05-08 11:23:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:23:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +27: [2023-05-08 11:23:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +27: [2023-05-08 11:23:05,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +22: [2023-05-08 11:23:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:23:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:23:05,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:23:05,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:23:05,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:23:05,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:23:05,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:23:05,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +27: [2023-05-08 11:23:05,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +27: [2023-05-08 11:23:05,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +27: [2023-05-08 11:23:05,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +27: [2023-05-08 11:23:05,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +27: [2023-05-08 11:23:05,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +18: [2023-05-08 11:23:05,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +24: [2023-05-08 11:23:05,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +24: [2023-05-08 11:23:05,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +18: [2023-05-08 11:23:05,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +14: [2023-05-08 11:23:05,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +14: [2023-05-08 11:23:05,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +14: [2023-05-08 11:23:05,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +14: [2023-05-08 11:23:05,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +14: [2023-05-08 11:23:05,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +14: [2023-05-08 11:23:05,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +14: [2023-05-08 11:23:05,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +16: [2023-05-08 11:23:05,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +14: [2023-05-08 11:23:05,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +16: [2023-05-08 11:23:05,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:23:05,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +24: [2023-05-08 11:23:05,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +24: [2023-05-08 11:23:05,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... + 3: [2023-05-08 11:23:05,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +24: [2023-05-08 11:23:05,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +24: [2023-05-08 11:23:05,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +22: [2023-05-08 11:23:05,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +16: [2023-05-08 11:23:05,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:23:05,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:23:05,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:23:05,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +30: [2023-05-08 11:23:05,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +30: [2023-05-08 11:23:05,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +30: [2023-05-08 11:23:05,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +27: [2023-05-08 11:23:05,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +30: [2023-05-08 11:23:05,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +30: [2023-05-08 11:23:05,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +30: [2023-05-08 11:23:05,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt... +22: [2023-05-08 11:23:05,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:23:05,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +30: [2023-05-08 11:23:05,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +14: [2023-05-08 11:23:05,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +14: [2023-05-08 11:23:05,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 7: [2023-05-08 11:23:05,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 7: [2023-05-08 11:23:05,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 7: [2023-05-08 11:23:05,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 3: [2023-05-08 11:23:05,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 4: [2023-05-08 11:23:05,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +24: [2023-05-08 11:23:05,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +24: [2023-05-08 11:23:05,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 4: [2023-05-08 11:23:05,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +14: [2023-05-08 11:23:05,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +22: [2023-05-08 11:23:05,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +14: [2023-05-08 11:23:05,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +16: [2023-05-08 11:23:05,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +27: [2023-05-08 11:23:05,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +19: [2023-05-08 11:23:05,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +19: [2023-05-08 11:23:05,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +19: [2023-05-08 11:23:05,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +19: [2023-05-08 11:23:05,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:23:05,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +19: [2023-05-08 11:23:05,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +19: [2023-05-08 11:23:05,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +22: [2023-05-08 11:23:05,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +19: [2023-05-08 11:23:05,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +27: [2023-05-08 11:23:05,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:23:05,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +22: [2023-05-08 11:23:05,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +14: [2023-05-08 11:23:05,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:23:05,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +14: [2023-05-08 11:23:05,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:23:05,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +24: [2023-05-08 11:23:05,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +24: [2023-05-08 11:23:05,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 7: [2023-05-08 11:23:05,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +25: [2023-05-08 11:23:05,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +25: [2023-05-08 11:23:05,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +25: [2023-05-08 11:23:05,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +25: [2023-05-08 11:23:05,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +25: [2023-05-08 11:23:05,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +25: [2023-05-08 11:23:05,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +25: [2023-05-08 11:23:05,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +14: [2023-05-08 11:23:05,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +14: [2023-05-08 11:23:05,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:23:05,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +15: [2023-05-08 11:23:05,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +15: [2023-05-08 11:23:05,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:23:05,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +15: [2023-05-08 11:23:05,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +15: [2023-05-08 11:23:05,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +15: [2023-05-08 11:23:05,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:23:05,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +27: [2023-05-08 11:23:05,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +24: [2023-05-08 11:23:05,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +15: [2023-05-08 11:23:05,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +24: [2023-05-08 11:23:05,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +30: [2023-05-08 11:23:05,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +30: [2023-05-08 11:23:05,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +29: [2023-05-08 11:23:05,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:23:05,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:23:05,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +29: [2023-05-08 11:23:05,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +29: [2023-05-08 11:23:05,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:23:05,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:23:05,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:23:05,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +27: [2023-05-08 11:23:05,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +29: [2023-05-08 11:23:05,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +24: [2023-05-08 11:23:05,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +19: [2023-05-08 11:23:05,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 2: [2023-05-08 11:23:05,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +24: [2023-05-08 11:23:05,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 4: [2023-05-08 11:23:05,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +13: [2023-05-08 11:23:05,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +13: [2023-05-08 11:23:05,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +13: [2023-05-08 11:23:05,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +13: [2023-05-08 11:23:05,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +13: [2023-05-08 11:23:05,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +13: [2023-05-08 11:23:05,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +13: [2023-05-08 11:23:05,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +13: [2023-05-08 11:23:05,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +27: [2023-05-08 11:23:05,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +30: [2023-05-08 11:23:05,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +19: [2023-05-08 11:23:05,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +19: [2023-05-08 11:23:05,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +15: [2023-05-08 11:23:05,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +15: [2023-05-08 11:23:05,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +15: [2023-05-08 11:23:05,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +15: [2023-05-08 11:23:05,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 7: [2023-05-08 11:23:05,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 7: [2023-05-08 11:23:05,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 8: [2023-05-08 11:23:05,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +15: [2023-05-08 11:23:05,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:23:05,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:23:05,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +15: [2023-05-08 11:23:05,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +25: [2023-05-08 11:23:05,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +25: [2023-05-08 11:23:05,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +29: [2023-05-08 11:23:05,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:23:05,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:23:05,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:23:05,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +13: [2023-05-08 11:23:05,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +13: [2023-05-08 11:23:05,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +30: [2023-05-08 11:23:05,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +21: [2023-05-08 11:23:05,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +21: [2023-05-08 11:23:05,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +21: [2023-05-08 11:23:05,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +21: [2023-05-08 11:23:05,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +21: [2023-05-08 11:23:05,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +19: [2023-05-08 11:23:05,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +21: [2023-05-08 11:23:05,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +21: [2023-05-08 11:23:05,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +21: [2023-05-08 11:23:05,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +13: [2023-05-08 11:23:05,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +15: [2023-05-08 11:23:05,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +15: [2023-05-08 11:23:05,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +25: [2023-05-08 11:23:05,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +19: [2023-05-08 11:23:05,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:23:05,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +29: [2023-05-08 11:23:05,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:23:05,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:23:05,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:23:05,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:23:05,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +29: [2023-05-08 11:23:05,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +15: [2023-05-08 11:23:05,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +29: [2023-05-08 11:23:05,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +15: [2023-05-08 11:23:05,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +26: [2023-05-08 11:23:05,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:23:05,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +15: [2023-05-08 11:23:05,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +26: [2023-05-08 11:23:05,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +26: [2023-05-08 11:23:05,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +13: [2023-05-08 11:23:05,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +19: [2023-05-08 11:23:05,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +29: [2023-05-08 11:23:05,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +26: [2023-05-08 11:23:05,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:23:05,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:23:05,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:23:05,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +23: [2023-05-08 11:23:05,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:23:05,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:23:05,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:23:05,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:23:05,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +13: [2023-05-08 11:23:05,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +13: [2023-05-08 11:23:05,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +26: [2023-05-08 11:23:05,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:23:05,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:23:05,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:23:05,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:23:05,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +21: [2023-05-08 11:23:05,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +23: [2023-05-08 11:23:05,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +29: [2023-05-08 11:23:05,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +25: [2023-05-08 11:23:05,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +21: [2023-05-08 11:23:05,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +21: [2023-05-08 11:23:05,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +13: [2023-05-08 11:23:05,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +19: [2023-05-08 11:23:05,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:23:05,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +15: [2023-05-08 11:23:05,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +15: [2023-05-08 11:23:05,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +15: [2023-05-08 11:23:05,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 8: [2023-05-08 11:23:05,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:23:05,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +15: [2023-05-08 11:23:05,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +15: [2023-05-08 11:23:05,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 2: [2023-05-08 11:23:05,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +21: [2023-05-08 11:23:05,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +21: [2023-05-08 11:23:05,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:23:05,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +13: [2023-05-08 11:23:05,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +21: [2023-05-08 11:23:05,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:23:05,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +12: [2023-05-08 11:23:05,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +12: [2023-05-08 11:23:05,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +31: [2023-05-08 11:23:05,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +12: [2023-05-08 11:23:05,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +12: [2023-05-08 11:23:05,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +31: [2023-05-08 11:23:05,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +31: [2023-05-08 11:23:05,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +31: [2023-05-08 11:23:05,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +12: [2023-05-08 11:23:05,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +12: [2023-05-08 11:23:05,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +31: [2023-05-08 11:23:05,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +31: [2023-05-08 11:23:05,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +31: [2023-05-08 11:23:05,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +12: [2023-05-08 11:23:05,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +31: [2023-05-08 11:23:05,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +28: [2023-05-08 11:23:05,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +21: [2023-05-08 11:23:05,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +28: [2023-05-08 11:23:05,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +28: [2023-05-08 11:23:05,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +28: [2023-05-08 11:23:05,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +28: [2023-05-08 11:23:05,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +28: [2023-05-08 11:23:05,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +28: [2023-05-08 11:23:05,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +28: [2023-05-08 11:23:05,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 8: [2023-05-08 11:23:05,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 8: [2023-05-08 11:23:05,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 8: [2023-05-08 11:23:05,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +26: [2023-05-08 11:23:05,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:23:05,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +26: [2023-05-08 11:23:05,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:23:05,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +10: [2023-05-08 11:23:05,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +10: [2023-05-08 11:23:05,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +10: [2023-05-08 11:23:05,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +10: [2023-05-08 11:23:05,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +10: [2023-05-08 11:23:05,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +23: [2023-05-08 11:23:05,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +15: [2023-05-08 11:23:05,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:23:05,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +10: [2023-05-08 11:23:05,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +10: [2023-05-08 11:23:05,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +21: [2023-05-08 11:23:05,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:23:05,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +12: [2023-05-08 11:23:05,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +22: [2023-05-08 11:23:05,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +23: [2023-05-08 11:23:05,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:23:05,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:23:05,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:23:05,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +12: [2023-05-08 11:23:05,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +28: [2023-05-08 11:23:05,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +28: [2023-05-08 11:23:05,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +26: [2023-05-08 11:23:05,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +22: [2023-05-08 11:23:05,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +22: [2023-05-08 11:23:05,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +26: [2023-05-08 11:23:05,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +26: [2023-05-08 11:23:05,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +31: [2023-05-08 11:23:05,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +17: [2023-05-08 11:23:05,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +17: [2023-05-08 11:23:05,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +17: [2023-05-08 11:23:05,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +17: [2023-05-08 11:23:05,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +17: [2023-05-08 11:23:05,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +31: [2023-05-08 11:23:05,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +31: [2023-05-08 11:23:05,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +17: [2023-05-08 11:23:05,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +17: [2023-05-08 11:23:05,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +17: [2023-05-08 11:23:05,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +28: [2023-05-08 11:23:05,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +28: [2023-05-08 11:23:05,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +23: [2023-05-08 11:23:05,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +22: [2023-05-08 11:23:05,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +23: [2023-05-08 11:23:05,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +23: [2023-05-08 11:23:05,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +12: [2023-05-08 11:23:05,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:23:05,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +26: [2023-05-08 11:23:05,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:23:05,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:23:05,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:23:05,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:23:05,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +10: [2023-05-08 11:23:05,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +10: [2023-05-08 11:23:05,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +22: [2023-05-08 11:23:05,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +12: [2023-05-08 11:23:05,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:23:05,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:23:05,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +31: [2023-05-08 11:23:05,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:23:05,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:23:05,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +12: [2023-05-08 11:23:05,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +31: [2023-05-08 11:23:05,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +28: [2023-05-08 11:23:05,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +17: [2023-05-08 11:23:05,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +17: [2023-05-08 11:23:05,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +17: [2023-05-08 11:23:05,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +17: [2023-05-08 11:23:05,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +26: [2023-05-08 11:23:05,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +10: [2023-05-08 11:23:05,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:23:05,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +20: [2023-05-08 11:23:05,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +20: [2023-05-08 11:23:05,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +10: [2023-05-08 11:23:05,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:23:05,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +20: [2023-05-08 11:23:05,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +10: [2023-05-08 11:23:05,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:23:05,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:23:05,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:23:05,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +22: [2023-05-08 11:23:05,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +22: [2023-05-08 11:23:05,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +22: [2023-05-08 11:23:05,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 1: [2023-05-08 11:23:05,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +17: [2023-05-08 11:23:05,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +17: [2023-05-08 11:23:05,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:23:05,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +22: [2023-05-08 11:23:05,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +22: [2023-05-08 11:23:05,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +22: [2023-05-08 11:23:05,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 6: [2023-05-08 11:23:05,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +17: [2023-05-08 11:23:05,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +17: [2023-05-08 11:23:05,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:23:05,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:23:05,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +20: [2023-05-08 11:23:05,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:23:05,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +20: [2023-05-08 11:23:05,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 1: [2023-05-08 11:23:05,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:23:05,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +20: [2023-05-08 11:23:05,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +20: [2023-05-08 11:23:05,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +20: [2023-05-08 11:23:05,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 1: [2023-05-08 11:23:05,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:23:05,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +20: [2023-05-08 11:23:05,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +20: [2023-05-08 11:23:05,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +20: [2023-05-08 11:23:05,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 0: [2023-05-08 11:23:05,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:23:05,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +26: [2023-05-08 11:23:05,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +26: [2023-05-08 11:23:05,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +26: [2023-05-08 11:23:05,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 0: [2023-05-08 11:23:05,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:23:05,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +23: [2023-05-08 11:23:05,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +27: [2023-05-08 11:23:05,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:23:05,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 0: [2023-05-08 11:23:05,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:23:05,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +27: [2023-05-08 11:23:05,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:23:05,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +27: [2023-05-08 11:23:05,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:23:05,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +27: [2023-05-08 11:23:05,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +27: [2023-05-08 11:23:05,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:23:05,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +17: [2023-05-08 11:23:05,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +17: [2023-05-08 11:23:05,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +17: [2023-05-08 11:23:05,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +26: [2023-05-08 11:23:05,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +17: [2023-05-08 11:23:05,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +26: [2023-05-08 11:23:05,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +26: [2023-05-08 11:23:05,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +23: [2023-05-08 11:23:05,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +26: [2023-05-08 11:23:05,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +23: [2023-05-08 11:23:05,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +23: [2023-05-08 11:23:05,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +23: [2023-05-08 11:23:05,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +27: [2023-05-08 11:23:05,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:23:05,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:23:05,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:23:05,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +16: [2023-05-08 11:23:05,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +16: [2023-05-08 11:23:05,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 9: [2023-05-08 11:23:05,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:23:05,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +17: [2023-05-08 11:23:05,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +17: [2023-05-08 11:23:05,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +17: [2023-05-08 11:23:05,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +17: [2023-05-08 11:23:05,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +26: [2023-05-08 11:23:05,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +26: [2023-05-08 11:23:05,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 9: [2023-05-08 11:23:05,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:23:05,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +16: [2023-05-08 11:23:05,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +16: [2023-05-08 11:23:05,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +27: [2023-05-08 11:23:05,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +16: [2023-05-08 11:23:05,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +23: [2023-05-08 11:23:05,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +23: [2023-05-08 11:23:05,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +23: [2023-05-08 11:23:05,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +27: [2023-05-08 11:23:05,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +16: [2023-05-08 11:23:05,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +23: [2023-05-08 11:23:05,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +23: [2023-05-08 11:23:05,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +23: [2023-05-08 11:23:05,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +27: [2023-05-08 11:23:05,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:23:05,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +23: [2023-05-08 11:23:05,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +26: [2023-05-08 11:23:05,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +23: [2023-05-08 11:23:05,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 9: [2023-05-08 11:23:05,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +13: [2023-05-08 11:23:05,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +13: [2023-05-08 11:23:05,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +13: [2023-05-08 11:23:05,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +13: [2023-05-08 11:23:05,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +13: [2023-05-08 11:23:05,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +13: [2023-05-08 11:23:05,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +13: [2023-05-08 11:23:05,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +13: [2023-05-08 11:23:05,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +26: [2023-05-08 11:23:05,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +26: [2023-05-08 11:23:05,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +26: [2023-05-08 11:23:05,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +17: [2023-05-08 11:23:05,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +17: [2023-05-08 11:23:05,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +26: [2023-05-08 11:23:05,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 9: [2023-05-08 11:23:05,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +17: [2023-05-08 11:23:05,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +16: [2023-05-08 11:23:05,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +17: [2023-05-08 11:23:05,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +17: [2023-05-08 11:23:05,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +27: [2023-05-08 11:23:05,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +16: [2023-05-08 11:23:05,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 9: [2023-05-08 11:23:05,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +17: [2023-05-08 11:23:05,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 9: [2023-05-08 11:23:05,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:23:05,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:23:05,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +21: [2023-05-08 11:23:05,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:23:05,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:23:05,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:23:05,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +13: [2023-05-08 11:23:05,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:23:05,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +13: [2023-05-08 11:23:05,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:23:05,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +19: [2023-05-08 11:23:05,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:23:05,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:23:05,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +19: [2023-05-08 11:23:05,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +19: [2023-05-08 11:23:05,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:23:05,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +17: [2023-05-08 11:23:05,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +16: [2023-05-08 11:23:05,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +16: [2023-05-08 11:23:05,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +13: [2023-05-08 11:23:05,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:23:05,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +19: [2023-05-08 11:23:05,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +13: [2023-05-08 11:23:05,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +17: [2023-05-08 11:23:05,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +16: [2023-05-08 11:23:05,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +16: [2023-05-08 11:23:05,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 9: [2023-05-08 11:23:05,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +13: [2023-05-08 11:23:05,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +21: [2023-05-08 11:23:05,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +13: [2023-05-08 11:23:05,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 2: [2023-05-08 11:23:05,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 2: [2023-05-08 11:23:05,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 2: [2023-05-08 11:23:05,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 5: [2023-05-08 11:23:05,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +21: [2023-05-08 11:23:05,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +16: [2023-05-08 11:23:05,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +16: [2023-05-08 11:23:05,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +18: [2023-05-08 11:23:05,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +18: [2023-05-08 11:23:05,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +18: [2023-05-08 11:23:05,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +18: [2023-05-08 11:23:05,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +19: [2023-05-08 11:23:05,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +19: [2023-05-08 11:23:05,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +11: [2023-05-08 11:23:05,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +13: [2023-05-08 11:23:05,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +11: [2023-05-08 11:23:05,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +11: [2023-05-08 11:23:05,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +11: [2023-05-08 11:23:05,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +19: [2023-05-08 11:23:05,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +20: [2023-05-08 11:23:05,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +20: [2023-05-08 11:23:05,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +20: [2023-05-08 11:23:05,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +20: [2023-05-08 11:23:05,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +20: [2023-05-08 11:23:05,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +13: [2023-05-08 11:23:05,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +20: [2023-05-08 11:23:05,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +20: [2023-05-08 11:23:05,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +20: [2023-05-08 11:23:05,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +24: [2023-05-08 11:23:05,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +24: [2023-05-08 11:23:05,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +24: [2023-05-08 11:23:05,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +24: [2023-05-08 11:23:05,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +24: [2023-05-08 11:23:05,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +24: [2023-05-08 11:23:05,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +24: [2023-05-08 11:23:05,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +24: [2023-05-08 11:23:05,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +19: [2023-05-08 11:23:05,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +14: [2023-05-08 11:23:05,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +14: [2023-05-08 11:23:05,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +14: [2023-05-08 11:23:05,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +14: [2023-05-08 11:23:05,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +21: [2023-05-08 11:23:05,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +14: [2023-05-08 11:23:05,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +14: [2023-05-08 11:23:05,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +27: [2023-05-08 11:23:05,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +27: [2023-05-08 11:23:05,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +27: [2023-05-08 11:23:05,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +14: [2023-05-08 11:23:05,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +14: [2023-05-08 11:23:05,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +14: [2023-05-08 11:23:05,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +21: [2023-05-08 11:23:05,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +27: [2023-05-08 11:23:05,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +14: [2023-05-08 11:23:05,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +30: [2023-05-08 11:23:05,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +14: [2023-05-08 11:23:05,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:23:05,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 2: [2023-05-08 11:23:05,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +14: [2023-05-08 11:23:05,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +12: [2023-05-08 11:23:05,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +11: [2023-05-08 11:23:05,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +12: [2023-05-08 11:23:05,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 2: [2023-05-08 11:23:05,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +19: [2023-05-08 11:23:05,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +19: [2023-05-08 11:23:05,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 6: [2023-05-08 11:23:05,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +12: [2023-05-08 11:23:05,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +11: [2023-05-08 11:23:05,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +11: [2023-05-08 11:23:05,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +12: [2023-05-08 11:23:05,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 6: [2023-05-08 11:23:05,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 0: [2023-05-08 11:23:05,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 0: [2023-05-08 11:23:05,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +18: [2023-05-08 11:23:05,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +18: [2023-05-08 11:23:05,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +30: [2023-05-08 11:23:05,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +30: [2023-05-08 11:23:05,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +11: [2023-05-08 11:23:05,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +13: [2023-05-08 11:23:05,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +13: [2023-05-08 11:23:05,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +13: [2023-05-08 11:23:05,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +13: [2023-05-08 11:23:05,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 6: [2023-05-08 11:23:05,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +19: [2023-05-08 11:23:05,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:23:05,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +18: [2023-05-08 11:23:05,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +21: [2023-05-08 11:23:05,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:23:05,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +19: [2023-05-08 11:23:05,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +19: [2023-05-08 11:23:05,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +24: [2023-05-08 11:23:05,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +18: [2023-05-08 11:23:05,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +24: [2023-05-08 11:23:05,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +14: [2023-05-08 11:23:05,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +21: [2023-05-08 11:23:05,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 7: [2023-05-08 11:23:05,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +30: [2023-05-08 11:23:05,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +30: [2023-05-08 11:23:05,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +12: [2023-05-08 11:23:05,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +27: [2023-05-08 11:23:05,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +20: [2023-05-08 11:23:05,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +14: [2023-05-08 11:23:05,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +24: [2023-05-08 11:23:05,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +24: [2023-05-08 11:23:05,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +30: [2023-05-08 11:23:05,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 3: [2023-05-08 11:23:05,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 3: [2023-05-08 11:23:05,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +14: [2023-05-08 11:23:05,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +14: [2023-05-08 11:23:05,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +14: [2023-05-08 11:23:05,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +13: [2023-05-08 11:23:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +11: [2023-05-08 11:23:05,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +27: [2023-05-08 11:23:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +27: [2023-05-08 11:23:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +11: [2023-05-08 11:23:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +12: [2023-05-08 11:23:05,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +12: [2023-05-08 11:23:05,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +14: [2023-05-08 11:23:05,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +30: [2023-05-08 11:23:05,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +21: [2023-05-08 11:23:05,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +20: [2023-05-08 11:23:05,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +20: [2023-05-08 11:23:05,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +19: [2023-05-08 11:23:05,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:23:05,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +13: [2023-05-08 11:23:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +13: [2023-05-08 11:23:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +13: [2023-05-08 11:23:05,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +18: [2023-05-08 11:23:05,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +18: [2023-05-08 11:23:05,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +19: [2023-05-08 11:23:05,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +18: [2023-05-08 11:23:05,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +24: [2023-05-08 11:23:05,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +21: [2023-05-08 11:23:05,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +19: [2023-05-08 11:23:05,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +18: [2023-05-08 11:23:05,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 2: [2023-05-08 11:23:05,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 2: [2023-05-08 11:23:05,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +24: [2023-05-08 11:23:05,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +19: [2023-05-08 11:23:05,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +19: [2023-05-08 11:23:05,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +19: [2023-05-08 11:23:05,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +20: [2023-05-08 11:23:05,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +20: [2023-05-08 11:23:05,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +11: [2023-05-08 11:23:05,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:23:05,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +19: [2023-05-08 11:23:05,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +14: [2023-05-08 11:23:05,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +24: [2023-05-08 11:23:05,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +24: [2023-05-08 11:23:05,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +27: [2023-05-08 11:23:05,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +27: [2023-05-08 11:23:05,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 3: [2023-05-08 11:23:05,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +19: [2023-05-08 11:23:05,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:23:05,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +14: [2023-05-08 11:23:05,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +27: [2023-05-08 11:23:05,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +27: [2023-05-08 11:23:05,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +16: [2023-05-08 11:23:05,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +16: [2023-05-08 11:23:05,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +16: [2023-05-08 11:23:05,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +16: [2023-05-08 11:23:05,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +12: [2023-05-08 11:23:05,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +20: [2023-05-08 11:23:05,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +12: [2023-05-08 11:23:05,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +16: [2023-05-08 11:23:05,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +16: [2023-05-08 11:23:05,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +30: [2023-05-08 11:23:05,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +14: [2023-05-08 11:23:05,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +20: [2023-05-08 11:23:05,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +30: [2023-05-08 11:23:05,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +16: [2023-05-08 11:23:05,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +14: [2023-05-08 11:23:05,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:23:05,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +30: [2023-05-08 11:23:05,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +11: [2023-05-08 11:23:05,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +30: [2023-05-08 11:23:05,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:23:05,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +30: [2023-05-08 11:23:05,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:23:05,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +13: [2023-05-08 11:23:05,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +16: [2023-05-08 11:23:05,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +30: [2023-05-08 11:23:05,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +13: [2023-05-08 11:23:05,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +19: [2023-05-08 11:23:05,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +19: [2023-05-08 11:23:05,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 6: [2023-05-08 11:23:05,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +30: [2023-05-08 11:23:05,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:23:05,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +12: [2023-05-08 11:23:05,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 6: [2023-05-08 11:23:05,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 6: [2023-05-08 11:23:05,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +14: [2023-05-08 11:23:05,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +13: [2023-05-08 11:23:05,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +13: [2023-05-08 11:23:05,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +12: [2023-05-08 11:23:05,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +12: [2023-05-08 11:23:05,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +27: [2023-05-08 11:23:05,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +30: [2023-05-08 11:23:05,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +27: [2023-05-08 11:23:05,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +13: [2023-05-08 11:23:05,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +13: [2023-05-08 11:23:05,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +20: [2023-05-08 11:23:05,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +30: [2023-05-08 11:23:05,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +13: [2023-05-08 11:23:05,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +13: [2023-05-08 11:23:05,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 7: [2023-05-08 11:23:05,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +30: [2023-05-08 11:23:05,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +18: [2023-05-08 11:23:05,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +19: [2023-05-08 11:23:05,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +18: [2023-05-08 11:23:05,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +18: [2023-05-08 11:23:05,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +18: [2023-05-08 11:23:05,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +18: [2023-05-08 11:23:05,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +18: [2023-05-08 11:23:05,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +18: [2023-05-08 11:23:05,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +18: [2023-05-08 11:23:05,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +18: [2023-05-08 11:23:05,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +18: [2023-05-08 11:23:05,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 2: [2023-05-08 11:23:05,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:23:05,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +30: [2023-05-08 11:23:05,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +30: [2023-05-08 11:23:05,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +30: [2023-05-08 11:23:05,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 4: [2023-05-08 11:23:05,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +11: [2023-05-08 11:23:05,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +11: [2023-05-08 11:23:05,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +11: [2023-05-08 11:23:05,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +11: [2023-05-08 11:23:05,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +29: [2023-05-08 11:23:05,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +29: [2023-05-08 11:23:05,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +29: [2023-05-08 11:23:05,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +29: [2023-05-08 11:23:05,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +29: [2023-05-08 11:23:05,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +29: [2023-05-08 11:23:05,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +29: [2023-05-08 11:23:05,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +11: [2023-05-08 11:23:05,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +11: [2023-05-08 11:23:05,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +11: [2023-05-08 11:23:05,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +18: [2023-05-08 11:23:05,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +11: [2023-05-08 11:23:05,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +11: [2023-05-08 11:23:05,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +19: [2023-05-08 11:23:05,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +19: [2023-05-08 11:23:05,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +29: [2023-05-08 11:23:05,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 7: [2023-05-08 11:23:05,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +11: [2023-05-08 11:23:05,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 7: [2023-05-08 11:23:05,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 4: [2023-05-08 11:23:05,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 2: [2023-05-08 11:23:05,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +18: [2023-05-08 11:23:05,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 4: [2023-05-08 11:23:05,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:23:05,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +14: [2023-05-08 11:23:05,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 3: [2023-05-08 11:23:05,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 4: [2023-05-08 11:23:05,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +31: [2023-05-08 11:23:05,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +31: [2023-05-08 11:23:05,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +31: [2023-05-08 11:23:05,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +31: [2023-05-08 11:23:05,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +27: [2023-05-08 11:23:05,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 3: [2023-05-08 11:23:05,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +10: [2023-05-08 11:23:05,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 3: [2023-05-08 11:23:05,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +10: [2023-05-08 11:23:05,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +10: [2023-05-08 11:23:05,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +10: [2023-05-08 11:23:05,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +12: [2023-05-08 11:23:05,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +16: [2023-05-08 11:23:05,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +16: [2023-05-08 11:23:05,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +14: [2023-05-08 11:23:05,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +14: [2023-05-08 11:23:05,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +14: [2023-05-08 11:23:05,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +14: [2023-05-08 11:23:05,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +12: [2023-05-08 11:23:05,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 0: [2023-05-08 11:23:05,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 2: [2023-05-08 11:23:05,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +25: [2023-05-08 11:23:05,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:23:05,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:23:05,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +23: [2023-05-08 11:23:05,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +23: [2023-05-08 11:23:05,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +23: [2023-05-08 11:23:05,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +23: [2023-05-08 11:23:05,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +23: [2023-05-08 11:23:05,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +23: [2023-05-08 11:23:05,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +23: [2023-05-08 11:23:05,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +25: [2023-05-08 11:23:05,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:23:05,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:23:05,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:23:05,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:23:05,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +18: [2023-05-08 11:23:05,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 3: [2023-05-08 11:23:05,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +18: [2023-05-08 11:23:05,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +30: [2023-05-08 11:23:05,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:23:05,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +25: [2023-05-08 11:23:05,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +25: [2023-05-08 11:23:05,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:23:05,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:23:05,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +28: [2023-05-08 11:23:05,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:23:05,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:23:05,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 7: [2023-05-08 11:23:05,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:23:05,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:23:05,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:23:05,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +18: [2023-05-08 11:23:05,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +28: [2023-05-08 11:23:05,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:23:05,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:23:05,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:23:05,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +31: [2023-05-08 11:23:05,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +11: [2023-05-08 11:23:05,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +11: [2023-05-08 11:23:05,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +28: [2023-05-08 11:23:05,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:23:05,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +10: [2023-05-08 11:23:05,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +10: [2023-05-08 11:23:05,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 5: [2023-05-08 11:23:05,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +28: [2023-05-08 11:23:05,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 5: [2023-05-08 11:23:05,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +14: [2023-05-08 11:23:05,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:23:05,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +11: [2023-05-08 11:23:05,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +29: [2023-05-08 11:23:05,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +29: [2023-05-08 11:23:05,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +11: [2023-05-08 11:23:05,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +28: [2023-05-08 11:23:05,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +16: [2023-05-08 11:23:05,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +16: [2023-05-08 11:23:05,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +28: [2023-05-08 11:23:05,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +31: [2023-05-08 11:23:05,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +28: [2023-05-08 11:23:05,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 3: [2023-05-08 11:23:05,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +31: [2023-05-08 11:23:05,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +30: [2023-05-08 11:23:05,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +14: [2023-05-08 11:23:05,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:23:05,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:23:05,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:23:05,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:23:05,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:23:05,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:23:05,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +16: [2023-05-08 11:23:05,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +14: [2023-05-08 11:23:05,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 2: [2023-05-08 11:23:05,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +21: [2023-05-08 11:23:05,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +21: [2023-05-08 11:23:05,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +21: [2023-05-08 11:23:05,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +18: [2023-05-08 11:23:05,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +12: [2023-05-08 11:23:05,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +12: [2023-05-08 11:23:05,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:23:05,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +10: [2023-05-08 11:23:05,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:23:05,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +21: [2023-05-08 11:23:05,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +10: [2023-05-08 11:23:05,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:23:05,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:23:05,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:23:05,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:23:05,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +25: [2023-05-08 11:23:05,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +25: [2023-05-08 11:23:05,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +29: [2023-05-08 11:23:05,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +29: [2023-05-08 11:23:05,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +29: [2023-05-08 11:23:05,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +29: [2023-05-08 11:23:05,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +29: [2023-05-08 11:23:05,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +18: [2023-05-08 11:23:05,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +18: [2023-05-08 11:23:05,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +11: [2023-05-08 11:23:05,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +30: [2023-05-08 11:23:05,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +10: [2023-05-08 11:23:05,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +11: [2023-05-08 11:23:05,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 9: [2023-05-08 11:23:05,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 5: [2023-05-08 11:23:05,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 9: [2023-05-08 11:23:05,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. +14: [2023-05-08 11:23:05,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +25: [2023-05-08 11:23:05,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +16: [2023-05-08 11:23:05,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +25: [2023-05-08 11:23:05,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +11: [2023-05-08 11:23:05,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +11: [2023-05-08 11:23:05,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +29: [2023-05-08 11:23:05,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +30: [2023-05-08 11:23:05,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +29: [2023-05-08 11:23:05,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +23: [2023-05-08 11:23:05,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +23: [2023-05-08 11:23:05,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +30: [2023-05-08 11:23:05,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 8: [2023-05-08 11:23:05,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +28: [2023-05-08 11:23:05,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +23: [2023-05-08 11:23:05,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +23: [2023-05-08 11:23:05,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:23:05,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 4: [2023-05-08 11:23:05,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +10: [2023-05-08 11:23:05,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +28: [2023-05-08 11:23:05,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +28: [2023-05-08 11:23:05,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +28: [2023-05-08 11:23:05,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +21: [2023-05-08 11:23:05,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +10: [2023-05-08 11:23:05,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +28: [2023-05-08 11:23:05,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +16: [2023-05-08 11:23:05,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +21: [2023-05-08 11:23:05,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +21: [2023-05-08 11:23:05,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 3: [2023-05-08 11:23:05,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +21: [2023-05-08 11:23:05,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:23:05,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +15: [2023-05-08 11:23:05,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +15: [2023-05-08 11:23:05,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +15: [2023-05-08 11:23:05,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +29: [2023-05-08 11:23:05,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +12: [2023-05-08 11:23:05,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:23:05,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:23:05,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +31: [2023-05-08 11:23:05,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +15: [2023-05-08 11:23:05,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +31: [2023-05-08 11:23:05,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +29: [2023-05-08 11:23:05,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:23:05,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +29: [2023-05-08 11:23:05,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +30: [2023-05-08 11:23:05,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +23: [2023-05-08 11:23:05,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +15: [2023-05-08 11:23:05,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +15: [2023-05-08 11:23:05,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +15: [2023-05-08 11:23:05,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 8: [2023-05-08 11:23:05,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:23:05,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +15: [2023-05-08 11:23:05,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 4: [2023-05-08 11:23:05,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 4: [2023-05-08 11:23:05,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 9: [2023-05-08 11:23:05,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +23: [2023-05-08 11:23:05,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +23: [2023-05-08 11:23:05,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:23:05,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +25: [2023-05-08 11:23:05,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:23:05,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:23:05,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +31: [2023-05-08 11:23:05,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:23:05,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +31: [2023-05-08 11:23:05,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +31: [2023-05-08 11:23:05,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:23:05,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +31: [2023-05-08 11:23:05,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:23:05,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +31: [2023-05-08 11:23:05,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +10: [2023-05-08 11:23:05,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +31: [2023-05-08 11:23:05,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:23:05,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +31: [2023-05-08 11:23:05,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +31: [2023-05-08 11:23:05,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +28: [2023-05-08 11:23:05,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 5: [2023-05-08 11:23:05,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +10: [2023-05-08 11:23:05,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:23:05,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +31: [2023-05-08 11:23:05,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:23:05,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +12: [2023-05-08 11:23:05,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +29: [2023-05-08 11:23:05,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +12: [2023-05-08 11:23:05,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +31: [2023-05-08 11:23:05,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:23:05,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +31: [2023-05-08 11:23:05,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +10: [2023-05-08 11:23:05,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +21: [2023-05-08 11:23:05,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +31: [2023-05-08 11:23:05,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +21: [2023-05-08 11:23:05,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 4: [2023-05-08 11:23:05,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:23:05,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +28: [2023-05-08 11:23:05,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:23:05,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +31: [2023-05-08 11:23:05,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 4: [2023-05-08 11:23:05,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:23:05,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:23:05,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +15: [2023-05-08 11:23:05,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +15: [2023-05-08 11:23:05,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +28: [2023-05-08 11:23:05,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:23:05,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +25: [2023-05-08 11:23:05,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +25: [2023-05-08 11:23:05,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +25: [2023-05-08 11:23:05,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +10: [2023-05-08 11:23:05,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +10: [2023-05-08 11:23:05,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +25: [2023-05-08 11:23:05,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +25: [2023-05-08 11:23:05,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +25: [2023-05-08 11:23:05,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +25: [2023-05-08 11:23:05,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +17: [2023-05-08 11:23:05,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +17: [2023-05-08 11:23:05,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +17: [2023-05-08 11:23:05,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +17: [2023-05-08 11:23:05,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +31: [2023-05-08 11:23:05,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:23:05,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +10: [2023-05-08 11:23:05,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +21: [2023-05-08 11:23:05,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +21: [2023-05-08 11:23:05,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +21: [2023-05-08 11:23:05,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 8: [2023-05-08 11:23:05,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 5: [2023-05-08 11:23:05,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 6: [2023-05-08 11:23:05,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:23:05,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +15: [2023-05-08 11:23:05,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +15: [2023-05-08 11:23:05,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +10: [2023-05-08 11:23:05,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +29: [2023-05-08 11:23:05,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +15: [2023-05-08 11:23:05,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +15: [2023-05-08 11:23:05,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +31: [2023-05-08 11:23:05,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:23:05,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +17: [2023-05-08 11:23:05,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +17: [2023-05-08 11:23:05,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +17: [2023-05-08 11:23:05,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +24: [2023-05-08 11:23:05,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +15: [2023-05-08 11:23:05,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +25: [2023-05-08 11:23:05,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +24: [2023-05-08 11:23:05,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +22: [2023-05-08 11:23:05,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +22: [2023-05-08 11:23:05,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +22: [2023-05-08 11:23:05,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +22: [2023-05-08 11:23:05,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +17: [2023-05-08 11:23:05,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:23:05,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +31: [2023-05-08 11:23:05,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +15: [2023-05-08 11:23:05,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +24: [2023-05-08 11:23:05,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +15: [2023-05-08 11:23:05,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +22: [2023-05-08 11:23:05,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +22: [2023-05-08 11:23:05,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +24: [2023-05-08 11:23:05,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +22: [2023-05-08 11:23:05,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +12: [2023-05-08 11:23:05,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +21: [2023-05-08 11:23:05,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +28: [2023-05-08 11:23:05,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +28: [2023-05-08 11:23:05,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +28: [2023-05-08 11:23:05,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +28: [2023-05-08 11:23:05,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 8: [2023-05-08 11:23:05,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 8: [2023-05-08 11:23:05,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +22: [2023-05-08 11:23:05,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +28: [2023-05-08 11:23:05,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +21: [2023-05-08 11:23:05,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +29: [2023-05-08 11:23:05,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +12: [2023-05-08 11:23:05,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +29: [2023-05-08 11:23:05,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +31: [2023-05-08 11:23:05,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +10: [2023-05-08 11:23:05,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 9: [2023-05-08 11:23:05,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +25: [2023-05-08 11:23:05,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:23:05,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +15: [2023-05-08 11:23:05,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:23:05,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 9: [2023-05-08 11:23:05,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +15: [2023-05-08 11:23:05,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:23:05,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +24: [2023-05-08 11:23:05,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +24: [2023-05-08 11:23:05,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +29: [2023-05-08 11:23:05,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +29: [2023-05-08 11:23:05,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 4: [2023-05-08 11:23:05,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +31: [2023-05-08 11:23:05,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +15: [2023-05-08 11:23:05,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +29: [2023-05-08 11:23:05,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +24: [2023-05-08 11:23:05,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +29: [2023-05-08 11:23:05,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +15: [2023-05-08 11:23:05,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +24: [2023-05-08 11:23:05,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:23:05,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +17: [2023-05-08 11:23:05,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +17: [2023-05-08 11:23:05,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +31: [2023-05-08 11:23:05,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +15: [2023-05-08 11:23:05,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:23:05,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 9: [2023-05-08 11:23:05,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +17: [2023-05-08 11:23:05,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:23:05,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... + 1: [2023-05-08 11:23:05,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +31: [2023-05-08 11:23:05,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:23:05,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +25: [2023-05-08 11:23:05,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +22: [2023-05-08 11:23:05,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +15: [2023-05-08 11:23:05,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +28: [2023-05-08 11:23:05,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +17: [2023-05-08 11:23:05,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +17: [2023-05-08 11:23:05,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +22: [2023-05-08 11:23:05,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +15: [2023-05-08 11:23:05,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 1: [2023-05-08 11:23:05,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 1: [2023-05-08 11:23:05,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 1: [2023-05-08 11:23:05,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt. + 0: [2023-05-08 11:23:05,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +15: [2023-05-08 11:23:05,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +24: [2023-05-08 11:23:05,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +24: [2023-05-08 11:23:05,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +31: [2023-05-08 11:23:05,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +24: [2023-05-08 11:23:05,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +24: [2023-05-08 11:23:05,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +24: [2023-05-08 11:23:05,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +24: [2023-05-08 11:23:05,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +15: [2023-05-08 11:23:05,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +15: [2023-05-08 11:23:05,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +17: [2023-05-08 11:23:05,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +22: [2023-05-08 11:23:05,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +22: [2023-05-08 11:23:05,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +22: [2023-05-08 11:23:05,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +17: [2023-05-08 11:23:05,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +24: [2023-05-08 11:23:05,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +24: [2023-05-08 11:23:05,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 9: [2023-05-08 11:23:05,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +22: [2023-05-08 11:23:05,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +22: [2023-05-08 11:23:05,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 0: [2023-05-08 11:23:05,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +22: [2023-05-08 11:23:05,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 1: [2023-05-08 11:23:05,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +22: [2023-05-08 11:23:05,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 0: [2023-05-08 11:23:05,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt... +22: [2023-05-08 11:23:05,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +17: [2023-05-08 11:23:05,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +15: [2023-05-08 11:23:05,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +15: [2023-05-08 11:23:05,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +22: [2023-05-08 11:23:05,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:23:05,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:23:05,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +26: [2023-05-08 11:23:05,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +26: [2023-05-08 11:23:05,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +26: [2023-05-08 11:23:05,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +26: [2023-05-08 11:23:05,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +22: [2023-05-08 11:23:05,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:23:05,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +26: [2023-05-08 11:23:05,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +26: [2023-05-08 11:23:05,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +26: [2023-05-08 11:23:05,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +15: [2023-05-08 11:23:05,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +27: [2023-05-08 11:23:05,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +27: [2023-05-08 11:23:05,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +27: [2023-05-08 11:23:05,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +27: [2023-05-08 11:23:05,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +27: [2023-05-08 11:23:05,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +27: [2023-05-08 11:23:05,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +27: [2023-05-08 11:23:05,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +26: [2023-05-08 11:23:05,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +27: [2023-05-08 11:23:05,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +16: [2023-05-08 11:23:05,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 1: [2023-05-08 11:23:05,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +16: [2023-05-08 11:23:05,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 9: [2023-05-08 11:23:05,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +20: [2023-05-08 11:23:05,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +20: [2023-05-08 11:23:05,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 9: [2023-05-08 11:23:05,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +16: [2023-05-08 11:23:05,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 9: [2023-05-08 11:23:05,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +16: [2023-05-08 11:23:05,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +20: [2023-05-08 11:23:05,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +20: [2023-05-08 11:23:05,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +22: [2023-05-08 11:23:05,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:23:05,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +22: [2023-05-08 11:23:05,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:23:05,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +22: [2023-05-08 11:23:05,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:23:05,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +26: [2023-05-08 11:23:05,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +27: [2023-05-08 11:23:05,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +17: [2023-05-08 11:23:05,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +17: [2023-05-08 11:23:05,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +16: [2023-05-08 11:23:05,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +26: [2023-05-08 11:23:05,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +27: [2023-05-08 11:23:05,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +27: [2023-05-08 11:23:05,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +20: [2023-05-08 11:23:05,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:23:05,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:23:05,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +13: [2023-05-08 11:23:05,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +13: [2023-05-08 11:23:05,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +13: [2023-05-08 11:23:05,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +17: [2023-05-08 11:23:05,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +13: [2023-05-08 11:23:05,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +16: [2023-05-08 11:23:05,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +13: [2023-05-08 11:23:05,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +13: [2023-05-08 11:23:05,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +13: [2023-05-08 11:23:05,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +16: [2023-05-08 11:23:05,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt. +13: [2023-05-08 11:23:05,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +26: [2023-05-08 11:23:05,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +22: [2023-05-08 11:23:05,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt... +22: [2023-05-08 11:23:05,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +19: [2023-05-08 11:23:05,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +21: [2023-05-08 11:23:05,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +21: [2023-05-08 11:23:05,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +21: [2023-05-08 11:23:05,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +21: [2023-05-08 11:23:05,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +21: [2023-05-08 11:23:05,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +19: [2023-05-08 11:23:05,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +21: [2023-05-08 11:23:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +21: [2023-05-08 11:23:05,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +20: [2023-05-08 11:23:05,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:23:05,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +11: [2023-05-08 11:23:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +11: [2023-05-08 11:23:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +19: [2023-05-08 11:23:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +19: [2023-05-08 11:23:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +19: [2023-05-08 11:23:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +19: [2023-05-08 11:23:05,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +11: [2023-05-08 11:23:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +19: [2023-05-08 11:23:05,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +11: [2023-05-08 11:23:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +21: [2023-05-08 11:23:05,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +23: [2023-05-08 11:23:05,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +23: [2023-05-08 11:23:05,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +23: [2023-05-08 11:23:05,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +23: [2023-05-08 11:23:05,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +19: [2023-05-08 11:23:05,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +26: [2023-05-08 11:23:05,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +26: [2023-05-08 11:23:05,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +26: [2023-05-08 11:23:05,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +26: [2023-05-08 11:23:05,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +26: [2023-05-08 11:23:05,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +17: [2023-05-08 11:23:05,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +17: [2023-05-08 11:23:05,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +26: [2023-05-08 11:23:05,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +14: [2023-05-08 11:23:05,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +14: [2023-05-08 11:23:05,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +14: [2023-05-08 11:23:05,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +14: [2023-05-08 11:23:05,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +14: [2023-05-08 11:23:05,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +14: [2023-05-08 11:23:05,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +27: [2023-05-08 11:23:05,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +27: [2023-05-08 11:23:05,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +14: [2023-05-08 11:23:05,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +14: [2023-05-08 11:23:05,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +17: [2023-05-08 11:23:05,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +17: [2023-05-08 11:23:05,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +26: [2023-05-08 11:23:05,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +27: [2023-05-08 11:23:05,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +11: [2023-05-08 11:23:05,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +13: [2023-05-08 11:23:05,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +13: [2023-05-08 11:23:05,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +11: [2023-05-08 11:23:05,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +11: [2023-05-08 11:23:05,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +11: [2023-05-08 11:23:05,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:23:05,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +16: [2023-05-08 11:23:05,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +16: [2023-05-08 11:23:05,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +23: [2023-05-08 11:23:05,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:23:05,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +27: [2023-05-08 11:23:05,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +20: [2023-05-08 11:23:05,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 2: [2023-05-08 11:23:05,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +20: [2023-05-08 11:23:05,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +20: [2023-05-08 11:23:05,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 2: [2023-05-08 11:23:05,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +23: [2023-05-08 11:23:05,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:23:05,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +16: [2023-05-08 11:23:05,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +26: [2023-05-08 11:23:05,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +20: [2023-05-08 11:23:05,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +20: [2023-05-08 11:23:05,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +23: [2023-05-08 11:23:05,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +26: [2023-05-08 11:23:05,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +26: [2023-05-08 11:23:05,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +17: [2023-05-08 11:23:05,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +21: [2023-05-08 11:23:05,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +16: [2023-05-08 11:23:05,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +16: [2023-05-08 11:23:05,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +26: [2023-05-08 11:23:05,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:23:05,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +26: [2023-05-08 11:23:05,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +21: [2023-05-08 11:23:05,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +21: [2023-05-08 11:23:05,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +13: [2023-05-08 11:23:05,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +20: [2023-05-08 11:23:05,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +20: [2023-05-08 11:23:05,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +19: [2023-05-08 11:23:05,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +13: [2023-05-08 11:23:05,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +13: [2023-05-08 11:23:05,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +14: [2023-05-08 11:23:05,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +14: [2023-05-08 11:23:05,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +19: [2023-05-08 11:23:05,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +26: [2023-05-08 11:23:05,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:23:05,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +16: [2023-05-08 11:23:05,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:23:05,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:23:05,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +16: [2023-05-08 11:23:05,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +11: [2023-05-08 11:23:05,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +23: [2023-05-08 11:23:05,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +27: [2023-05-08 11:23:05,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +14: [2023-05-08 11:23:05,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +14: [2023-05-08 11:23:05,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +17: [2023-05-08 11:23:05,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:23:05,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +26: [2023-05-08 11:23:05,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +26: [2023-05-08 11:23:05,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:23:05,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +13: [2023-05-08 11:23:05,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +17: [2023-05-08 11:23:05,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +17: [2023-05-08 11:23:05,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +17: [2023-05-08 11:23:05,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +21: [2023-05-08 11:23:05,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +14: [2023-05-08 11:23:05,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +14: [2023-05-08 11:23:05,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +21: [2023-05-08 11:23:05,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +13: [2023-05-08 11:23:05,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +19: [2023-05-08 11:23:05,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +19: [2023-05-08 11:23:05,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +19: [2023-05-08 11:23:05,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +11: [2023-05-08 11:23:05,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:23:05,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +21: [2023-05-08 11:23:05,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +11: [2023-05-08 11:23:05,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +17: [2023-05-08 11:23:05,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +17: [2023-05-08 11:23:05,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +11: [2023-05-08 11:23:05,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 2: [2023-05-08 11:23:05,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +26: [2023-05-08 11:23:05,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +14: [2023-05-08 11:23:05,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +23: [2023-05-08 11:23:05,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:23:05,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +14: [2023-05-08 11:23:05,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +13: [2023-05-08 11:23:05,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +21: [2023-05-08 11:23:05,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +19: [2023-05-08 11:23:05,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +19: [2023-05-08 11:23:05,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +11: [2023-05-08 11:23:05,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:23:05,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +26: [2023-05-08 11:23:05,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:23:05,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +23: [2023-05-08 11:23:05,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:23:05,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:23:05,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +23: [2023-05-08 11:23:05,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +17: [2023-05-08 11:23:05,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +17: [2023-05-08 11:23:05,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +12: [2023-05-08 11:23:05,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +12: [2023-05-08 11:23:05,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +12: [2023-05-08 11:23:05,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +19: [2023-05-08 11:23:05,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +12: [2023-05-08 11:23:05,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 2: [2023-05-08 11:23:05,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 0: [2023-05-08 11:23:05,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 0: [2023-05-08 11:23:05,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 0: [2023-05-08 11:23:05,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 2: [2023-05-08 11:23:05,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +12: [2023-05-08 11:23:05,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:23:05,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:23:05,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:23:05,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +12: [2023-05-08 11:23:05,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:23:05,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:23:05,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +12: [2023-05-08 11:23:05,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +19: [2023-05-08 11:23:05,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +19: [2023-05-08 11:23:05,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +19: [2023-05-08 11:23:05,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +19: [2023-05-08 11:23:05,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +12: [2023-05-08 11:23:05,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:23:05,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 0: [2023-05-08 11:23:05,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 0: [2023-05-08 11:23:05,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 0: [2023-05-08 11:23:05,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +19: [2023-05-08 11:23:05,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +19: [2023-05-08 11:23:05,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +19: [2023-05-08 11:23:05,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +19: [2023-05-08 11:23:05,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:23:05,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:23:05,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +30: [2023-05-08 11:23:05,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +30: [2023-05-08 11:23:05,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +30: [2023-05-08 11:23:05,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +12: [2023-05-08 11:23:05,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +31: [2023-05-08 11:23:05,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +31: [2023-05-08 11:23:05,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +31: [2023-05-08 11:23:05,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +31: [2023-05-08 11:23:05,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +31: [2023-05-08 11:23:05,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +31: [2023-05-08 11:23:05,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +31: [2023-05-08 11:23:05,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +31: [2023-05-08 11:23:05,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +30: [2023-05-08 11:23:05,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:23:05,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:23:05,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +30: [2023-05-08 11:23:05,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:23:05,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:23:05,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +19: [2023-05-08 11:23:05,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:23:05,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:23:05,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:23:05,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:23:05,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +19: [2023-05-08 11:23:05,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +19: [2023-05-08 11:23:05,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +19: [2023-05-08 11:23:05,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +18: [2023-05-08 11:23:05,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +18: [2023-05-08 11:23:05,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +17: [2023-05-08 11:23:05,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +17: [2023-05-08 11:23:05,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +17: [2023-05-08 11:23:05,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +17: [2023-05-08 11:23:05,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +17: [2023-05-08 11:23:05,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +17: [2023-05-08 11:23:05,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +17: [2023-05-08 11:23:05,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +17: [2023-05-08 11:23:05,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +31: [2023-05-08 11:23:05,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +31: [2023-05-08 11:23:05,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +31: [2023-05-08 11:23:05,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +31: [2023-05-08 11:23:05,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +18: [2023-05-08 11:23:05,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:23:05,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:23:05,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:23:05,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +18: [2023-05-08 11:23:05,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:23:05,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +18: [2023-05-08 11:23:05,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:23:05,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +28: [2023-05-08 11:23:05,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +28: [2023-05-08 11:23:05,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +30: [2023-05-08 11:23:05,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:23:05,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 2: [2023-05-08 11:23:05,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 2: [2023-05-08 11:23:05,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 2: [2023-05-08 11:23:05,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +28: [2023-05-08 11:23:05,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +28: [2023-05-08 11:23:05,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +28: [2023-05-08 11:23:05,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +28: [2023-05-08 11:23:05,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +28: [2023-05-08 11:23:05,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +28: [2023-05-08 11:23:05,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +31: [2023-05-08 11:23:05,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +27: [2023-05-08 11:23:05,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +27: [2023-05-08 11:23:05,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +27: [2023-05-08 11:23:05,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +27: [2023-05-08 11:23:05,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +17: [2023-05-08 11:23:05,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +17: [2023-05-08 11:23:05,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +31: [2023-05-08 11:23:05,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +17: [2023-05-08 11:23:05,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +17: [2023-05-08 11:23:05,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +25: [2023-05-08 11:23:05,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +25: [2023-05-08 11:23:05,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +25: [2023-05-08 11:23:05,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +31: [2023-05-08 11:23:05,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:23:05,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +25: [2023-05-08 11:23:05,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +25: [2023-05-08 11:23:05,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +25: [2023-05-08 11:23:05,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +25: [2023-05-08 11:23:05,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +31: [2023-05-08 11:23:05,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +18: [2023-05-08 11:23:05,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +31: [2023-05-08 11:23:05,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:23:05,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 2: [2023-05-08 11:23:05,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:23:05,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +18: [2023-05-08 11:23:05,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +31: [2023-05-08 11:23:05,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +31: [2023-05-08 11:23:05,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +27: [2023-05-08 11:23:05,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:23:05,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +31: [2023-05-08 11:23:05,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +28: [2023-05-08 11:23:05,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +30: [2023-05-08 11:23:05,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:23:05,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +27: [2023-05-08 11:23:05,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +27: [2023-05-08 11:23:05,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:23:05,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:23:05,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +10: [2023-05-08 11:23:05,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +10: [2023-05-08 11:23:05,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +10: [2023-05-08 11:23:05,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +10: [2023-05-08 11:23:05,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +28: [2023-05-08 11:23:05,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +30: [2023-05-08 11:23:05,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +17: [2023-05-08 11:23:05,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +18: [2023-05-08 11:23:05,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +18: [2023-05-08 11:23:05,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +28: [2023-05-08 11:23:05,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 6: [2023-05-08 11:23:05,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +31: [2023-05-08 11:23:05,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +18: [2023-05-08 11:23:05,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +18: [2023-05-08 11:23:05,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 6: [2023-05-08 11:23:05,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +17: [2023-05-08 11:23:05,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +13: [2023-05-08 11:23:05,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +13: [2023-05-08 11:23:05,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +13: [2023-05-08 11:23:05,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +13: [2023-05-08 11:23:05,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +17: [2023-05-08 11:23:05,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 6: [2023-05-08 11:23:05,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +17: [2023-05-08 11:23:05,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +10: [2023-05-08 11:23:05,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +10: [2023-05-08 11:23:05,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +10: [2023-05-08 11:23:05,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +10: [2023-05-08 11:23:05,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +10: [2023-05-08 11:23:05,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +14: [2023-05-08 11:23:05,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 8: [2023-05-08 11:23:05,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +14: [2023-05-08 11:23:05,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +14: [2023-05-08 11:23:05,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +14: [2023-05-08 11:23:05,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 1: [2023-05-08 11:23:05,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +28: [2023-05-08 11:23:05,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +10: [2023-05-08 11:23:05,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +28: [2023-05-08 11:23:05,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +28: [2023-05-08 11:23:05,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +27: [2023-05-08 11:23:05,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +27: [2023-05-08 11:23:05,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +28: [2023-05-08 11:23:05,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +10: [2023-05-08 11:23:05,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +28: [2023-05-08 11:23:05,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 6: [2023-05-08 11:23:05,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +25: [2023-05-08 11:23:05,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +12: [2023-05-08 11:23:05,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +12: [2023-05-08 11:23:05,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +12: [2023-05-08 11:23:05,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +12: [2023-05-08 11:23:05,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +12: [2023-05-08 11:23:05,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +30: [2023-05-08 11:23:05,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +30: [2023-05-08 11:23:05,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +12: [2023-05-08 11:23:05,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:23:05,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +30: [2023-05-08 11:23:05,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +30: [2023-05-08 11:23:05,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +30: [2023-05-08 11:23:05,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +30: [2023-05-08 11:23:05,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +30: [2023-05-08 11:23:05,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 5: [2023-05-08 11:23:05,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 5: [2023-05-08 11:23:05,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 5: [2023-05-08 11:23:05,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +10: [2023-05-08 11:23:05,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +10: [2023-05-08 11:23:05,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:23:05,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +31: [2023-05-08 11:23:05,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +31: [2023-05-08 11:23:05,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +25: [2023-05-08 11:23:05,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +25: [2023-05-08 11:23:05,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +25: [2023-05-08 11:23:05,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +10: [2023-05-08 11:23:05,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:23:05,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 9: [2023-05-08 11:23:05,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +28: [2023-05-08 11:23:05,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:23:05,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:23:05,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +21: [2023-05-08 11:23:05,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +21: [2023-05-08 11:23:05,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 2: [2023-05-08 11:23:05,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 2: [2023-05-08 11:23:05,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +28: [2023-05-08 11:23:05,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:23:05,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +13: [2023-05-08 11:23:05,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:23:05,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +13: [2023-05-08 11:23:05,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 3: [2023-05-08 11:23:05,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 3: [2023-05-08 11:23:05,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +31: [2023-05-08 11:23:05,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +13: [2023-05-08 11:23:05,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:23:05,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 9: [2023-05-08 11:23:05,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +28: [2023-05-08 11:23:05,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +14: [2023-05-08 11:23:05,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +13: [2023-05-08 11:23:05,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +14: [2023-05-08 11:23:05,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +27: [2023-05-08 11:23:05,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:23:05,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +31: [2023-05-08 11:23:05,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:23:05,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +31: [2023-05-08 11:23:05,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 4: [2023-05-08 11:23:05,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 4: [2023-05-08 11:23:05,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 4: [2023-05-08 11:23:05,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +21: [2023-05-08 11:23:05,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 6: [2023-05-08 11:23:05,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +27: [2023-05-08 11:23:05,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:23:05,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 8: [2023-05-08 11:23:05,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 8: [2023-05-08 11:23:05,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +25: [2023-05-08 11:23:05,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 8: [2023-05-08 11:23:05,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 8: [2023-05-08 11:23:05,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +14: [2023-05-08 11:23:05,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +14: [2023-05-08 11:23:05,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +28: [2023-05-08 11:23:05,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:23:05,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +28: [2023-05-08 11:23:05,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:23:05,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:23:05,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +27: [2023-05-08 11:23:05,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:23:05,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +12: [2023-05-08 11:23:05,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:23:05,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:23:05,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +29: [2023-05-08 11:23:05,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +29: [2023-05-08 11:23:05,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +29: [2023-05-08 11:23:05,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +29: [2023-05-08 11:23:05,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +29: [2023-05-08 11:23:05,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +29: [2023-05-08 11:23:05,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +25: [2023-05-08 11:23:05,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +21: [2023-05-08 11:23:05,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +21: [2023-05-08 11:23:05,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +29: [2023-05-08 11:23:05,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +29: [2023-05-08 11:23:05,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +12: [2023-05-08 11:23:05,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +21: [2023-05-08 11:23:05,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +21: [2023-05-08 11:23:05,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:23:05,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +10: [2023-05-08 11:23:05,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +28: [2023-05-08 11:23:05,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +30: [2023-05-08 11:23:05,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +23: [2023-05-08 11:23:05,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +23: [2023-05-08 11:23:05,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +23: [2023-05-08 11:23:05,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +23: [2023-05-08 11:23:05,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +23: [2023-05-08 11:23:05,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +23: [2023-05-08 11:23:05,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +30: [2023-05-08 11:23:05,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +30: [2023-05-08 11:23:05,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +23: [2023-05-08 11:23:05,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +14: [2023-05-08 11:23:05,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +12: [2023-05-08 11:23:05,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +23: [2023-05-08 11:23:05,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +10: [2023-05-08 11:23:05,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:23:05,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +14: [2023-05-08 11:23:05,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 1: [2023-05-08 11:23:05,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +20: [2023-05-08 11:23:05,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +20: [2023-05-08 11:23:05,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +10: [2023-05-08 11:23:05,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +20: [2023-05-08 11:23:05,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +20: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +20: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +20: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +20: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +20: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +12: [2023-05-08 11:23:05,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +11: [2023-05-08 11:23:05,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +11: [2023-05-08 11:23:05,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +11: [2023-05-08 11:23:05,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +11: [2023-05-08 11:23:05,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +11: [2023-05-08 11:23:05,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +11: [2023-05-08 11:23:05,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +11: [2023-05-08 11:23:05,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +11: [2023-05-08 11:23:05,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +13: [2023-05-08 11:23:05,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +10: [2023-05-08 11:23:05,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:23:05,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:23:05,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +10: [2023-05-08 11:23:05,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +10: [2023-05-08 11:23:05,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +10: [2023-05-08 11:23:05,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 5: [2023-05-08 11:23:05,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +30: [2023-05-08 11:23:05,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +13: [2023-05-08 11:23:05,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 6: [2023-05-08 11:23:05,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 4: [2023-05-08 11:23:05,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:23:05,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +16: [2023-05-08 11:23:05,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +16: [2023-05-08 11:23:05,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +16: [2023-05-08 11:23:05,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +16: [2023-05-08 11:23:05,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +14: [2023-05-08 11:23:05,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +14: [2023-05-08 11:23:05,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +30: [2023-05-08 11:23:05,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +13: [2023-05-08 11:23:05,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +13: [2023-05-08 11:23:05,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +16: [2023-05-08 11:23:05,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +16: [2023-05-08 11:23:05,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +16: [2023-05-08 11:23:05,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +30: [2023-05-08 11:23:05,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +29: [2023-05-08 11:23:05,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +29: [2023-05-08 11:23:05,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +21: [2023-05-08 11:23:05,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +13: [2023-05-08 11:23:05,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 1: [2023-05-08 11:23:05,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +25: [2023-05-08 11:23:05,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +25: [2023-05-08 11:23:05,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +25: [2023-05-08 11:23:05,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +29: [2023-05-08 11:23:05,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +25: [2023-05-08 11:23:05,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +21: [2023-05-08 11:23:05,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +21: [2023-05-08 11:23:05,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +21: [2023-05-08 11:23:05,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +13: [2023-05-08 11:23:05,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +16: [2023-05-08 11:23:05,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +21: [2023-05-08 11:23:05,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +21: [2023-05-08 11:23:05,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 3: [2023-05-08 11:23:05,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:23:05,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +28: [2023-05-08 11:23:05,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:23:05,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +28: [2023-05-08 11:23:05,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +10: [2023-05-08 11:23:05,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +10: [2023-05-08 11:23:05,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +29: [2023-05-08 11:23:05,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 3: [2023-05-08 11:23:05,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 6: [2023-05-08 11:23:05,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 1: [2023-05-08 11:23:05,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 9: [2023-05-08 11:23:05,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:23:05,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:23:05,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:23:05,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:23:05,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +28: [2023-05-08 11:23:05,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 9: [2023-05-08 11:23:05,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +28: [2023-05-08 11:23:05,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +20: [2023-05-08 11:23:05,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +23: [2023-05-08 11:23:05,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +14: [2023-05-08 11:23:05,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +14: [2023-05-08 11:23:05,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 5: [2023-05-08 11:23:05,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 5: [2023-05-08 11:23:05,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 7: [2023-05-08 11:23:05,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +13: [2023-05-08 11:23:05,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 6: [2023-05-08 11:23:05,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +21: [2023-05-08 11:23:05,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:23:05,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +21: [2023-05-08 11:23:05,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +29: [2023-05-08 11:23:05,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 7: [2023-05-08 11:23:05,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +29: [2023-05-08 11:23:05,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 7: [2023-05-08 11:23:05,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +13: [2023-05-08 11:23:05,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 1: [2023-05-08 11:23:05,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. +29: [2023-05-08 11:23:05,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt. + 4: [2023-05-08 11:23:05,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +20: [2023-05-08 11:23:05,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +30: [2023-05-08 11:23:05,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +12: [2023-05-08 11:23:05,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +23: [2023-05-08 11:23:05,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +29: [2023-05-08 11:23:05,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +25: [2023-05-08 11:23:05,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:23:05,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:23:05,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 1: [2023-05-08 11:23:05,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:23:05,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +11: [2023-05-08 11:23:05,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +25: [2023-05-08 11:23:05,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +11: [2023-05-08 11:23:05,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +11: [2023-05-08 11:23:05,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +16: [2023-05-08 11:23:05,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +23: [2023-05-08 11:23:05,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +29: [2023-05-08 11:23:05,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 3: [2023-05-08 11:23:05,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +29: [2023-05-08 11:23:05,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +20: [2023-05-08 11:23:05,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +14: [2023-05-08 11:23:05,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:23:05,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +29: [2023-05-08 11:23:05,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +20: [2023-05-08 11:23:05,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +14: [2023-05-08 11:23:05,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +11: [2023-05-08 11:23:05,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +16: [2023-05-08 11:23:05,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +29: [2023-05-08 11:23:05,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:23:05,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +29: [2023-05-08 11:23:05,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +29: [2023-05-08 11:23:05,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:23:05,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +23: [2023-05-08 11:23:05,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +15: [2023-05-08 11:23:05,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +15: [2023-05-08 11:23:05,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +30: [2023-05-08 11:23:05,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +15: [2023-05-08 11:23:05,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +15: [2023-05-08 11:23:05,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +15: [2023-05-08 11:23:05,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +15: [2023-05-08 11:23:05,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +15: [2023-05-08 11:23:05,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +15: [2023-05-08 11:23:05,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +29: [2023-05-08 11:23:05,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 5: [2023-05-08 11:23:05,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +16: [2023-05-08 11:23:05,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +24: [2023-05-08 11:23:05,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +24: [2023-05-08 11:23:05,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +24: [2023-05-08 11:23:05,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +11: [2023-05-08 11:23:05,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +11: [2023-05-08 11:23:05,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +23: [2023-05-08 11:23:05,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +16: [2023-05-08 11:23:05,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +20: [2023-05-08 11:23:05,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +24: [2023-05-08 11:23:05,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +24: [2023-05-08 11:23:05,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +24: [2023-05-08 11:23:05,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +24: [2023-05-08 11:23:05,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +24: [2023-05-08 11:23:05,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +20: [2023-05-08 11:23:05,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 5: [2023-05-08 11:23:05,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +11: [2023-05-08 11:23:05,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +11: [2023-05-08 11:23:05,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +25: [2023-05-08 11:23:05,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:23:05,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +25: [2023-05-08 11:23:05,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:23:05,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 3: [2023-05-08 11:23:05,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +23: [2023-05-08 11:23:05,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +16: [2023-05-08 11:23:05,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +29: [2023-05-08 11:23:05,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:23:05,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 7: [2023-05-08 11:23:05,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +20: [2023-05-08 11:23:05,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 7: [2023-05-08 11:23:05,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 5: [2023-05-08 11:23:05,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +25: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 7: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +23: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +20: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +20: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +20: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +20: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +20: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +20: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +20: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +23: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +23: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +29: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +23: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +20: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +23: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +29: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +23: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +29: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +16: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +23: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +29: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +23: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +16: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +23: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +29: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +29: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +20: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +29: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +16: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +16: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +16: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +11: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +11: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +29: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +20: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +23: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +11: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +29: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +20: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +20: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +11: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +11: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +11: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +11: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +23: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +11: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +16: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +23: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +23: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +16: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +29: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +16: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +16: [2023-05-08 11:23:05,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +29: [2023-05-08 11:23:05,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +11: [2023-05-08 11:23:05,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +11: [2023-05-08 11:23:05,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +11: [2023-05-08 11:23:05,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +11: [2023-05-08 11:23:05,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +16: [2023-05-08 11:23:05,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +24: [2023-05-08 11:23:05,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +24: [2023-05-08 11:23:05,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 3: [2023-05-08 11:23:05,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +15: [2023-05-08 11:23:05,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +15: [2023-05-08 11:23:05,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +15: [2023-05-08 11:23:05,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +15: [2023-05-08 11:23:05,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +29: [2023-05-08 11:23:05,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:23:05,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:23:05,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +29: [2023-05-08 11:23:05,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... +16: [2023-05-08 11:23:05,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +24: [2023-05-08 11:23:05,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +25: [2023-05-08 11:23:05,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +25: [2023-05-08 11:23:05,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 4: [2023-05-08 11:23:05,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +24: [2023-05-08 11:23:05,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +24: [2023-05-08 11:23:05,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +15: [2023-05-08 11:23:05,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +15: [2023-05-08 11:23:05,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +15: [2023-05-08 11:23:05,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +15: [2023-05-08 11:23:05,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +16: [2023-05-08 11:23:05,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +16: [2023-05-08 11:23:05,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +16: [2023-05-08 11:23:05,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +16: [2023-05-08 11:23:05,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +24: [2023-05-08 11:23:05,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +24: [2023-05-08 11:23:05,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +24: [2023-05-08 11:23:05,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +24: [2023-05-08 11:23:05,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +24: [2023-05-08 11:23:05,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +24: [2023-05-08 11:23:05,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +24: [2023-05-08 11:23:05,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +24: [2023-05-08 11:23:05,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +24: [2023-05-08 11:23:05,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +15: [2023-05-08 11:23:05,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +15: [2023-05-08 11:23:05,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +15: [2023-05-08 11:23:05,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +15: [2023-05-08 11:23:05,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +15: [2023-05-08 11:23:05,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +15: [2023-05-08 11:23:05,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +15: [2023-05-08 11:23:05,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +24: [2023-05-08 11:23:05,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +15: [2023-05-08 11:23:05,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +15: [2023-05-08 11:23:05,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +15: [2023-05-08 11:23:05,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +15: [2023-05-08 11:23:05,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +15: [2023-05-08 11:23:05,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +29: [2023-05-08 11:23:05,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +24: [2023-05-08 11:23:05,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +29: [2023-05-08 11:23:05,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt... + 7: [2023-05-08 11:23:05,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 7: [2023-05-08 11:23:05,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +24: [2023-05-08 11:23:05,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +18: [2023-05-08 11:23:05,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +18: [2023-05-08 11:23:05,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +18: [2023-05-08 11:23:05,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +18: [2023-05-08 11:23:05,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +18: [2023-05-08 11:23:05,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +18: [2023-05-08 11:23:05,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +24: [2023-05-08 11:23:05,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +24: [2023-05-08 11:23:05,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +18: [2023-05-08 11:23:05,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +18: [2023-05-08 11:23:05,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +24: [2023-05-08 11:23:05,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt... + 5: [2023-05-08 11:23:05,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt... + 5: [2023-05-08 11:23:05,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt... + 5: [2023-05-08 11:23:05,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt... + 3: [2023-05-08 11:23:05,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt... + 3: [2023-05-08 11:23:05,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt... + 3: [2023-05-08 11:23:05,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt... + 3: [2023-05-08 11:23:05,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt... + 7: [2023-05-08 11:23:05,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt... + 4: [2023-05-08 11:23:05,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt... + 4: [2023-05-08 11:23:05,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt... + 4: [2023-05-08 11:23:05,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt... +29: [2023-05-08 11:23:05,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt... +29: [2023-05-08 11:23:05,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt... +29: [2023-05-08 11:23:05,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt... +29: [2023-05-08 11:23:05,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt... + 7: [2023-05-08 11:23:05,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +20: [2023-05-08 11:23:05,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt... +20: [2023-05-08 11:23:05,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt... +20: [2023-05-08 11:23:05,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt... +20: [2023-05-08 11:23:05,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt... + 9: [2023-05-08 11:23:05,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +18: [2023-05-08 11:23:05,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +18: [2023-05-08 11:23:05,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +11: [2023-05-08 11:23:05,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt... +23: [2023-05-08 11:23:05,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt... +23: [2023-05-08 11:23:05,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt... +11: [2023-05-08 11:23:05,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt... +23: [2023-05-08 11:23:05,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt... +23: [2023-05-08 11:23:05,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt... +11: [2023-05-08 11:23:05,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt... +11: [2023-05-08 11:23:05,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt... + 9: [2023-05-08 11:23:05,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +16: [2023-05-08 11:23:05,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt... +16: [2023-05-08 11:23:05,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt... +16: [2023-05-08 11:23:05,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt... +16: [2023-05-08 11:23:05,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt... +18: [2023-05-08 11:23:05,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +18: [2023-05-08 11:23:05,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +22: [2023-05-08 11:23:05,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +22: [2023-05-08 11:23:05,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +27: [2023-05-08 11:23:05,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +27: [2023-05-08 11:23:05,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +22: [2023-05-08 11:23:05,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 9: [2023-05-08 11:23:05,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +22: [2023-05-08 11:23:05,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 9: [2023-05-08 11:23:05,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt... + 7: [2023-05-08 11:23:05,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt... + 9: [2023-05-08 11:23:05,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt... + 7: [2023-05-08 11:23:05,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt... + 9: [2023-05-08 11:23:05,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +27: [2023-05-08 11:23:05,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +27: [2023-05-08 11:23:05,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +27: [2023-05-08 11:23:05,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +27: [2023-05-08 11:23:05,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +27: [2023-05-08 11:23:05,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +18: [2023-05-08 11:23:05,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +27: [2023-05-08 11:23:05,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +18: [2023-05-08 11:23:05,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +24: [2023-05-08 11:23:05,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt... +24: [2023-05-08 11:23:05,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt... +24: [2023-05-08 11:23:05,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt... +24: [2023-05-08 11:23:05,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt... + 9: [2023-05-08 11:23:05,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +22: [2023-05-08 11:23:05,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +22: [2023-05-08 11:23:05,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +22: [2023-05-08 11:23:05,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +22: [2023-05-08 11:23:05,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +18: [2023-05-08 11:23:05,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +18: [2023-05-08 11:23:05,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +18: [2023-05-08 11:23:05,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +18: [2023-05-08 11:23:05,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +22: [2023-05-08 11:23:05,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +18: [2023-05-08 11:23:05,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +18: [2023-05-08 11:23:05,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +22: [2023-05-08 11:23:05,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +22: [2023-05-08 11:23:05,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +15: [2023-05-08 11:23:05,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt... +15: [2023-05-08 11:23:05,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt... +15: [2023-05-08 11:23:05,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt... +15: [2023-05-08 11:23:05,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt... +22: [2023-05-08 11:23:05,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +22: [2023-05-08 11:23:05,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +22: [2023-05-08 11:23:05,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +18: [2023-05-08 11:23:05,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +18: [2023-05-08 11:23:05,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +22: [2023-05-08 11:23:05,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +22: [2023-05-08 11:23:05,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +18: [2023-05-08 11:23:05,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +18: [2023-05-08 11:23:05,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +18: [2023-05-08 11:23:05,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +18: [2023-05-08 11:23:05,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +18: [2023-05-08 11:23:05,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +18: [2023-05-08 11:23:05,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +27: [2023-05-08 11:23:05,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +27: [2023-05-08 11:23:05,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +22: [2023-05-08 11:23:05,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +27: [2023-05-08 11:23:05,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +22: [2023-05-08 11:23:05,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +27: [2023-05-08 11:23:05,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +22: [2023-05-08 11:23:05,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +27: [2023-05-08 11:23:05,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +16: [2023-05-08 11:23:05,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +16: [2023-05-08 11:23:05,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +22: [2023-05-08 11:23:05,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +22: [2023-05-08 11:23:05,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +22: [2023-05-08 11:23:05,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +27: [2023-05-08 11:23:05,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +27: [2023-05-08 11:23:05,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +16: [2023-05-08 11:23:05,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +16: [2023-05-08 11:23:05,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +27: [2023-05-08 11:23:05,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +27: [2023-05-08 11:23:05,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +27: [2023-05-08 11:23:05,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +27: [2023-05-08 11:23:05,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +27: [2023-05-08 11:23:05,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +27: [2023-05-08 11:23:05,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +27: [2023-05-08 11:23:05,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +27: [2023-05-08 11:23:05,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +16: [2023-05-08 11:23:05,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +22: [2023-05-08 11:23:05,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +27: [2023-05-08 11:23:05,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +22: [2023-05-08 11:23:05,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +16: [2023-05-08 11:23:05,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +22: [2023-05-08 11:23:05,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +16: [2023-05-08 11:23:05,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +16: [2023-05-08 11:23:05,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +22: [2023-05-08 11:23:05,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +22: [2023-05-08 11:23:05,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +22: [2023-05-08 11:23:05,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt... + 0: [2023-05-08 11:23:05,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt... + 9: [2023-05-08 11:23:05,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt... +22: [2023-05-08 11:23:05,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt... +22: [2023-05-08 11:23:05,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +22: [2023-05-08 11:23:05,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +22: [2023-05-08 11:23:05,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +22: [2023-05-08 11:23:05,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +26: [2023-05-08 11:23:05,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +26: [2023-05-08 11:23:05,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +26: [2023-05-08 11:23:05,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +22: [2023-05-08 11:23:05,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +26: [2023-05-08 11:23:05,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 0: [2023-05-08 11:23:05,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 0: > overriding learning rate value to 0.0002 + 0: > overriding minimum learning rate value to 2e-05 + 0: > overriding warmup iterations value to 0 + 0: > overriding total number of iterations value to 1 + 0: > overriding decay style value to cosine + 0: [2023-05-08 11:23:05,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +11: [2023-05-08 11:23:05,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +11: [2023-05-08 11:23:05,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +27: [2023-05-08 11:23:05,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +27: [2023-05-08 11:23:05,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +27: [2023-05-08 11:23:05,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +27: [2023-05-08 11:23:05,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +11: [2023-05-08 11:23:05,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +11: [2023-05-08 11:23:05,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +22: [2023-05-08 11:23:05,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +22: [2023-05-08 11:23:05,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +22: [2023-05-08 11:23:05,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +22: [2023-05-08 11:23:05,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +22: [2023-05-08 11:23:05,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +22: [2023-05-08 11:23:05,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +22: [2023-05-08 11:23:05,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +22: [2023-05-08 11:23:05,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +22: [2023-05-08 11:23:05,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +22: [2023-05-08 11:23:05,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +16: [2023-05-08 11:23:05,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +16: [2023-05-08 11:23:05,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +16: [2023-05-08 11:23:05,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +20: [2023-05-08 11:23:05,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +16: [2023-05-08 11:23:05,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +20: [2023-05-08 11:23:05,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +20: [2023-05-08 11:23:05,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +26: [2023-05-08 11:23:05,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +20: [2023-05-08 11:23:05,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +26: [2023-05-08 11:23:05,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +26: [2023-05-08 11:23:05,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +26: [2023-05-08 11:23:05,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +11: [2023-05-08 11:23:05,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +18: [2023-05-08 11:23:05,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt... +18: [2023-05-08 11:23:05,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt... +18: [2023-05-08 11:23:05,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt... +18: [2023-05-08 11:23:05,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt... +11: [2023-05-08 11:23:05,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +11: [2023-05-08 11:23:05,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +11: [2023-05-08 11:23:05,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +14: [2023-05-08 11:23:05,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +14: [2023-05-08 11:23:05,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +14: [2023-05-08 11:23:05,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +14: [2023-05-08 11:23:05,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +16: [2023-05-08 11:23:05,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +16: [2023-05-08 11:23:05,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +14: [2023-05-08 11:23:05,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +14: [2023-05-08 11:23:05,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +14: [2023-05-08 11:23:05,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +14: [2023-05-08 11:23:05,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +20: [2023-05-08 11:23:05,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +21: [2023-05-08 11:23:05,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +20: [2023-05-08 11:23:05,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +21: [2023-05-08 11:23:05,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +21: [2023-05-08 11:23:05,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +20: [2023-05-08 11:23:05,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +21: [2023-05-08 11:23:05,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +21: [2023-05-08 11:23:05,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +21: [2023-05-08 11:23:05,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +21: [2023-05-08 11:23:05,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +20: [2023-05-08 11:23:05,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +21: [2023-05-08 11:23:05,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +16: [2023-05-08 11:23:05,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +24: [2023-05-08 11:23:05,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +24: [2023-05-08 11:23:05,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +24: [2023-05-08 11:23:05,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +24: [2023-05-08 11:23:05,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 2: [2023-05-08 11:23:05,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +16: [2023-05-08 11:23:05,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +15: [2023-05-08 11:23:05,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +26: [2023-05-08 11:23:05,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +26: [2023-05-08 11:23:05,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +27: [2023-05-08 11:23:05,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt... +27: [2023-05-08 11:23:05,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt... +27: [2023-05-08 11:23:05,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt... +27: [2023-05-08 11:23:05,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt... +26: [2023-05-08 11:23:05,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +26: [2023-05-08 11:23:05,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +15: [2023-05-08 11:23:05,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +26: [2023-05-08 11:23:05,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +26: [2023-05-08 11:23:05,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +26: [2023-05-08 11:23:05,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +26: [2023-05-08 11:23:05,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +26: [2023-05-08 11:23:05,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +26: [2023-05-08 11:23:05,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +26: [2023-05-08 11:23:05,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +15: [2023-05-08 11:23:05,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +26: [2023-05-08 11:23:05,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +15: [2023-05-08 11:23:05,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +13: [2023-05-08 11:23:05,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +13: [2023-05-08 11:23:05,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +13: [2023-05-08 11:23:05,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +13: [2023-05-08 11:23:05,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +13: [2023-05-08 11:23:05,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +13: [2023-05-08 11:23:05,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +14: [2023-05-08 11:23:05,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +13: [2023-05-08 11:23:05,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +13: [2023-05-08 11:23:05,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +11: [2023-05-08 11:23:05,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +11: [2023-05-08 11:23:05,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +14: [2023-05-08 11:23:05,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +24: [2023-05-08 11:23:05,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +24: [2023-05-08 11:23:05,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +21: [2023-05-08 11:23:05,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +21: [2023-05-08 11:23:05,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +24: [2023-05-08 11:23:05,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +14: [2023-05-08 11:23:05,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +24: [2023-05-08 11:23:05,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +15: [2023-05-08 11:23:05,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +20: [2023-05-08 11:23:05,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +11: [2023-05-08 11:23:05,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +20: [2023-05-08 11:23:05,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +21: [2023-05-08 11:23:05,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +11: [2023-05-08 11:23:05,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +26: [2023-05-08 11:23:05,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +15: [2023-05-08 11:23:05,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +11: [2023-05-08 11:23:05,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +11: [2023-05-08 11:23:05,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +26: [2023-05-08 11:23:05,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +20: [2023-05-08 11:23:05,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +20: [2023-05-08 11:23:05,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +14: [2023-05-08 11:23:05,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +15: [2023-05-08 11:23:05,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +15: [2023-05-08 11:23:05,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +19: [2023-05-08 11:23:05,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +19: [2023-05-08 11:23:05,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +19: [2023-05-08 11:23:05,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +19: [2023-05-08 11:23:05,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +19: [2023-05-08 11:23:05,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +19: [2023-05-08 11:23:05,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +19: [2023-05-08 11:23:05,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +19: [2023-05-08 11:23:05,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... + 0: [2023-05-08 11:23:05,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... + 0: [2023-05-08 11:23:05,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... + 0: [2023-05-08 11:23:05,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... +28: [2023-05-08 11:23:05,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +28: [2023-05-08 11:23:05,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +28: [2023-05-08 11:23:05,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +28: [2023-05-08 11:23:05,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +28: [2023-05-08 11:23:05,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +28: [2023-05-08 11:23:05,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +17: [2023-05-08 11:23:05,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +17: [2023-05-08 11:23:05,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +14: [2023-05-08 11:23:05,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +17: [2023-05-08 11:23:05,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +17: [2023-05-08 11:23:05,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 2: [2023-05-08 11:23:05,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +28: [2023-05-08 11:23:05,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +12: [2023-05-08 11:23:05,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +12: [2023-05-08 11:23:05,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +28: [2023-05-08 11:23:05,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +14: [2023-05-08 11:23:05,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +23: [2023-05-08 11:23:05,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +14: [2023-05-08 11:23:05,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +14: [2023-05-08 11:23:05,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +14: [2023-05-08 11:23:05,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +23: [2023-05-08 11:23:05,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +21: [2023-05-08 11:23:05,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +23: [2023-05-08 11:23:05,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +12: [2023-05-08 11:23:05,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +23: [2023-05-08 11:23:05,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +12: [2023-05-08 11:23:05,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +14: [2023-05-08 11:23:05,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +26: [2023-05-08 11:23:05,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +13: [2023-05-08 11:23:05,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +13: [2023-05-08 11:23:05,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +14: [2023-05-08 11:23:05,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +22: [2023-05-08 11:23:05,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt... +22: [2023-05-08 11:23:05,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt... +22: [2023-05-08 11:23:05,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt... +22: [2023-05-08 11:23:05,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt... +19: [2023-05-08 11:23:05,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +19: [2023-05-08 11:23:05,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +14: [2023-05-08 11:23:05,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +14: [2023-05-08 11:23:05,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +14: [2023-05-08 11:23:05,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +13: [2023-05-08 11:23:05,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +19: [2023-05-08 11:23:05,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +13: [2023-05-08 11:23:05,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +19: [2023-05-08 11:23:05,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +14: [2023-05-08 11:23:05,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +21: [2023-05-08 11:23:05,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +26: [2023-05-08 11:23:05,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +21: [2023-05-08 11:23:05,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +21: [2023-05-08 11:23:05,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +21: [2023-05-08 11:23:05,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +21: [2023-05-08 11:23:05,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +21: [2023-05-08 11:23:05,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +21: [2023-05-08 11:23:05,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +21: [2023-05-08 11:23:05,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +21: [2023-05-08 11:23:05,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +21: [2023-05-08 11:23:05,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +21: [2023-05-08 11:23:05,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +20: [2023-05-08 11:23:05,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +11: [2023-05-08 11:23:05,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +24: [2023-05-08 11:23:05,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +24: [2023-05-08 11:23:05,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +24: [2023-05-08 11:23:05,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +24: [2023-05-08 11:23:05,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +20: [2023-05-08 11:23:05,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +11: [2023-05-08 11:23:05,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +20: [2023-05-08 11:23:05,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +15: [2023-05-08 11:23:05,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +14: [2023-05-08 11:23:05,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +15: [2023-05-08 11:23:05,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +17: [2023-05-08 11:23:05,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +20: [2023-05-08 11:23:05,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +26: [2023-05-08 11:23:05,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +26: [2023-05-08 11:23:05,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +12: [2023-05-08 11:23:05,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +23: [2023-05-08 11:23:05,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +26: [2023-05-08 11:23:05,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +15: [2023-05-08 11:23:05,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +12: [2023-05-08 11:23:05,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +12: [2023-05-08 11:23:05,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +15: [2023-05-08 11:23:05,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +17: [2023-05-08 11:23:05,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +13: [2023-05-08 11:23:05,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +28: [2023-05-08 11:23:05,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +12: [2023-05-08 11:23:05,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +15: [2023-05-08 11:23:05,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +15: [2023-05-08 11:23:05,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +19: [2023-05-08 11:23:05,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +13: [2023-05-08 11:23:05,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +15: [2023-05-08 11:23:05,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +23: [2023-05-08 11:23:05,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +15: [2023-05-08 11:23:05,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +19: [2023-05-08 11:23:05,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +17: [2023-05-08 11:23:05,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +13: [2023-05-08 11:23:05,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +13: [2023-05-08 11:23:05,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +28: [2023-05-08 11:23:05,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +17: [2023-05-08 11:23:05,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +19: [2023-05-08 11:23:05,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +19: [2023-05-08 11:23:05,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +19: [2023-05-08 11:23:05,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +23: [2023-05-08 11:23:05,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +23: [2023-05-08 11:23:05,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +21: [2023-05-08 11:23:05,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +24: [2023-05-08 11:23:05,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +24: [2023-05-08 11:23:05,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +13: [2023-05-08 11:23:05,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +13: [2023-05-08 11:23:05,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +13: [2023-05-08 11:23:05,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +13: [2023-05-08 11:23:05,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +13: [2023-05-08 11:23:05,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +13: [2023-05-08 11:23:05,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +13: [2023-05-08 11:23:05,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +13: [2023-05-08 11:23:05,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +14: [2023-05-08 11:23:05,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +13: [2023-05-08 11:23:05,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +13: [2023-05-08 11:23:05,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +13: [2023-05-08 11:23:05,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +13: [2023-05-08 11:23:05,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +14: [2023-05-08 11:23:05,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +14: [2023-05-08 11:23:05,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +14: [2023-05-08 11:23:05,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +26: [2023-05-08 11:23:05,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +26: [2023-05-08 11:23:05,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +24: [2023-05-08 11:23:05,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +30: [2023-05-08 11:23:05,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +30: [2023-05-08 11:23:05,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +30: [2023-05-08 11:23:05,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +26: [2023-05-08 11:23:05,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +30: [2023-05-08 11:23:05,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +19: [2023-05-08 11:23:05,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +12: [2023-05-08 11:23:05,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +26: [2023-05-08 11:23:05,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +26: [2023-05-08 11:23:05,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +12: [2023-05-08 11:23:05,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +28: [2023-05-08 11:23:05,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +26: [2023-05-08 11:23:05,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +26: [2023-05-08 11:23:05,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +23: [2023-05-08 11:23:05,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +26: [2023-05-08 11:23:05,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +28: [2023-05-08 11:23:05,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +26: [2023-05-08 11:23:05,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +26: [2023-05-08 11:23:05,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +26: [2023-05-08 11:23:05,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +26: [2023-05-08 11:23:05,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +24: [2023-05-08 11:23:05,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 2: [2023-05-08 11:23:05,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +23: [2023-05-08 11:23:05,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +21: [2023-05-08 11:23:05,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +12: [2023-05-08 11:23:05,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +28: [2023-05-08 11:23:05,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +12: [2023-05-08 11:23:05,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +28: [2023-05-08 11:23:05,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +17: [2023-05-08 11:23:05,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +28: [2023-05-08 11:23:05,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +28: [2023-05-08 11:23:05,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +21: [2023-05-08 11:23:05,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +21: [2023-05-08 11:23:05,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +12: [2023-05-08 11:23:05,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +12: [2023-05-08 11:23:05,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +28: [2023-05-08 11:23:05,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +17: [2023-05-08 11:23:05,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +21: [2023-05-08 11:23:05,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +19: [2023-05-08 11:23:05,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +26: [2023-05-08 11:23:05,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +19: [2023-05-08 11:23:05,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +28: [2023-05-08 11:23:05,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +19: [2023-05-08 11:23:05,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +28: [2023-05-08 11:23:05,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +19: [2023-05-08 11:23:05,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +19: [2023-05-08 11:23:05,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +19: [2023-05-08 11:23:05,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +19: [2023-05-08 11:23:05,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +28: [2023-05-08 11:23:05,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +19: [2023-05-08 11:23:05,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 6: [2023-05-08 11:23:05,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +23: [2023-05-08 11:23:05,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +23: [2023-05-08 11:23:05,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 6: [2023-05-08 11:23:05,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +25: [2023-05-08 11:23:05,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +25: [2023-05-08 11:23:05,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +25: [2023-05-08 11:23:05,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +25: [2023-05-08 11:23:05,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +10: [2023-05-08 11:23:05,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +10: [2023-05-08 11:23:05,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +10: [2023-05-08 11:23:05,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +10: [2023-05-08 11:23:05,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +30: [2023-05-08 11:23:05,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +10: [2023-05-08 11:23:05,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +10: [2023-05-08 11:23:05,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +10: [2023-05-08 11:23:05,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +31: [2023-05-08 11:23:05,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +31: [2023-05-08 11:23:05,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +30: [2023-05-08 11:23:05,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +30: [2023-05-08 11:23:05,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +25: [2023-05-08 11:23:05,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +25: [2023-05-08 11:23:05,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +25: [2023-05-08 11:23:05,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +10: [2023-05-08 11:23:05,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +17: [2023-05-08 11:23:05,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +17: [2023-05-08 11:23:05,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +31: [2023-05-08 11:23:05,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +30: [2023-05-08 11:23:05,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +26: [2023-05-08 11:23:05,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +25: [2023-05-08 11:23:05,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +26: [2023-05-08 11:23:05,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +26: [2023-05-08 11:23:05,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +31: [2023-05-08 11:23:05,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +31: [2023-05-08 11:23:05,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +26: [2023-05-08 11:23:05,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +31: [2023-05-08 11:23:05,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +31: [2023-05-08 11:23:05,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +23: [2023-05-08 11:23:05,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +23: [2023-05-08 11:23:05,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +28: [2023-05-08 11:23:05,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +19: [2023-05-08 11:23:05,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +28: [2023-05-08 11:23:05,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +28: [2023-05-08 11:23:05,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +31: [2023-05-08 11:23:05,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +19: [2023-05-08 11:23:05,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +19: [2023-05-08 11:23:05,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +28: [2023-05-08 11:23:05,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +19: [2023-05-08 11:23:05,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +19: [2023-05-08 11:23:05,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +19: [2023-05-08 11:23:05,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +30: [2023-05-08 11:23:05,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +30: [2023-05-08 11:23:05,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +30: [2023-05-08 11:23:05,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +30: [2023-05-08 11:23:05,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +30: [2023-05-08 11:23:05,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +19: [2023-05-08 11:23:05,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +19: [2023-05-08 11:23:05,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +19: [2023-05-08 11:23:05,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +19: [2023-05-08 11:23:05,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +30: [2023-05-08 11:23:05,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +30: [2023-05-08 11:23:05,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +19: [2023-05-08 11:23:05,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +19: [2023-05-08 11:23:05,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +23: [2023-05-08 11:23:05,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +12: [2023-05-08 11:23:05,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +28: [2023-05-08 11:23:05,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +28: [2023-05-08 11:23:05,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +28: [2023-05-08 11:23:05,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +23: [2023-05-08 11:23:05,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +17: [2023-05-08 11:23:05,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +28: [2023-05-08 11:23:05,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +30: [2023-05-08 11:23:05,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +12: [2023-05-08 11:23:05,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 6: [2023-05-08 11:23:05,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +19: [2023-05-08 11:23:05,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +19: [2023-05-08 11:23:05,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 6: [2023-05-08 11:23:05,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +19: [2023-05-08 11:23:05,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +17: [2023-05-08 11:23:05,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 6: [2023-05-08 11:23:05,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +19: [2023-05-08 11:23:05,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 6: [2023-05-08 11:23:05,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +10: [2023-05-08 11:23:05,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +17: [2023-05-08 11:23:05,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +25: [2023-05-08 11:23:05,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +19: [2023-05-08 11:23:05,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +19: [2023-05-08 11:23:05,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +17: [2023-05-08 11:23:05,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 4: [2023-05-08 11:23:05,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 4: [2023-05-08 11:23:05,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +31: [2023-05-08 11:23:05,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +31: [2023-05-08 11:23:05,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +10: [2023-05-08 11:23:05,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +10: [2023-05-08 11:23:05,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 8: [2023-05-08 11:23:05,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +25: [2023-05-08 11:23:05,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +30: [2023-05-08 11:23:05,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +30: [2023-05-08 11:23:05,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 6: [2023-05-08 11:23:05,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +25: [2023-05-08 11:23:05,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 6: [2023-05-08 11:23:05,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +21: [2023-05-08 11:23:05,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt... +21: [2023-05-08 11:23:05,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt... +21: [2023-05-08 11:23:05,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt... +21: [2023-05-08 11:23:05,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt... + 6: [2023-05-08 11:23:05,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +30: [2023-05-08 11:23:05,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +30: [2023-05-08 11:23:05,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +10: [2023-05-08 11:23:05,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +17: [2023-05-08 11:23:05,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +17: [2023-05-08 11:23:05,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +17: [2023-05-08 11:23:05,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +17: [2023-05-08 11:23:05,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +17: [2023-05-08 11:23:05,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +17: [2023-05-08 11:23:05,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +12: [2023-05-08 11:23:05,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +12: [2023-05-08 11:23:05,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +12: [2023-05-08 11:23:05,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +12: [2023-05-08 11:23:05,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt... + 2: [2023-05-08 11:23:05,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt... + 2: [2023-05-08 11:23:05,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt... +12: [2023-05-08 11:23:05,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt... +12: [2023-05-08 11:23:05,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +17: [2023-05-08 11:23:05,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +12: [2023-05-08 11:23:05,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +12: [2023-05-08 11:23:05,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +10: [2023-05-08 11:23:05,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +10: [2023-05-08 11:23:05,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +31: [2023-05-08 11:23:05,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +10: [2023-05-08 11:23:05,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +25: [2023-05-08 11:23:05,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +17: [2023-05-08 11:23:05,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +14: [2023-05-08 11:23:05,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt... +14: [2023-05-08 11:23:05,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt... +14: [2023-05-08 11:23:05,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt... +14: [2023-05-08 11:23:05,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt... + 4: [2023-05-08 11:23:05,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +30: [2023-05-08 11:23:05,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +30: [2023-05-08 11:23:05,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +30: [2023-05-08 11:23:05,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 4: [2023-05-08 11:23:05,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +31: [2023-05-08 11:23:05,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +31: [2023-05-08 11:23:05,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +31: [2023-05-08 11:23:05,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +31: [2023-05-08 11:23:05,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +31: [2023-05-08 11:23:05,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +31: [2023-05-08 11:23:05,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +31: [2023-05-08 11:23:05,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +13: [2023-05-08 11:23:05,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt... +13: [2023-05-08 11:23:05,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt... +13: [2023-05-08 11:23:05,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt... +13: [2023-05-08 11:23:05,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt... +25: [2023-05-08 11:23:05,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +25: [2023-05-08 11:23:05,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +30: [2023-05-08 11:23:05,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +25: [2023-05-08 11:23:05,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +30: [2023-05-08 11:23:05,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +25: [2023-05-08 11:23:05,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +25: [2023-05-08 11:23:05,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +25: [2023-05-08 11:23:05,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +25: [2023-05-08 11:23:05,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +25: [2023-05-08 11:23:05,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +10: [2023-05-08 11:23:05,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +25: [2023-05-08 11:23:05,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +25: [2023-05-08 11:23:05,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +25: [2023-05-08 11:23:05,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +10: [2023-05-08 11:23:05,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +10: [2023-05-08 11:23:05,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +10: [2023-05-08 11:23:05,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +30: [2023-05-08 11:23:05,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +12: [2023-05-08 11:23:05,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +12: [2023-05-08 11:23:05,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +30: [2023-05-08 11:23:05,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 1: [2023-05-08 11:23:05,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +10: [2023-05-08 11:23:05,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 3: [2023-05-08 11:23:05,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +10: [2023-05-08 11:23:05,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +10: [2023-05-08 11:23:05,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +10: [2023-05-08 11:23:05,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +31: [2023-05-08 11:23:05,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +17: [2023-05-08 11:23:05,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +30: [2023-05-08 11:23:05,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +17: [2023-05-08 11:23:05,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +17: [2023-05-08 11:23:05,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +26: [2023-05-08 11:23:05,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt... +26: [2023-05-08 11:23:05,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt... +26: [2023-05-08 11:23:05,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt... +26: [2023-05-08 11:23:05,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt... +12: [2023-05-08 11:23:05,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +12: [2023-05-08 11:23:05,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +25: [2023-05-08 11:23:05,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 0: [2023-05-08 11:23:05,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 0: [2023-05-08 11:23:05,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +19: [2023-05-08 11:23:05,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt... +19: [2023-05-08 11:23:05,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt... +19: [2023-05-08 11:23:05,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt... +19: [2023-05-08 11:23:05,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt... + 8: [2023-05-08 11:23:05,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 8: [2023-05-08 11:23:05,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +31: [2023-05-08 11:23:05,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +10: [2023-05-08 11:23:05,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +18: [2023-05-08 11:23:05,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +18: [2023-05-08 11:23:05,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +18: [2023-05-08 11:23:05,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +18: [2023-05-08 11:23:05,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +25: [2023-05-08 11:23:05,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +25: [2023-05-08 11:23:05,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 4: [2023-05-08 11:23:05,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +25: [2023-05-08 11:23:05,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +28: [2023-05-08 11:23:05,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt... +28: [2023-05-08 11:23:05,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt... +28: [2023-05-08 11:23:05,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt... +28: [2023-05-08 11:23:05,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt... +25: [2023-05-08 11:23:05,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 6: [2023-05-08 11:23:05,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +30: [2023-05-08 11:23:05,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +30: [2023-05-08 11:23:05,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +30: [2023-05-08 11:23:05,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +31: [2023-05-08 11:23:05,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +30: [2023-05-08 11:23:05,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +31: [2023-05-08 11:23:05,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +31: [2023-05-08 11:23:05,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +30: [2023-05-08 11:23:05,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +30: [2023-05-08 11:23:05,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +30: [2023-05-08 11:23:05,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +30: [2023-05-08 11:23:05,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +30: [2023-05-08 11:23:05,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +31: [2023-05-08 11:23:05,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +30: [2023-05-08 11:23:05,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +12: [2023-05-08 11:23:05,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +30: [2023-05-08 11:23:05,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +12: [2023-05-08 11:23:05,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 4: [2023-05-08 11:23:05,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +17: [2023-05-08 11:23:05,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 3: [2023-05-08 11:23:05,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +12: [2023-05-08 11:23:05,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +17: [2023-05-08 11:23:05,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +12: [2023-05-08 11:23:05,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +30: [2023-05-08 11:23:05,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +25: [2023-05-08 11:23:05,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +17: [2023-05-08 11:23:05,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +25: [2023-05-08 11:23:05,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +25: [2023-05-08 11:23:05,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +17: [2023-05-08 11:23:05,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +17: [2023-05-08 11:23:05,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +17: [2023-05-08 11:23:05,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +17: [2023-05-08 11:23:05,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +12: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +12: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +25: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +12: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +12: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +12: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +17: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +12: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +17: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +12: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +31: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +17: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +17: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +12: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +12: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +12: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 8: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +12: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 6: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +17: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +31: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +10: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +17: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +31: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 4: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 4: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +12: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +31: [2023-05-08 11:23:05,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +10: [2023-05-08 11:23:05,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +10: [2023-05-08 11:23:05,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +18: [2023-05-08 11:23:05,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +10: [2023-05-08 11:23:05,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +18: [2023-05-08 11:23:05,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +25: [2023-05-08 11:23:05,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +25: [2023-05-08 11:23:05,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +18: [2023-05-08 11:23:05,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +25: [2023-05-08 11:23:05,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +25: [2023-05-08 11:23:05,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +18: [2023-05-08 11:23:05,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 7: [2023-05-08 11:23:05,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 7: [2023-05-08 11:23:05,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +30: [2023-05-08 11:23:05,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +30: [2023-05-08 11:23:05,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +30: [2023-05-08 11:23:05,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 8: [2023-05-08 11:23:05,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +30: [2023-05-08 11:23:05,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 7: [2023-05-08 11:23:05,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +17: [2023-05-08 11:23:05,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +17: [2023-05-08 11:23:05,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +17: [2023-05-08 11:23:05,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +17: [2023-05-08 11:23:05,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 6: [2023-05-08 11:23:05,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt... + 6: [2023-05-08 11:23:05,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt... + 6: [2023-05-08 11:23:05,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt... + 6: [2023-05-08 11:23:05,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt... + 1: [2023-05-08 11:23:05,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 0: [2023-05-08 11:23:05,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +31: [2023-05-08 11:23:05,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +31: [2023-05-08 11:23:05,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +31: [2023-05-08 11:23:05,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +31: [2023-05-08 11:23:05,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 3: [2023-05-08 11:23:05,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 3: [2023-05-08 11:23:05,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +18: [2023-05-08 11:23:05,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +18: [2023-05-08 11:23:05,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +25: [2023-05-08 11:23:05,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +25: [2023-05-08 11:23:05,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 0: [2023-05-08 11:23:05,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 8: [2023-05-08 11:23:05,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +25: [2023-05-08 11:23:05,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +25: [2023-05-08 11:23:05,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 1: [2023-05-08 11:23:05,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +18: [2023-05-08 11:23:05,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +18: [2023-05-08 11:23:05,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +25: [2023-05-08 11:23:05,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +25: [2023-05-08 11:23:05,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +25: [2023-05-08 11:23:05,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +25: [2023-05-08 11:23:05,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 5: [2023-05-08 11:23:05,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 5: [2023-05-08 11:23:05,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 5: [2023-05-08 11:23:05,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 5: [2023-05-08 11:23:05,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +31: [2023-05-08 11:23:05,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +31: [2023-05-08 11:23:05,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +10: [2023-05-08 11:23:05,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt... +31: [2023-05-08 11:23:05,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +10: [2023-05-08 11:23:05,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt... +10: [2023-05-08 11:23:05,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt... +10: [2023-05-08 11:23:05,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt... + 3: [2023-05-08 11:23:05,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +18: [2023-05-08 11:23:05,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +13: [2023-05-08 11:23:05,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +13: [2023-05-08 11:23:05,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +31: [2023-05-08 11:23:05,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +18: [2023-05-08 11:23:05,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 3: [2023-05-08 11:23:05,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +13: [2023-05-08 11:23:05,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +13: [2023-05-08 11:23:05,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +31: [2023-05-08 11:23:05,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt... +31: [2023-05-08 11:23:05,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt... +31: [2023-05-08 11:23:05,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt... +31: [2023-05-08 11:23:05,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt... + 8: [2023-05-08 11:23:05,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 1: [2023-05-08 11:23:05,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +18: [2023-05-08 11:23:05,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +25: [2023-05-08 11:23:05,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt... +25: [2023-05-08 11:23:05,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt... +25: [2023-05-08 11:23:05,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt... +25: [2023-05-08 11:23:05,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt... + 8: [2023-05-08 11:23:05,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt... + 8: [2023-05-08 11:23:05,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt... + 8: [2023-05-08 11:23:05,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt... + 8: [2023-05-08 11:23:05,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt... + 5: [2023-05-08 11:23:05,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 0: [2023-05-08 11:23:05,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 0: [2023-05-08 11:23:05,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +27: [2023-05-08 11:23:05,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +27: [2023-05-08 11:23:05,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 8: [2023-05-08 11:23:05,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +27: [2023-05-08 11:23:05,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +27: [2023-05-08 11:23:05,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +18: [2023-05-08 11:23:05,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 7: [2023-05-08 11:23:05,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 5: [2023-05-08 11:23:05,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +17: [2023-05-08 11:23:05,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt... +17: [2023-05-08 11:23:05,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt... +17: [2023-05-08 11:23:05,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt... +17: [2023-05-08 11:23:05,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt... +14: [2023-05-08 11:23:05,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +14: [2023-05-08 11:23:05,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +14: [2023-05-08 11:23:05,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +30: [2023-05-08 11:23:05,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt... +14: [2023-05-08 11:23:05,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +30: [2023-05-08 11:23:05,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt... +30: [2023-05-08 11:23:05,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt... +21: [2023-05-08 11:23:05,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +21: [2023-05-08 11:23:05,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +21: [2023-05-08 11:23:05,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +30: [2023-05-08 11:23:05,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt... + 0: [2023-05-08 11:23:05,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 0: [2023-05-08 11:23:05,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 0: [2023-05-08 11:23:05,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +21: [2023-05-08 11:23:05,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 7: [2023-05-08 11:23:05,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +13: [2023-05-08 11:23:05,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 8: [2023-05-08 11:23:05,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 8: [2023-05-08 11:23:05,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +13: [2023-05-08 11:23:05,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +28: [2023-05-08 11:23:05,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +13: [2023-05-08 11:23:05,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +13: [2023-05-08 11:23:05,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +28: [2023-05-08 11:23:05,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +28: [2023-05-08 11:23:05,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +28: [2023-05-08 11:23:05,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +29: [2023-05-08 11:23:05,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +29: [2023-05-08 11:23:05,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +29: [2023-05-08 11:23:05,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +12: [2023-05-08 11:23:05,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt... +12: [2023-05-08 11:23:05,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt... +12: [2023-05-08 11:23:05,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt... +12: [2023-05-08 11:23:05,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt... +31: [2023-05-08 11:23:05,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +29: [2023-05-08 11:23:05,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +31: [2023-05-08 11:23:05,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 1: [2023-05-08 11:23:05,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 1: [2023-05-08 11:23:05,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 1: [2023-05-08 11:23:05,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 1: [2023-05-08 11:23:05,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 7: [2023-05-08 11:23:05,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +27: [2023-05-08 11:23:05,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +27: [2023-05-08 11:23:05,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +14: [2023-05-08 11:23:05,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +21: [2023-05-08 11:23:05,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +21: [2023-05-08 11:23:05,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +21: [2023-05-08 11:23:05,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +14: [2023-05-08 11:23:05,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +14: [2023-05-08 11:23:05,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +27: [2023-05-08 11:23:05,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +27: [2023-05-08 11:23:05,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 8: [2023-05-08 11:23:05,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +31: [2023-05-08 11:23:05,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +31: [2023-05-08 11:23:05,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +14: [2023-05-08 11:23:05,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 2: [2023-05-08 11:23:05,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 8: [2023-05-08 11:23:05,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 2: [2023-05-08 11:23:05,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +28: [2023-05-08 11:23:05,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +21: [2023-05-08 11:23:05,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +28: [2023-05-08 11:23:05,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +31: [2023-05-08 11:23:05,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 7: [2023-05-08 11:23:05,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +28: [2023-05-08 11:23:05,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +28: [2023-05-08 11:23:05,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 7: [2023-05-08 11:23:05,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +31: [2023-05-08 11:23:05,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +13: [2023-05-08 11:23:05,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +13: [2023-05-08 11:23:05,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +29: [2023-05-08 11:23:05,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +29: [2023-05-08 11:23:05,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +13: [2023-05-08 11:23:05,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +13: [2023-05-08 11:23:05,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +29: [2023-05-08 11:23:05,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +29: [2023-05-08 11:23:05,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 1: [2023-05-08 11:23:05,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 5: [2023-05-08 11:23:05,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 5: [2023-05-08 11:23:05,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 5: [2023-05-08 11:23:05,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 5: [2023-05-08 11:23:05,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +27: [2023-05-08 11:23:05,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +27: [2023-05-08 11:23:05,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +31: [2023-05-08 11:23:05,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +27: [2023-05-08 11:23:05,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +27: [2023-05-08 11:23:05,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +13: [2023-05-08 11:23:05,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +13: [2023-05-08 11:23:05,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +14: [2023-05-08 11:23:05,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +31: [2023-05-08 11:23:05,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +13: [2023-05-08 11:23:05,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +13: [2023-05-08 11:23:05,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +28: [2023-05-08 11:23:05,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +28: [2023-05-08 11:23:05,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +21: [2023-05-08 11:23:05,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +14: [2023-05-08 11:23:05,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 9: [2023-05-08 11:23:05,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 9: [2023-05-08 11:23:05,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 9: [2023-05-08 11:23:05,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. + 9: [2023-05-08 11:23:05,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +10: [2023-05-08 11:23:05,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +21: [2023-05-08 11:23:05,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +10: [2023-05-08 11:23:05,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +10: [2023-05-08 11:23:05,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +10: [2023-05-08 11:23:05,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt. +21: [2023-05-08 11:23:05,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +21: [2023-05-08 11:23:05,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 1: [2023-05-08 11:23:05,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +21: [2023-05-08 11:23:05,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +21: [2023-05-08 11:23:05,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 1: [2023-05-08 11:23:05,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt... + 1: [2023-05-08 11:23:05,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt... + 1: [2023-05-08 11:23:05,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt... + 1: [2023-05-08 11:23:05,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt... +14: [2023-05-08 11:23:05,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +14: [2023-05-08 11:23:05,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +14: [2023-05-08 11:23:05,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +14: [2023-05-08 11:23:05,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +28: [2023-05-08 11:23:05,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +28: [2023-05-08 11:23:05,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +28: [2023-05-08 11:23:05,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +28: [2023-05-08 11:23:05,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +29: [2023-05-08 11:23:05,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +29: [2023-05-08 11:23:05,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +27: [2023-05-08 11:23:05,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +10: [2023-05-08 11:23:05,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +10: [2023-05-08 11:23:05,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +28: [2023-05-08 11:23:05,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +10: [2023-05-08 11:23:05,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... +27: [2023-05-08 11:23:05,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +28: [2023-05-08 11:23:05,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +27: [2023-05-08 11:23:05,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +10: [2023-05-08 11:23:05,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt... + 2: [2023-05-08 11:23:05,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +27: [2023-05-08 11:23:05,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +29: [2023-05-08 11:23:05,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +29: [2023-05-08 11:23:05,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +21: [2023-05-08 11:23:05,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +21: [2023-05-08 11:23:05,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +29: [2023-05-08 11:23:05,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 1: [2023-05-08 11:23:05,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +14: [2023-05-08 11:23:05,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 2: [2023-05-08 11:23:05,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +29: [2023-05-08 11:23:05,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 1: [2023-05-08 11:23:05,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +14: [2023-05-08 11:23:05,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +22: [2023-05-08 11:23:05,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +29: [2023-05-08 11:23:05,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +22: [2023-05-08 11:23:05,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +22: [2023-05-08 11:23:05,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 9: [2023-05-08 11:23:05,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +22: [2023-05-08 11:23:05,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +29: [2023-05-08 11:23:05,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 2: [2023-05-08 11:23:05,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +10: [2023-05-08 11:23:05,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 9: [2023-05-08 11:23:05,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +10: [2023-05-08 11:23:05,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... + 2: [2023-05-08 11:23:05,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +22: [2023-05-08 11:23:05,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +26: [2023-05-08 11:23:05,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +26: [2023-05-08 11:23:05,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +22: [2023-05-08 11:23:05,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +22: [2023-05-08 11:23:05,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +22: [2023-05-08 11:23:05,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +10: [2023-05-08 11:23:05,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +10: [2023-05-08 11:23:05,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +10: [2023-05-08 11:23:05,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +10: [2023-05-08 11:23:05,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +26: [2023-05-08 11:23:05,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +16: [2023-05-08 11:23:05,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +10: [2023-05-08 11:23:05,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +16: [2023-05-08 11:23:05,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +16: [2023-05-08 11:23:05,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +10: [2023-05-08 11:23:05,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +16: [2023-05-08 11:23:05,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 9: [2023-05-08 11:23:05,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. + 9: [2023-05-08 11:23:05,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +26: [2023-05-08 11:23:05,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +26: [2023-05-08 11:23:05,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +26: [2023-05-08 11:23:05,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt. +26: [2023-05-08 11:23:05,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 9: [2023-05-08 11:23:05,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt... +16: [2023-05-08 11:23:05,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +16: [2023-05-08 11:23:05,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +22: [2023-05-08 11:23:05,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +22: [2023-05-08 11:23:05,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +16: [2023-05-08 11:23:05,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +16: [2023-05-08 11:23:05,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +22: [2023-05-08 11:23:05,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +22: [2023-05-08 11:23:05,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +26: [2023-05-08 11:23:05,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +22: [2023-05-08 11:23:05,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +22: [2023-05-08 11:23:05,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +22: [2023-05-08 11:23:05,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +22: [2023-05-08 11:23:05,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +26: [2023-05-08 11:23:05,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +26: [2023-05-08 11:23:05,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +26: [2023-05-08 11:23:05,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +26: [2023-05-08 11:23:05,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +26: [2023-05-08 11:23:05,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +26: [2023-05-08 11:23:05,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +16: [2023-05-08 11:23:05,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +26: [2023-05-08 11:23:05,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +26: [2023-05-08 11:23:05,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +16: [2023-05-08 11:23:05,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +16: [2023-05-08 11:23:05,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +16: [2023-05-08 11:23:05,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +16: [2023-05-08 11:23:05,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +16: [2023-05-08 11:23:05,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +16: [2023-05-08 11:23:05,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +16: [2023-05-08 11:23:05,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +20: [2023-05-08 11:23:05,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt. +20: [2023-05-08 11:23:05,994] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 164 +20: [2023-05-08 11:23:05,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +20: [2023-05-08 11:23:05,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +20: [2023-05-08 11:23:05,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +20: [2023-05-08 11:23:05,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +20: [2023-05-08 11:23:05,998] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 164 +20: [2023-05-08 11:23:06,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +20: [2023-05-08 11:23:06,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +20: [2023-05-08 11:23:06,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +20: [2023-05-08 11:23:06,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +15: [2023-05-08 11:23:06,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +15: [2023-05-08 11:23:06,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +15: [2023-05-08 11:23:06,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +15: [2023-05-08 11:23:06,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 4: [2023-05-08 11:23:06,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt. + 4: [2023-05-08 11:23:06,018] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 32 + 4: [2023-05-08 11:23:06,023] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 32 +20: [2023-05-08 11:23:06,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +23: [2023-05-08 11:23:06,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +23: [2023-05-08 11:23:06,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +23: [2023-05-08 11:23:06,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +23: [2023-05-08 11:23:06,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +20: [2023-05-08 11:23:06,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +25: [2023-05-08 11:23:06,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +25: [2023-05-08 11:23:06,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +25: [2023-05-08 11:23:06,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +20: [2023-05-08 11:23:06,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 6: [2023-05-08 11:23:06,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 6: [2023-05-08 11:23:06,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +20: [2023-05-08 11:23:06,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +15: [2023-05-08 11:23:06,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +23: [2023-05-08 11:23:06,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt. + 6: [2023-05-08 11:23:06,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +23: [2023-05-08 11:23:06,025] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 184 +15: [2023-05-08 11:23:06,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 6: [2023-05-08 11:23:06,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +15: [2023-05-08 11:23:06,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +15: [2023-05-08 11:23:06,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +25: [2023-05-08 11:23:06,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +20: [2023-05-08 11:23:06,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +17: [2023-05-08 11:23:06,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +17: [2023-05-08 11:23:06,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +17: [2023-05-08 11:23:06,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +20: [2023-05-08 11:23:06,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +20: [2023-05-08 11:23:06,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +20: [2023-05-08 11:23:06,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +17: [2023-05-08 11:23:06,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +23: [2023-05-08 11:23:06,032] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 184 +25: [2023-05-08 11:23:06,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +25: [2023-05-08 11:23:06,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 6: [2023-05-08 11:23:06,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 6: [2023-05-08 11:23:06,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +25: [2023-05-08 11:23:06,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 6: [2023-05-08 11:23:06,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 6: [2023-05-08 11:23:06,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +23: [2023-05-08 11:23:06,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +25: [2023-05-08 11:23:06,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +15: [2023-05-08 11:23:06,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +15: [2023-05-08 11:23:06,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +23: [2023-05-08 11:23:06,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +23: [2023-05-08 11:23:06,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +23: [2023-05-08 11:23:06,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +30: [2023-05-08 11:23:06,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +30: [2023-05-08 11:23:06,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +30: [2023-05-08 11:23:06,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +19: [2023-05-08 11:23:06,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +19: [2023-05-08 11:23:06,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +30: [2023-05-08 11:23:06,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +19: [2023-05-08 11:23:06,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +17: [2023-05-08 11:23:06,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +19: [2023-05-08 11:23:06,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +11: [2023-05-08 11:23:06,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +11: [2023-05-08 11:23:06,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +17: [2023-05-08 11:23:06,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +11: [2023-05-08 11:23:06,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +24: [2023-05-08 11:23:06,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +24: [2023-05-08 11:23:06,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +24: [2023-05-08 11:23:06,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +24: [2023-05-08 11:23:06,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +17: [2023-05-08 11:23:06,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +11: [2023-05-08 11:23:06,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +17: [2023-05-08 11:23:06,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +15: [2023-05-08 11:23:06,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +15: [2023-05-08 11:23:06,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +30: [2023-05-08 11:23:06,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +30: [2023-05-08 11:23:06,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +30: [2023-05-08 11:23:06,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +23: [2023-05-08 11:23:06,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +19: [2023-05-08 11:23:06,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +19: [2023-05-08 11:23:06,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +30: [2023-05-08 11:23:06,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +23: [2023-05-08 11:23:06,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +19: [2023-05-08 11:23:06,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +19: [2023-05-08 11:23:06,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +25: [2023-05-08 11:23:06,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +25: [2023-05-08 11:23:06,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +24: [2023-05-08 11:23:06,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +15: [2023-05-08 11:23:06,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +25: [2023-05-08 11:23:06,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +25: [2023-05-08 11:23:06,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +11: [2023-05-08 11:23:06,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 6: [2023-05-08 11:23:06,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +15: [2023-05-08 11:23:06,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +24: [2023-05-08 11:23:06,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +23: [2023-05-08 11:23:06,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +23: [2023-05-08 11:23:06,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 6: [2023-05-08 11:23:06,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 6: [2023-05-08 11:23:06,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +24: [2023-05-08 11:23:06,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +24: [2023-05-08 11:23:06,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 6: [2023-05-08 11:23:06,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 6: [2023-05-08 11:23:06,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +11: [2023-05-08 11:23:06,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +15: [2023-05-08 11:23:06,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 6: [2023-05-08 11:23:06,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +25: [2023-05-08 11:23:06,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +25: [2023-05-08 11:23:06,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +11: [2023-05-08 11:23:06,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +11: [2023-05-08 11:23:06,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 6: [2023-05-08 11:23:06,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 6: [2023-05-08 11:23:06,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +17: [2023-05-08 11:23:06,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +15: [2023-05-08 11:23:06,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +17: [2023-05-08 11:23:06,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +17: [2023-05-08 11:23:06,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +12: [2023-05-08 11:23:06,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +17: [2023-05-08 11:23:06,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +12: [2023-05-08 11:23:06,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +12: [2023-05-08 11:23:06,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +23: [2023-05-08 11:23:06,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +23: [2023-05-08 11:23:06,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +17: [2023-05-08 11:23:06,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +17: [2023-05-08 11:23:06,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +17: [2023-05-08 11:23:06,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +17: [2023-05-08 11:23:06,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +23: [2023-05-08 11:23:06,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +12: [2023-05-08 11:23:06,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +16: [2023-05-08 11:23:06,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt. +16: [2023-05-08 11:23:06,065] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 132 +23: [2023-05-08 11:23:06,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +30: [2023-05-08 11:23:06,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +25: [2023-05-08 11:23:06,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +30: [2023-05-08 11:23:06,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +30: [2023-05-08 11:23:06,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +30: [2023-05-08 11:23:06,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +30: [2023-05-08 11:23:06,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +19: [2023-05-08 11:23:06,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +19: [2023-05-08 11:23:06,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +19: [2023-05-08 11:23:06,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +30: [2023-05-08 11:23:06,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +24: [2023-05-08 11:23:06,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +28: [2023-05-08 11:23:06,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +28: [2023-05-08 11:23:06,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +28: [2023-05-08 11:23:06,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +25: [2023-05-08 11:23:06,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +24: [2023-05-08 11:23:06,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +19: [2023-05-08 11:23:06,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +19: [2023-05-08 11:23:06,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +19: [2023-05-08 11:23:06,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +19: [2023-05-08 11:23:06,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +19: [2023-05-08 11:23:06,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +30: [2023-05-08 11:23:06,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +30: [2023-05-08 11:23:06,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +16: [2023-05-08 11:23:06,069] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 132 +12: [2023-05-08 11:23:06,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +11: [2023-05-08 11:23:06,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +28: [2023-05-08 11:23:06,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +11: [2023-05-08 11:23:06,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +12: [2023-05-08 11:23:06,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +12: [2023-05-08 11:23:06,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +24: [2023-05-08 11:23:06,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +16: [2023-05-08 11:23:06,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt. +24: [2023-05-08 11:23:06,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +16: [2023-05-08 11:23:06,072] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 134 +12: [2023-05-08 11:23:06,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +24: [2023-05-08 11:23:06,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +24: [2023-05-08 11:23:06,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +11: [2023-05-08 11:23:06,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +24: [2023-05-08 11:23:06,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +11: [2023-05-08 11:23:06,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt. +11: [2023-05-08 11:23:06,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +24: [2023-05-08 11:23:06,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +11: [2023-05-08 11:23:06,074] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 92 + 7: [2023-05-08 11:23:06,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt. + 7: [2023-05-08 11:23:06,076] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 62 +28: [2023-05-08 11:23:06,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +28: [2023-05-08 11:23:06,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +16: [2023-05-08 11:23:06,076] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 134 +28: [2023-05-08 11:23:06,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +28: [2023-05-08 11:23:06,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +29: [2023-05-08 11:23:06,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt. +29: [2023-05-08 11:23:06,079] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 238 +11: [2023-05-08 11:23:06,079] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 92 +11: [2023-05-08 11:23:06,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +11: [2023-05-08 11:23:06,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 7: [2023-05-08 11:23:06,080] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 62 +29: [2023-05-08 11:23:06,083] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 238 +11: [2023-05-08 11:23:06,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 9: [2023-05-08 11:23:06,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt. + 9: [2023-05-08 11:23:06,084] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 78 +12: [2023-05-08 11:23:06,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +12: [2023-05-08 11:23:06,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +12: [2023-05-08 11:23:06,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +12: [2023-05-08 11:23:06,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +11: [2023-05-08 11:23:06,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 7: [2023-05-08 11:23:06,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt. + 7: [2023-05-08 11:23:06,088] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 56 + 9: [2023-05-08 11:23:06,088] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 78 +12: [2023-05-08 11:23:06,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +11: [2023-05-08 11:23:06,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt. +11: [2023-05-08 11:23:06,090] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 90 +12: [2023-05-08 11:23:06,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +12: [2023-05-08 11:23:06,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +20: [2023-05-08 11:23:06,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt. +20: [2023-05-08 11:23:06,091] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 162 +12: [2023-05-08 11:23:06,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 7: [2023-05-08 11:23:06,092] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 56 +18: [2023-05-08 11:23:06,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +18: [2023-05-08 11:23:06,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +31: [2023-05-08 11:23:06,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +31: [2023-05-08 11:23:06,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +31: [2023-05-08 11:23:06,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +31: [2023-05-08 11:23:06,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +18: [2023-05-08 11:23:06,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +11: [2023-05-08 11:23:06,094] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 90 +27: [2023-05-08 11:23:06,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +27: [2023-05-08 11:23:06,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 5: [2023-05-08 11:23:06,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 5: [2023-05-08 11:23:06,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +20: [2023-05-08 11:23:06,095] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 162 + 5: [2023-05-08 11:23:06,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 5: [2023-05-08 11:23:06,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +18: [2023-05-08 11:23:06,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt. +18: [2023-05-08 11:23:06,096] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 150 +27: [2023-05-08 11:23:06,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +27: [2023-05-08 11:23:06,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 4: [2023-05-08 11:23:06,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt. + 4: [2023-05-08 11:23:06,097] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 34 + 4: [2023-05-08 11:23:06,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +18: [2023-05-08 11:23:06,100] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 150 + 4: [2023-05-08 11:23:06,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 4: [2023-05-08 11:23:06,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 4: [2023-05-08 11:23:06,101] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 34 + 4: [2023-05-08 11:23:06,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +18: [2023-05-08 11:23:06,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +18: [2023-05-08 11:23:06,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 9: [2023-05-08 11:23:06,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt. +28: [2023-05-08 11:23:06,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 9: [2023-05-08 11:23:06,096] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 74 +28: [2023-05-08 11:23:06,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +28: [2023-05-08 11:23:06,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +28: [2023-05-08 11:23:06,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +28: [2023-05-08 11:23:06,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +28: [2023-05-08 11:23:06,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +28: [2023-05-08 11:23:06,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +28: [2023-05-08 11:23:06,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +31: [2023-05-08 11:23:06,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +31: [2023-05-08 11:23:06,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +13: [2023-05-08 11:23:06,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +13: [2023-05-08 11:23:06,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +31: [2023-05-08 11:23:06,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +13: [2023-05-08 11:23:06,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +13: [2023-05-08 11:23:06,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +31: [2023-05-08 11:23:06,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 5: [2023-05-08 11:23:06,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +27: [2023-05-08 11:23:06,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +27: [2023-05-08 11:23:06,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +21: [2023-05-08 11:23:06,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +18: [2023-05-08 11:23:06,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +21: [2023-05-08 11:23:06,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +18: [2023-05-08 11:23:06,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 9: [2023-05-08 11:23:06,103] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 74 +27: [2023-05-08 11:23:06,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +27: [2023-05-08 11:23:06,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 5: [2023-05-08 11:23:06,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 5: [2023-05-08 11:23:06,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 5: [2023-05-08 11:23:06,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +21: [2023-05-08 11:23:06,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 5: [2023-05-08 11:23:06,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt. + 5: [2023-05-08 11:23:06,108] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 44 + 4: [2023-05-08 11:23:06,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 4: [2023-05-08 11:23:06,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 3: [2023-05-08 11:23:06,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt. + 4: [2023-05-08 11:23:06,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 3: [2023-05-08 11:23:06,111] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 28 +13: [2023-05-08 11:23:06,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +13: [2023-05-08 11:23:06,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +13: [2023-05-08 11:23:06,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 4: [2023-05-08 11:23:06,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 4: [2023-05-08 11:23:06,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 3: [2023-05-08 11:23:06,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt. + 4: [2023-05-08 11:23:06,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 4: [2023-05-08 11:23:06,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 4: [2023-05-08 11:23:06,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 3: [2023-05-08 11:23:06,114] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 30 + 4: [2023-05-08 11:23:06,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +13: [2023-05-08 11:23:06,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +16: [2023-05-08 11:23:06,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt. + 5: [2023-05-08 11:23:06,112] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 44 +16: [2023-05-08 11:23:06,115] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 128 + 0: [2023-05-08 11:23:06,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +21: [2023-05-08 11:23:06,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +21: [2023-05-08 11:23:06,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +18: [2023-05-08 11:23:06,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 4: [2023-05-08 11:23:06,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 3: [2023-05-08 11:23:06,116] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 28 + 4: [2023-05-08 11:23:06,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 4: [2023-05-08 11:23:06,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 7: [2023-05-08 11:23:06,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt. + 0: [2023-05-08 11:23:06,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 7: [2023-05-08 11:23:06,116] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 58 +21: [2023-05-08 11:23:06,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 3: [2023-05-08 11:23:06,118] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 30 +18: [2023-05-08 11:23:06,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +18: [2023-05-08 11:23:06,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +18: [2023-05-08 11:23:06,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +31: [2023-05-08 11:23:06,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 5: [2023-05-08 11:23:06,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +16: [2023-05-08 11:23:06,119] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 128 +18: [2023-05-08 11:23:06,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +18: [2023-05-08 11:23:06,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +18: [2023-05-08 11:23:06,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 5: [2023-05-08 11:23:06,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 0: [2023-05-08 11:23:06,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 5: [2023-05-08 11:23:06,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 5: [2023-05-08 11:23:06,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 0: [2023-05-08 11:23:06,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 4: [2023-05-08 11:23:06,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 4: [2023-05-08 11:23:06,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 4: [2023-05-08 11:23:06,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 7: [2023-05-08 11:23:06,121] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 58 + 4: [2023-05-08 11:23:06,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 5: [2023-05-08 11:23:06,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt. + 5: [2023-05-08 11:23:06,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt. + 5: [2023-05-08 11:23:06,122] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 46 + 5: [2023-05-08 11:23:06,123] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 40 +31: [2023-05-08 11:23:06,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +21: [2023-05-08 11:23:06,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +27: [2023-05-08 11:23:06,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +27: [2023-05-08 11:23:06,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +27: [2023-05-08 11:23:06,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +27: [2023-05-08 11:23:06,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +27: [2023-05-08 11:23:06,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +27: [2023-05-08 11:23:06,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 5: [2023-05-08 11:23:06,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 5: [2023-05-08 11:23:06,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +27: [2023-05-08 11:23:06,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +27: [2023-05-08 11:23:06,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 5: [2023-05-08 11:23:06,127] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 46 + 5: [2023-05-08 11:23:06,127] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 40 +18: [2023-05-08 11:23:06,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +18: [2023-05-08 11:23:06,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +13: [2023-05-08 11:23:06,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 0: [2023-05-08 11:23:06,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +21: [2023-05-08 11:23:06,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 0: [2023-05-08 11:23:06,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 0: [2023-05-08 11:23:06,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. + 0: [2023-05-08 11:23:06,130] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 0 + 0: [2023-05-08 11:23:06,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +21: [2023-05-08 11:23:06,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +13: [2023-05-08 11:23:06,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +13: [2023-05-08 11:23:06,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +13: [2023-05-08 11:23:06,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 0: [2023-05-08 11:23:06,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +13: [2023-05-08 11:23:06,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +13: [2023-05-08 11:23:06,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +31: [2023-05-08 11:23:06,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +31: [2023-05-08 11:23:06,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +11: [2023-05-08 11:23:06,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt. +21: [2023-05-08 11:23:06,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +11: [2023-05-08 11:23:06,132] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 94 +21: [2023-05-08 11:23:06,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +31: [2023-05-08 11:23:06,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +21: [2023-05-08 11:23:06,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +31: [2023-05-08 11:23:06,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +10: [2023-05-08 11:23:06,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +10: [2023-05-08 11:23:06,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +10: [2023-05-08 11:23:06,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +10: [2023-05-08 11:23:06,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 5: [2023-05-08 11:23:06,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 0: [2023-05-08 11:23:06,135] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 0 + 5: [2023-05-08 11:23:06,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +11: [2023-05-08 11:23:06,137] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 94 + 0: could not find arguments in the checkpoint ... + 0: checkpoint version 3.0 +21: [2023-05-08 11:23:06,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +21: [2023-05-08 11:23:06,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 9: [2023-05-08 11:23:06,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 9: [2023-05-08 11:23:06,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 1: [2023-05-08 11:23:06,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 1: [2023-05-08 11:23:06,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 1: [2023-05-08 11:23:06,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 1: [2023-05-08 11:23:06,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 9: [2023-05-08 11:23:06,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +31: [2023-05-08 11:23:06,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 9: [2023-05-08 11:23:06,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +31: [2023-05-08 11:23:06,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +10: [2023-05-08 11:23:06,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 3: [2023-05-08 11:23:06,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +10: [2023-05-08 11:23:06,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +29: [2023-05-08 11:23:06,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt. +13: [2023-05-08 11:23:06,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +29: [2023-05-08 11:23:06,143] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 232 + 3: [2023-05-08 11:23:06,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 3: [2023-05-08 11:23:06,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 3: [2023-05-08 11:23:06,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +10: [2023-05-08 11:23:06,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +10: [2023-05-08 11:23:06,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +21: [2023-05-08 11:23:06,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +21: [2023-05-08 11:23:06,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +13: [2023-05-08 11:23:06,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +14: [2023-05-08 11:23:06,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +29: [2023-05-08 11:23:06,147] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 232 +14: [2023-05-08 11:23:06,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +14: [2023-05-08 11:23:06,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +15: [2023-05-08 11:23:06,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt. + 0: [2023-05-08 11:23:06,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 0: [2023-05-08 11:23:06,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 1: [2023-05-08 11:23:06,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 9: [2023-05-08 11:23:06,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 9: [2023-05-08 11:23:06,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 9: [2023-05-08 11:23:06,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +15: [2023-05-08 11:23:06,150] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 122 +13: [2023-05-08 11:23:06,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt. + 1: [2023-05-08 11:23:06,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +13: [2023-05-08 11:23:06,150] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 108 +29: [2023-05-08 11:23:06,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt. +14: [2023-05-08 11:23:06,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +29: [2023-05-08 11:23:06,150] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 234 + 9: [2023-05-08 11:23:06,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 1: [2023-05-08 11:23:06,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 1: [2023-05-08 11:23:06,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 2: [2023-05-08 11:23:06,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 2: [2023-05-08 11:23:06,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 3: [2023-05-08 11:23:06,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 2: [2023-05-08 11:23:06,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 3: [2023-05-08 11:23:06,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 2: [2023-05-08 11:23:06,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 3: [2023-05-08 11:23:06,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 3: [2023-05-08 11:23:06,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 3: [2023-05-08 11:23:06,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 3: [2023-05-08 11:23:06,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +13: [2023-05-08 11:23:06,155] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 108 + 3: [2023-05-08 11:23:06,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 3: [2023-05-08 11:23:06,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +29: [2023-05-08 11:23:06,155] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 234 + 0: [2023-05-08 11:23:06,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 3: [2023-05-08 11:23:06,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 3: [2023-05-08 11:23:06,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 3: [2023-05-08 11:23:06,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 3: [2023-05-08 11:23:06,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 0: [2023-05-08 11:23:06,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 0: [2023-05-08 11:23:06,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +10: [2023-05-08 11:23:06,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +15: [2023-05-08 11:23:06,154] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 122 + 0: [2023-05-08 11:23:06,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +23: [2023-05-08 11:23:06,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt. +23: [2023-05-08 11:23:06,156] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 186 +14: [2023-05-08 11:23:06,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 7: [2023-05-08 11:23:06,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 3: [2023-05-08 11:23:06,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 3: [2023-05-08 11:23:06,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 3: [2023-05-08 11:23:06,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 3: [2023-05-08 11:23:06,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 7: [2023-05-08 11:23:06,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 7: [2023-05-08 11:23:06,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 7: [2023-05-08 11:23:06,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +29: [2023-05-08 11:23:06,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +20: [2023-05-08 11:23:06,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt. +20: [2023-05-08 11:23:06,159] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 160 +29: [2023-05-08 11:23:06,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +29: [2023-05-08 11:23:06,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. +14: [2023-05-08 11:23:06,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +14: [2023-05-08 11:23:06,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 0: [2023-05-08 11:23:06,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +10: [2023-05-08 11:23:06,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +23: [2023-05-08 11:23:06,162] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 186 + 2: [2023-05-08 11:23:06,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 2: [2023-05-08 11:23:06,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +29: [2023-05-08 11:23:06,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt. + 2: [2023-05-08 11:23:06,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 0: [2023-05-08 11:23:06,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 2: [2023-05-08 11:23:06,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 9: [2023-05-08 11:23:06,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt. +18: [2023-05-08 11:23:06,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt. +20: [2023-05-08 11:23:06,163] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 160 +14: [2023-05-08 11:23:06,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 9: [2023-05-08 11:23:06,163] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 72 +18: [2023-05-08 11:23:06,163] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 144 + 1: [2023-05-08 11:23:06,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 9: [2023-05-08 11:23:06,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +10: [2023-05-08 11:23:06,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +10: [2023-05-08 11:23:06,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +10: [2023-05-08 11:23:06,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 1: [2023-05-08 11:23:06,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +10: [2023-05-08 11:23:06,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 7: [2023-05-08 11:23:06,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 7: [2023-05-08 11:23:06,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 7: [2023-05-08 11:23:06,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 7: [2023-05-08 11:23:06,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 9: [2023-05-08 11:23:06,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 7: [2023-05-08 11:23:06,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 9: [2023-05-08 11:23:06,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 9: [2023-05-08 11:23:06,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 7: [2023-05-08 11:23:06,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +18: [2023-05-08 11:23:06,168] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 144 + 4: [2023-05-08 11:23:06,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt. + 5: [2023-05-08 11:23:06,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt. + 4: [2023-05-08 11:23:06,168] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 38 + 5: [2023-05-08 11:23:06,168] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 42 + 9: [2023-05-08 11:23:06,168] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 72 + 7: [2023-05-08 11:23:06,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +14: [2023-05-08 11:23:06,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 1: [2023-05-08 11:23:06,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 1: [2023-05-08 11:23:06,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +10: [2023-05-08 11:23:06,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 7: [2023-05-08 11:23:06,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +10: [2023-05-08 11:23:06,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +29: [2023-05-08 11:23:06,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +14: [2023-05-08 11:23:06,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 9: [2023-05-08 11:23:06,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 9: [2023-05-08 11:23:06,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 2: [2023-05-08 11:23:06,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt. + 2: [2023-05-08 11:23:06,171] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 18 + 7: [2023-05-08 11:23:06,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 7: [2023-05-08 11:23:06,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 7: [2023-05-08 11:23:06,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 7: [2023-05-08 11:23:06,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 4: [2023-05-08 11:23:06,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt. + 4: [2023-05-08 11:23:06,172] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 36 +29: [2023-05-08 11:23:06,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +29: [2023-05-08 11:23:06,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... +29: [2023-05-08 11:23:06,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt... + 4: [2023-05-08 11:23:06,172] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 38 + 5: [2023-05-08 11:23:06,173] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 42 + 7: [2023-05-08 11:23:06,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 7: [2023-05-08 11:23:06,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 7: [2023-05-08 11:23:06,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +24: [2023-05-08 11:23:06,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt. + 7: [2023-05-08 11:23:06,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +24: [2023-05-08 11:23:06,173] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 194 + 1: [2023-05-08 11:23:06,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 1: [2023-05-08 11:23:06,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 1: [2023-05-08 11:23:06,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 1: [2023-05-08 11:23:06,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 2: [2023-05-08 11:23:06,175] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 18 + 4: [2023-05-08 11:23:06,176] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 36 + 9: [2023-05-08 11:23:06,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 9: [2023-05-08 11:23:06,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +24: [2023-05-08 11:23:06,177] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 194 + 2: [2023-05-08 11:23:06,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +14: [2023-05-08 11:23:06,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 2: [2023-05-08 11:23:06,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +14: [2023-05-08 11:23:06,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 4: [2023-05-08 11:23:06,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt... + 4: [2023-05-08 11:23:06,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt... + 4: [2023-05-08 11:23:06,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt... + 4: [2023-05-08 11:23:06,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt... +26: [2023-05-08 11:23:06,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt. + 8: [2023-05-08 11:23:06,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +26: [2023-05-08 11:23:06,180] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 214 + 2: [2023-05-08 11:23:06,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 8: [2023-05-08 11:23:06,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 8: [2023-05-08 11:23:06,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 2: [2023-05-08 11:23:06,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 8: [2023-05-08 11:23:06,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +18: [2023-05-08 11:23:06,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt. +18: [2023-05-08 11:23:06,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt. +18: [2023-05-08 11:23:06,181] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 148 +18: [2023-05-08 11:23:06,181] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 146 +29: [2023-05-08 11:23:06,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +26: [2023-05-08 11:23:06,185] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 214 +18: [2023-05-08 11:23:06,185] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 146 + 2: [2023-05-08 11:23:06,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 2: [2023-05-08 11:23:06,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +29: [2023-05-08 11:23:06,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +18: [2023-05-08 11:23:06,186] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 148 +14: [2023-05-08 11:23:06,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +14: [2023-05-08 11:23:06,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 2: [2023-05-08 11:23:06,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 2: [2023-05-08 11:23:06,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... +16: [2023-05-08 11:23:06,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt. +16: [2023-05-08 11:23:06,190] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 130 + 8: [2023-05-08 11:23:06,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +29: [2023-05-08 11:23:06,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +29: [2023-05-08 11:23:06,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 8: [2023-05-08 11:23:06,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 8: [2023-05-08 11:23:06,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 8: [2023-05-08 11:23:06,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +22: [2023-05-08 11:23:06,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt. +14: [2023-05-08 11:23:06,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +22: [2023-05-08 11:23:06,191] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 180 +29: [2023-05-08 11:23:06,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. +29: [2023-05-08 11:23:06,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 8: [2023-05-08 11:23:06,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +15: [2023-05-08 11:23:06,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt. +23: [2023-05-08 11:23:06,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt. + 8: [2023-05-08 11:23:06,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +23: [2023-05-08 11:23:06,193] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 190 + 8: [2023-05-08 11:23:06,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 8: [2023-05-08 11:23:06,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +15: [2023-05-08 11:23:06,193] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 126 + 8: [2023-05-08 11:23:06,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +14: [2023-05-08 11:23:06,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 8: [2023-05-08 11:23:06,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 8: [2023-05-08 11:23:06,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +16: [2023-05-08 11:23:06,194] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 130 + 8: [2023-05-08 11:23:06,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 3: [2023-05-08 11:23:06,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt. + 3: [2023-05-08 11:23:06,195] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 26 +20: [2023-05-08 11:23:06,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt. +20: [2023-05-08 11:23:06,195] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 166 +22: [2023-05-08 11:23:06,195] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 180 +15: [2023-05-08 11:23:06,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt. + 3: [2023-05-08 11:23:06,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt... + 3: [2023-05-08 11:23:06,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt... + 3: [2023-05-08 11:23:06,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt... + 3: [2023-05-08 11:23:06,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt... +15: [2023-05-08 11:23:06,196] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 120 +23: [2023-05-08 11:23:06,197] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 190 +15: [2023-05-08 11:23:06,197] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 126 +22: [2023-05-08 11:23:06,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt. +22: [2023-05-08 11:23:06,198] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 178 + 8: [2023-05-08 11:23:06,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 8: [2023-05-08 11:23:06,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 8: [2023-05-08 11:23:06,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 8: [2023-05-08 11:23:06,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +20: [2023-05-08 11:23:06,199] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 166 + 3: [2023-05-08 11:23:06,200] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 26 +15: [2023-05-08 11:23:06,200] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 120 +22: [2023-05-08 11:23:06,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt. +22: [2023-05-08 11:23:06,200] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 182 +30: [2023-05-08 11:23:06,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt. +22: [2023-05-08 11:23:06,202] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 178 +30: [2023-05-08 11:23:06,202] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 244 + 3: [2023-05-08 11:23:06,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt. +11: [2023-05-08 11:23:06,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt. + 3: [2023-05-08 11:23:06,202] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 24 +11: [2023-05-08 11:23:06,202] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 88 + 0: [2023-05-08 11:23:06,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. + 0: [2023-05-08 11:23:06,204] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 4 +22: [2023-05-08 11:23:06,204] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 182 + 7: [2023-05-08 11:23:06,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt... + 7: [2023-05-08 11:23:06,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt... + 7: [2023-05-08 11:23:06,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt... + 7: [2023-05-08 11:23:06,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt... +11: [2023-05-08 11:23:06,206] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 88 + 3: [2023-05-08 11:23:06,206] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 24 +30: [2023-05-08 11:23:06,206] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 244 +29: [2023-05-08 11:23:06,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt. + 2: [2023-05-08 11:23:06,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt. +24: [2023-05-08 11:23:06,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt. +24: [2023-05-08 11:23:06,208] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 198 + 2: [2023-05-08 11:23:06,208] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 20 +29: [2023-05-08 11:23:06,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt... + 0: [2023-05-08 11:23:06,209] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 4 + 2: [2023-05-08 11:23:06,212] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 20 +24: [2023-05-08 11:23:06,212] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 198 +22: [2023-05-08 11:23:06,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt. +21: [2023-05-08 11:23:06,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt. +22: [2023-05-08 11:23:06,215] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 176 +21: [2023-05-08 11:23:06,215] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 168 +19: [2023-05-08 11:23:06,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt. +17: [2023-05-08 11:23:06,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt. +19: [2023-05-08 11:23:06,216] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 158 +17: [2023-05-08 11:23:06,216] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 138 +21: [2023-05-08 11:23:06,219] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 168 +22: [2023-05-08 11:23:06,219] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 176 +19: [2023-05-08 11:23:06,220] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 158 +17: [2023-05-08 11:23:06,221] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 138 +29: [2023-05-08 11:23:06,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt. +29: [2023-05-08 11:23:06,222] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 236 +29: [2023-05-08 11:23:06,227] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 236 +21: [2023-05-08 11:23:06,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt. +21: [2023-05-08 11:23:06,231] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 172 +24: [2023-05-08 11:23:06,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt. +24: [2023-05-08 11:23:06,232] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 196 +21: [2023-05-08 11:23:06,235] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 172 +24: [2023-05-08 11:23:06,237] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 196 +26: [2023-05-08 11:23:06,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +26: [2023-05-08 11:23:06,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +26: [2023-05-08 11:23:06,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +26: [2023-05-08 11:23:06,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 0: [2023-05-08 11:23:06,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. + 0: [2023-05-08 11:23:06,245] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 2 +10: [2023-05-08 11:23:06,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt. +10: [2023-05-08 11:23:06,246] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 84 +23: [2023-05-08 11:23:06,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt. +23: [2023-05-08 11:23:06,247] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 188 +26: [2023-05-08 11:23:06,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt. +31: [2023-05-08 11:23:06,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt. +26: [2023-05-08 11:23:06,248] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 212 +31: [2023-05-08 11:23:06,248] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 250 + 0: [2023-05-08 11:23:06,250] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 2 +10: [2023-05-08 11:23:06,250] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 84 +23: [2023-05-08 11:23:06,251] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 188 +30: [2023-05-08 11:23:06,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt. +30: [2023-05-08 11:23:06,252] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 240 + 0: [2023-05-08 11:23:06,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. +31: [2023-05-08 11:23:06,252] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 250 + 0: [2023-05-08 11:23:06,253] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 6 +26: [2023-05-08 11:23:06,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +26: [2023-05-08 11:23:06,253] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 212 +26: [2023-05-08 11:23:06,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +26: [2023-05-08 11:23:06,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +26: [2023-05-08 11:23:06,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +26: [2023-05-08 11:23:06,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +26: [2023-05-08 11:23:06,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +26: [2023-05-08 11:23:06,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +26: [2023-05-08 11:23:06,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +26: [2023-05-08 11:23:06,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +26: [2023-05-08 11:23:06,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +30: [2023-05-08 11:23:06,257] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 240 +26: [2023-05-08 11:23:06,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +26: [2023-05-08 11:23:06,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +26: [2023-05-08 11:23:06,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +26: [2023-05-08 11:23:06,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +26: [2023-05-08 11:23:06,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 0: [2023-05-08 11:23:06,258] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 6 +26: [2023-05-08 11:23:06,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +26: [2023-05-08 11:23:06,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt. +26: [2023-05-08 11:23:06,266] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 210 +13: [2023-05-08 11:23:06,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt. +13: [2023-05-08 11:23:06,267] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 106 +12: [2023-05-08 11:23:06,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt. + 8: [2023-05-08 11:23:06,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt... + 8: [2023-05-08 11:23:06,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt... + 8: [2023-05-08 11:23:06,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt... + 8: [2023-05-08 11:23:06,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt... +12: [2023-05-08 11:23:06,267] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 98 +24: [2023-05-08 11:23:06,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt. +24: [2023-05-08 11:23:06,268] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 192 +26: [2023-05-08 11:23:06,271] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 210 +13: [2023-05-08 11:23:06,271] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 106 + 9: [2023-05-08 11:23:06,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt. +12: [2023-05-08 11:23:06,271] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 98 + 9: [2023-05-08 11:23:06,272] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 76 +24: [2023-05-08 11:23:06,272] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 192 +21: [2023-05-08 11:23:06,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt. +21: [2023-05-08 11:23:06,276] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 170 + 9: [2023-05-08 11:23:06,278] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 76 +21: [2023-05-08 11:23:06,280] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 170 +14: [2023-05-08 11:23:06,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt. +14: [2023-05-08 11:23:06,290] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 112 +27: [2023-05-08 11:23:06,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt. +27: [2023-05-08 11:23:06,292] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 216 + 8: [2023-05-08 11:23:06,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt. + 8: [2023-05-08 11:23:06,292] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 66 +17: [2023-05-08 11:23:06,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt. +17: [2023-05-08 11:23:06,296] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 140 +31: [2023-05-08 11:23:06,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt. +31: [2023-05-08 11:23:06,296] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 252 +14: [2023-05-08 11:23:06,296] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 112 +27: [2023-05-08 11:23:06,296] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 216 +26: [2023-05-08 11:23:06,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt... +26: [2023-05-08 11:23:06,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt... +26: [2023-05-08 11:23:06,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt... +26: [2023-05-08 11:23:06,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt... + 8: [2023-05-08 11:23:06,297] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 66 +27: [2023-05-08 11:23:06,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt. +27: [2023-05-08 11:23:06,298] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 222 +15: [2023-05-08 11:23:06,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt. +15: [2023-05-08 11:23:06,300] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 124 +25: [2023-05-08 11:23:06,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt. +31: [2023-05-08 11:23:06,300] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 252 +25: [2023-05-08 11:23:06,300] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 206 +17: [2023-05-08 11:23:06,301] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 140 +19: [2023-05-08 11:23:06,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt. +19: [2023-05-08 11:23:06,301] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 152 +27: [2023-05-08 11:23:06,303] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 222 +15: [2023-05-08 11:23:06,304] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 124 +25: [2023-05-08 11:23:06,304] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 206 +25: [2023-05-08 11:23:06,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt. +19: [2023-05-08 11:23:06,305] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 152 +25: [2023-05-08 11:23:06,305] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 200 +11: [2023-05-08 11:23:06,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +11: [2023-05-08 11:23:06,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +28: [2023-05-08 11:23:06,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt. +28: [2023-05-08 11:23:06,307] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 228 +11: [2023-05-08 11:23:06,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +11: [2023-05-08 11:23:06,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +25: [2023-05-08 11:23:06,309] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 200 +31: [2023-05-08 11:23:06,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt. +31: [2023-05-08 11:23:06,310] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 248 +14: [2023-05-08 11:23:06,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt. +14: [2023-05-08 11:23:06,310] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 114 + 8: [2023-05-08 11:23:06,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt. + 8: [2023-05-08 11:23:06,311] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 64 +28: [2023-05-08 11:23:06,311] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 228 +31: [2023-05-08 11:23:06,314] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 248 +14: [2023-05-08 11:23:06,315] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 114 +10: [2023-05-08 11:23:06,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt. +10: [2023-05-08 11:23:06,315] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 80 + 8: [2023-05-08 11:23:06,316] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 64 +19: [2023-05-08 11:23:06,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt. +19: [2023-05-08 11:23:06,316] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 156 +11: [2023-05-08 11:23:06,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +11: [2023-05-08 11:23:06,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +11: [2023-05-08 11:23:06,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +11: [2023-05-08 11:23:06,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +27: [2023-05-08 11:23:06,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt. +27: [2023-05-08 11:23:06,318] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 220 +11: [2023-05-08 11:23:06,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +28: [2023-05-08 11:23:06,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt. +11: [2023-05-08 11:23:06,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +28: [2023-05-08 11:23:06,319] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 226 +11: [2023-05-08 11:23:06,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +14: [2023-05-08 11:23:06,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt. +11: [2023-05-08 11:23:06,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +14: [2023-05-08 11:23:06,319] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 116 +11: [2023-05-08 11:23:06,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +11: [2023-05-08 11:23:06,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +11: [2023-05-08 11:23:06,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +11: [2023-05-08 11:23:06,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +10: [2023-05-08 11:23:06,320] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 80 +11: [2023-05-08 11:23:06,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +11: [2023-05-08 11:23:06,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +11: [2023-05-08 11:23:06,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +11: [2023-05-08 11:23:06,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +19: [2023-05-08 11:23:06,321] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 156 +12: [2023-05-08 11:23:06,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt. +12: [2023-05-08 11:23:06,321] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 100 +30: [2023-05-08 11:23:06,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt. +30: [2023-05-08 11:23:06,322] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 246 +13: [2023-05-08 11:23:06,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt. +13: [2023-05-08 11:23:06,322] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 104 +27: [2023-05-08 11:23:06,322] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 220 +28: [2023-05-08 11:23:06,323] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 226 +27: [2023-05-08 11:23:06,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt. +27: [2023-05-08 11:23:06,324] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 218 +14: [2023-05-08 11:23:06,324] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 116 +12: [2023-05-08 11:23:06,325] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 100 +28: [2023-05-08 11:23:06,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt. +14: [2023-05-08 11:23:06,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt. +28: [2023-05-08 11:23:06,325] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 224 +14: [2023-05-08 11:23:06,325] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 118 +13: [2023-05-08 11:23:06,326] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 104 +30: [2023-05-08 11:23:06,326] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 246 +28: [2023-05-08 11:23:06,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt. +28: [2023-05-08 11:23:06,327] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 230 +27: [2023-05-08 11:23:06,328] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 218 +14: [2023-05-08 11:23:06,329] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 118 +28: [2023-05-08 11:23:06,332] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 224 +28: [2023-05-08 11:23:06,332] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 230 +12: [2023-05-08 11:23:06,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt. +12: [2023-05-08 11:23:06,338] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 102 +25: [2023-05-08 11:23:06,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt. +25: [2023-05-08 11:23:06,340] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 202 + 6: [2023-05-08 11:23:06,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt. + 6: [2023-05-08 11:23:06,340] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 50 + 6: [2023-05-08 11:23:06,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt. + 6: [2023-05-08 11:23:06,341] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 54 +12: [2023-05-08 11:23:06,342] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 102 +10: [2023-05-08 11:23:06,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt. +10: [2023-05-08 11:23:06,344] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 82 + 6: [2023-05-08 11:23:06,345] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 50 +25: [2023-05-08 11:23:06,345] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 202 + 6: [2023-05-08 11:23:06,346] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 54 +10: [2023-05-08 11:23:06,348] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 82 +19: [2023-05-08 11:23:06,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +19: [2023-05-08 11:23:06,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +19: [2023-05-08 11:23:06,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +19: [2023-05-08 11:23:06,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +11: [2023-05-08 11:23:06,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt... +11: [2023-05-08 11:23:06,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt... +11: [2023-05-08 11:23:06,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt... +11: [2023-05-08 11:23:06,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt... + 2: [2023-05-08 11:23:06,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt. +10: [2023-05-08 11:23:06,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt. + 2: [2023-05-08 11:23:06,358] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 16 +10: [2023-05-08 11:23:06,358] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 86 +19: [2023-05-08 11:23:06,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +19: [2023-05-08 11:23:06,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 1: [2023-05-08 11:23:06,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt. +19: [2023-05-08 11:23:06,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +19: [2023-05-08 11:23:06,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 1: [2023-05-08 11:23:06,361] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 14 + 2: [2023-05-08 11:23:06,362] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 16 +19: [2023-05-08 11:23:06,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 7: [2023-05-08 11:23:06,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt. +19: [2023-05-08 11:23:06,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +19: [2023-05-08 11:23:06,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +19: [2023-05-08 11:23:06,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +19: [2023-05-08 11:23:06,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 7: [2023-05-08 11:23:06,362] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 60 +19: [2023-05-08 11:23:06,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +10: [2023-05-08 11:23:06,363] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 86 +19: [2023-05-08 11:23:06,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +19: [2023-05-08 11:23:06,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 6: [2023-05-08 11:23:06,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt. + 6: [2023-05-08 11:23:06,363] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 48 + 6: [2023-05-08 11:23:06,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt. + 6: [2023-05-08 11:23:06,364] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 52 +19: [2023-05-08 11:23:06,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +19: [2023-05-08 11:23:06,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +19: [2023-05-08 11:23:06,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +19: [2023-05-08 11:23:06,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 9: [2023-05-08 11:23:06,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 9: [2023-05-08 11:23:06,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 9: [2023-05-08 11:23:06,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 9: [2023-05-08 11:23:06,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 1: [2023-05-08 11:23:06,365] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 14 + 7: [2023-05-08 11:23:06,367] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 60 + 6: [2023-05-08 11:23:06,368] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 48 + 6: [2023-05-08 11:23:06,369] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 52 +31: [2023-05-08 11:23:06,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt. +31: [2023-05-08 11:23:06,369] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 254 + 9: [2023-05-08 11:23:06,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +31: [2023-05-08 11:23:06,374] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 254 + 9: [2023-05-08 11:23:06,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 9: [2023-05-08 11:23:06,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 9: [2023-05-08 11:23:06,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 9: [2023-05-08 11:23:06,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 9: [2023-05-08 11:23:06,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 9: [2023-05-08 11:23:06,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 9: [2023-05-08 11:23:06,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 9: [2023-05-08 11:23:06,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 9: [2023-05-08 11:23:06,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 9: [2023-05-08 11:23:06,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 9: [2023-05-08 11:23:06,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 9: [2023-05-08 11:23:06,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 9: [2023-05-08 11:23:06,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 9: [2023-05-08 11:23:06,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 1: [2023-05-08 11:23:06,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt. + 1: [2023-05-08 11:23:06,380] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 12 + 9: [2023-05-08 11:23:06,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +31: [2023-05-08 11:23:06,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +28: [2023-05-08 11:23:06,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +28: [2023-05-08 11:23:06,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +28: [2023-05-08 11:23:06,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +31: [2023-05-08 11:23:06,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +31: [2023-05-08 11:23:06,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 2: [2023-05-08 11:23:06,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt. +28: [2023-05-08 11:23:06,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +31: [2023-05-08 11:23:06,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 2: [2023-05-08 11:23:06,381] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 22 + 1: [2023-05-08 11:23:06,384] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 12 + 8: [2023-05-08 11:23:06,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt. + 1: [2023-05-08 11:23:06,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt. + 8: [2023-05-08 11:23:06,384] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 70 + 1: [2023-05-08 11:23:06,384] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 8 +19: [2023-05-08 11:23:06,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt. +19: [2023-05-08 11:23:06,385] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 154 + 2: [2023-05-08 11:23:06,385] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 22 +15: [2023-05-08 11:23:06,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +15: [2023-05-08 11:23:06,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +15: [2023-05-08 11:23:06,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +15: [2023-05-08 11:23:06,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 1: [2023-05-08 11:23:06,388] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 8 + 8: [2023-05-08 11:23:06,389] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 70 +24: [2023-05-08 11:23:06,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +24: [2023-05-08 11:23:06,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +24: [2023-05-08 11:23:06,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +24: [2023-05-08 11:23:06,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +19: [2023-05-08 11:23:06,390] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 154 +28: [2023-05-08 11:23:06,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +13: [2023-05-08 11:23:06,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +13: [2023-05-08 11:23:06,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +31: [2023-05-08 11:23:06,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +13: [2023-05-08 11:23:06,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +28: [2023-05-08 11:23:06,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +28: [2023-05-08 11:23:06,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +31: [2023-05-08 11:23:06,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +28: [2023-05-08 11:23:06,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +31: [2023-05-08 11:23:06,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +28: [2023-05-08 11:23:06,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +13: [2023-05-08 11:23:06,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +31: [2023-05-08 11:23:06,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +31: [2023-05-08 11:23:06,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +28: [2023-05-08 11:23:06,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +31: [2023-05-08 11:23:06,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +28: [2023-05-08 11:23:06,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +28: [2023-05-08 11:23:06,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +28: [2023-05-08 11:23:06,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +28: [2023-05-08 11:23:06,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +28: [2023-05-08 11:23:06,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +28: [2023-05-08 11:23:06,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +31: [2023-05-08 11:23:06,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +31: [2023-05-08 11:23:06,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +31: [2023-05-08 11:23:06,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +31: [2023-05-08 11:23:06,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +31: [2023-05-08 11:23:06,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +31: [2023-05-08 11:23:06,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +31: [2023-05-08 11:23:06,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +31: [2023-05-08 11:23:06,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +28: [2023-05-08 11:23:06,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +28: [2023-05-08 11:23:06,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +28: [2023-05-08 11:23:06,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +31: [2023-05-08 11:23:06,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +28: [2023-05-08 11:23:06,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +15: [2023-05-08 11:23:06,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +16: [2023-05-08 11:23:06,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +31: [2023-05-08 11:23:06,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +16: [2023-05-08 11:23:06,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +15: [2023-05-08 11:23:06,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +16: [2023-05-08 11:23:06,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +15: [2023-05-08 11:23:06,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +16: [2023-05-08 11:23:06,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +15: [2023-05-08 11:23:06,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +15: [2023-05-08 11:23:06,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +15: [2023-05-08 11:23:06,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +21: [2023-05-08 11:23:06,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +21: [2023-05-08 11:23:06,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +21: [2023-05-08 11:23:06,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +15: [2023-05-08 11:23:06,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +15: [2023-05-08 11:23:06,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +21: [2023-05-08 11:23:06,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +15: [2023-05-08 11:23:06,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +15: [2023-05-08 11:23:06,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +15: [2023-05-08 11:23:06,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +15: [2023-05-08 11:23:06,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +15: [2023-05-08 11:23:06,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +15: [2023-05-08 11:23:06,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +15: [2023-05-08 11:23:06,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +15: [2023-05-08 11:23:06,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +24: [2023-05-08 11:23:06,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +24: [2023-05-08 11:23:06,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +24: [2023-05-08 11:23:06,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +24: [2023-05-08 11:23:06,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +24: [2023-05-08 11:23:06,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +24: [2023-05-08 11:23:06,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 1: [2023-05-08 11:23:06,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt. +24: [2023-05-08 11:23:06,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +24: [2023-05-08 11:23:06,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +24: [2023-05-08 11:23:06,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +24: [2023-05-08 11:23:06,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +24: [2023-05-08 11:23:06,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +24: [2023-05-08 11:23:06,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 1: [2023-05-08 11:23:06,399] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 10 +13: [2023-05-08 11:23:06,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +19: [2023-05-08 11:23:06,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt... +19: [2023-05-08 11:23:06,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt... +13: [2023-05-08 11:23:06,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +19: [2023-05-08 11:23:06,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt... +19: [2023-05-08 11:23:06,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt... +13: [2023-05-08 11:23:06,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +24: [2023-05-08 11:23:06,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +24: [2023-05-08 11:23:06,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +24: [2023-05-08 11:23:06,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +24: [2023-05-08 11:23:06,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +13: [2023-05-08 11:23:06,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +13: [2023-05-08 11:23:06,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +25: [2023-05-08 11:23:06,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +25: [2023-05-08 11:23:06,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +25: [2023-05-08 11:23:06,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +13: [2023-05-08 11:23:06,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +13: [2023-05-08 11:23:06,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +13: [2023-05-08 11:23:06,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +13: [2023-05-08 11:23:06,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +16: [2023-05-08 11:23:06,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +16: [2023-05-08 11:23:06,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +13: [2023-05-08 11:23:06,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +16: [2023-05-08 11:23:06,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +25: [2023-05-08 11:23:06,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +16: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +16: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +22: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +16: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +13: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +22: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 1: [2023-05-08 11:23:06,404] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 10 +13: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +16: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +22: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +13: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +13: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +13: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +16: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +16: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +16: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +20: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +17: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +22: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +20: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +20: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +17: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +21: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +17: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +16: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +21: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +16: [2023-05-08 11:23:06,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +13: [2023-05-08 11:23:06,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +17: [2023-05-08 11:23:06,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +21: [2023-05-08 11:23:06,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +20: [2023-05-08 11:23:06,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +21: [2023-05-08 11:23:06,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +21: [2023-05-08 11:23:06,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +23: [2023-05-08 11:23:06,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +23: [2023-05-08 11:23:06,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +23: [2023-05-08 11:23:06,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +21: [2023-05-08 11:23:06,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +23: [2023-05-08 11:23:06,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +16: [2023-05-08 11:23:06,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +16: [2023-05-08 11:23:06,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +16: [2023-05-08 11:23:06,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +16: [2023-05-08 11:23:06,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +21: [2023-05-08 11:23:06,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +21: [2023-05-08 11:23:06,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +21: [2023-05-08 11:23:06,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +30: [2023-05-08 11:23:06,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +30: [2023-05-08 11:23:06,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 8: [2023-05-08 11:23:06,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt. +30: [2023-05-08 11:23:06,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +21: [2023-05-08 11:23:06,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +21: [2023-05-08 11:23:06,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +30: [2023-05-08 11:23:06,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 8: [2023-05-08 11:23:06,407] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 68 +21: [2023-05-08 11:23:06,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +17: [2023-05-08 11:23:06,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt. +18: [2023-05-08 11:23:06,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +17: [2023-05-08 11:23:06,408] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 142 +18: [2023-05-08 11:23:06,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +18: [2023-05-08 11:23:06,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +18: [2023-05-08 11:23:06,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +21: [2023-05-08 11:23:06,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +21: [2023-05-08 11:23:06,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +21: [2023-05-08 11:23:06,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +21: [2023-05-08 11:23:06,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +13: [2023-05-08 11:23:06,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt. +13: [2023-05-08 11:23:06,411] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 110 + 8: [2023-05-08 11:23:06,411] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 68 +21: [2023-05-08 11:23:06,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt. +21: [2023-05-08 11:23:06,412] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 174 +25: [2023-05-08 11:23:06,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +25: [2023-05-08 11:23:06,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +22: [2023-05-08 11:23:06,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +25: [2023-05-08 11:23:06,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +17: [2023-05-08 11:23:06,413] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 142 +25: [2023-05-08 11:23:06,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +22: [2023-05-08 11:23:06,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +20: [2023-05-08 11:23:06,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +25: [2023-05-08 11:23:06,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +22: [2023-05-08 11:23:06,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +22: [2023-05-08 11:23:06,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +25: [2023-05-08 11:23:06,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +20: [2023-05-08 11:23:06,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +20: [2023-05-08 11:23:06,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +22: [2023-05-08 11:23:06,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +22: [2023-05-08 11:23:06,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +22: [2023-05-08 11:23:06,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +22: [2023-05-08 11:23:06,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +22: [2023-05-08 11:23:06,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +25: [2023-05-08 11:23:06,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +25: [2023-05-08 11:23:06,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +25: [2023-05-08 11:23:06,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +20: [2023-05-08 11:23:06,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 9: [2023-05-08 11:23:06,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt... + 9: [2023-05-08 11:23:06,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt... + 9: [2023-05-08 11:23:06,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt... +20: [2023-05-08 11:23:06,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +25: [2023-05-08 11:23:06,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 9: [2023-05-08 11:23:06,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt... +20: [2023-05-08 11:23:06,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +25: [2023-05-08 11:23:06,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +22: [2023-05-08 11:23:06,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +17: [2023-05-08 11:23:06,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +25: [2023-05-08 11:23:06,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +13: [2023-05-08 11:23:06,415] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 110 + 5: [2023-05-08 11:23:06,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +22: [2023-05-08 11:23:06,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +22: [2023-05-08 11:23:06,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +17: [2023-05-08 11:23:06,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt. +17: [2023-05-08 11:23:06,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 5: [2023-05-08 11:23:06,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 5: [2023-05-08 11:23:06,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +17: [2023-05-08 11:23:06,416] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 136 +17: [2023-05-08 11:23:06,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +20: [2023-05-08 11:23:06,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +20: [2023-05-08 11:23:06,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +17: [2023-05-08 11:23:06,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +21: [2023-05-08 11:23:06,416] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 174 +17: [2023-05-08 11:23:06,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +17: [2023-05-08 11:23:06,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +20: [2023-05-08 11:23:06,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +20: [2023-05-08 11:23:06,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +22: [2023-05-08 11:23:06,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +22: [2023-05-08 11:23:06,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +20: [2023-05-08 11:23:06,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +22: [2023-05-08 11:23:06,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +20: [2023-05-08 11:23:06,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +22: [2023-05-08 11:23:06,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +25: [2023-05-08 11:23:06,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +25: [2023-05-08 11:23:06,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +25: [2023-05-08 11:23:06,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 5: [2023-05-08 11:23:06,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +20: [2023-05-08 11:23:06,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +25: [2023-05-08 11:23:06,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +20: [2023-05-08 11:23:06,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +20: [2023-05-08 11:23:06,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +20: [2023-05-08 11:23:06,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +17: [2023-05-08 11:23:06,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +30: [2023-05-08 11:23:06,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +30: [2023-05-08 11:23:06,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +27: [2023-05-08 11:23:06,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 6: [2023-05-08 11:23:06,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 6: [2023-05-08 11:23:06,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +17: [2023-05-08 11:23:06,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +18: [2023-05-08 11:23:06,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +18: [2023-05-08 11:23:06,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 6: [2023-05-08 11:23:06,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 0: [2023-05-08 11:23:06,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 0: [2023-05-08 11:23:06,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +27: [2023-05-08 11:23:06,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +27: [2023-05-08 11:23:06,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 0: [2023-05-08 11:23:06,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 6: [2023-05-08 11:23:06,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +27: [2023-05-08 11:23:06,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +17: [2023-05-08 11:23:06,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +30: [2023-05-08 11:23:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +30: [2023-05-08 11:23:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +17: [2023-05-08 11:23:06,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +30: [2023-05-08 11:23:06,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +17: [2023-05-08 11:23:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +18: [2023-05-08 11:23:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +18: [2023-05-08 11:23:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +17: [2023-05-08 11:23:06,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +30: [2023-05-08 11:23:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +17: [2023-05-08 11:23:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +17: [2023-05-08 11:23:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +17: [2023-05-08 11:23:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +30: [2023-05-08 11:23:06,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +23: [2023-05-08 11:23:06,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +23: [2023-05-08 11:23:06,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +18: [2023-05-08 11:23:06,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +23: [2023-05-08 11:23:06,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +18: [2023-05-08 11:23:06,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +18: [2023-05-08 11:23:06,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +17: [2023-05-08 11:23:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +30: [2023-05-08 11:23:06,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +23: [2023-05-08 11:23:06,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +23: [2023-05-08 11:23:06,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +18: [2023-05-08 11:23:06,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +18: [2023-05-08 11:23:06,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +23: [2023-05-08 11:23:06,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +23: [2023-05-08 11:23:06,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +23: [2023-05-08 11:23:06,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +23: [2023-05-08 11:23:06,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +30: [2023-05-08 11:23:06,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +30: [2023-05-08 11:23:06,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +30: [2023-05-08 11:23:06,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 0: [2023-05-08 11:23:06,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +30: [2023-05-08 11:23:06,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +17: [2023-05-08 11:23:06,420] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 136 +12: [2023-05-08 11:23:06,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +12: [2023-05-08 11:23:06,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +18: [2023-05-08 11:23:06,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +18: [2023-05-08 11:23:06,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +18: [2023-05-08 11:23:06,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +18: [2023-05-08 11:23:06,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +18: [2023-05-08 11:23:06,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +18: [2023-05-08 11:23:06,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +23: [2023-05-08 11:23:06,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +18: [2023-05-08 11:23:06,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +23: [2023-05-08 11:23:06,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +23: [2023-05-08 11:23:06,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +23: [2023-05-08 11:23:06,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +23: [2023-05-08 11:23:06,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +23: [2023-05-08 11:23:06,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +30: [2023-05-08 11:23:06,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +30: [2023-05-08 11:23:06,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +30: [2023-05-08 11:23:06,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +30: [2023-05-08 11:23:06,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 2: [2023-05-08 11:23:06,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +23: [2023-05-08 11:23:06,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 2: [2023-05-08 11:23:06,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 2: [2023-05-08 11:23:06,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +12: [2023-05-08 11:23:06,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +12: [2023-05-08 11:23:06,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 2: [2023-05-08 11:23:06,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +12: [2023-05-08 11:23:06,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt. +12: [2023-05-08 11:23:06,424] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 96 + 5: [2023-05-08 11:23:06,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 5: [2023-05-08 11:23:06,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +29: [2023-05-08 11:23:06,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +27: [2023-05-08 11:23:06,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 5: [2023-05-08 11:23:06,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +26: [2023-05-08 11:23:06,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt. + 5: [2023-05-08 11:23:06,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +29: [2023-05-08 11:23:06,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +29: [2023-05-08 11:23:06,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +26: [2023-05-08 11:23:06,426] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 208 +29: [2023-05-08 11:23:06,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +31: [2023-05-08 11:23:06,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt... +31: [2023-05-08 11:23:06,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt... +31: [2023-05-08 11:23:06,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt... +27: [2023-05-08 11:23:06,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 5: [2023-05-08 11:23:06,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 5: [2023-05-08 11:23:06,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +31: [2023-05-08 11:23:06,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt... +27: [2023-05-08 11:23:06,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 5: [2023-05-08 11:23:06,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 5: [2023-05-08 11:23:06,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 5: [2023-05-08 11:23:06,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 6: [2023-05-08 11:23:06,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +27: [2023-05-08 11:23:06,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +27: [2023-05-08 11:23:06,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +27: [2023-05-08 11:23:06,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 6: [2023-05-08 11:23:06,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 6: [2023-05-08 11:23:06,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 5: [2023-05-08 11:23:06,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 6: [2023-05-08 11:23:06,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 6: [2023-05-08 11:23:06,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 6: [2023-05-08 11:23:06,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 5: [2023-05-08 11:23:06,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 6: [2023-05-08 11:23:06,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 5: [2023-05-08 11:23:06,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 6: [2023-05-08 11:23:06,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 6: [2023-05-08 11:23:06,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 6: [2023-05-08 11:23:06,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 6: [2023-05-08 11:23:06,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 6: [2023-05-08 11:23:06,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +15: [2023-05-08 11:23:06,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt... +15: [2023-05-08 11:23:06,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt... +15: [2023-05-08 11:23:06,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt... +15: [2023-05-08 11:23:06,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt... + 0: [2023-05-08 11:23:06,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +28: [2023-05-08 11:23:06,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt... +28: [2023-05-08 11:23:06,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt... +28: [2023-05-08 11:23:06,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt... +28: [2023-05-08 11:23:06,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt... + 5: [2023-05-08 11:23:06,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 5: [2023-05-08 11:23:06,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 5: [2023-05-08 11:23:06,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 5: [2023-05-08 11:23:06,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +14: [2023-05-08 11:23:06,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +12: [2023-05-08 11:23:06,429] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 96 + 0: [2023-05-08 11:23:06,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +14: [2023-05-08 11:23:06,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +14: [2023-05-08 11:23:06,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 0: [2023-05-08 11:23:06,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 6: [2023-05-08 11:23:06,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 6: [2023-05-08 11:23:06,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 6: [2023-05-08 11:23:06,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 6: [2023-05-08 11:23:06,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +27: [2023-05-08 11:23:06,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +26: [2023-05-08 11:23:06,430] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 208 + 0: [2023-05-08 11:23:06,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +27: [2023-05-08 11:23:06,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +27: [2023-05-08 11:23:06,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +12: [2023-05-08 11:23:06,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 0: [2023-05-08 11:23:06,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +27: [2023-05-08 11:23:06,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 0: [2023-05-08 11:23:06,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 0: [2023-05-08 11:23:06,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +12: [2023-05-08 11:23:06,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +27: [2023-05-08 11:23:06,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +27: [2023-05-08 11:23:06,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +27: [2023-05-08 11:23:06,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +27: [2023-05-08 11:23:06,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 0: [2023-05-08 11:23:06,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 0: [2023-05-08 11:23:06,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +12: [2023-05-08 11:23:06,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +27: [2023-05-08 11:23:06,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 0: [2023-05-08 11:23:06,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +12: [2023-05-08 11:23:06,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +27: [2023-05-08 11:23:06,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 2: [2023-05-08 11:23:06,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +12: [2023-05-08 11:23:06,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +12: [2023-05-08 11:23:06,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 0: [2023-05-08 11:23:06,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +12: [2023-05-08 11:23:06,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 0: [2023-05-08 11:23:06,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +12: [2023-05-08 11:23:06,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +12: [2023-05-08 11:23:06,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +12: [2023-05-08 11:23:06,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 2: [2023-05-08 11:23:06,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +12: [2023-05-08 11:23:06,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +12: [2023-05-08 11:23:06,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 2: [2023-05-08 11:23:06,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 0: [2023-05-08 11:23:06,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 0: [2023-05-08 11:23:06,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 0: [2023-05-08 11:23:06,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 2: [2023-05-08 11:23:06,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 0: [2023-05-08 11:23:06,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 2: [2023-05-08 11:23:06,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 2: [2023-05-08 11:23:06,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 2: [2023-05-08 11:23:06,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +14: [2023-05-08 11:23:06,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 2: [2023-05-08 11:23:06,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 2: [2023-05-08 11:23:06,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 2: [2023-05-08 11:23:06,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 2: [2023-05-08 11:23:06,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 2: [2023-05-08 11:23:06,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +12: [2023-05-08 11:23:06,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +12: [2023-05-08 11:23:06,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +12: [2023-05-08 11:23:06,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +12: [2023-05-08 11:23:06,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 2: [2023-05-08 11:23:06,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 2: [2023-05-08 11:23:06,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 2: [2023-05-08 11:23:06,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 2: [2023-05-08 11:23:06,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +24: [2023-05-08 11:23:06,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt... +24: [2023-05-08 11:23:06,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt... +24: [2023-05-08 11:23:06,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt... +24: [2023-05-08 11:23:06,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt... +29: [2023-05-08 11:23:06,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +29: [2023-05-08 11:23:06,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +29: [2023-05-08 11:23:06,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +29: [2023-05-08 11:23:06,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +13: [2023-05-08 11:23:06,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt... +13: [2023-05-08 11:23:06,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt... +13: [2023-05-08 11:23:06,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt... +13: [2023-05-08 11:23:06,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt... +29: [2023-05-08 11:23:06,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +29: [2023-05-08 11:23:06,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +29: [2023-05-08 11:23:06,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +29: [2023-05-08 11:23:06,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +29: [2023-05-08 11:23:06,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +14: [2023-05-08 11:23:06,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +14: [2023-05-08 11:23:06,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +14: [2023-05-08 11:23:06,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +14: [2023-05-08 11:23:06,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +29: [2023-05-08 11:23:06,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +14: [2023-05-08 11:23:06,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +14: [2023-05-08 11:23:06,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +29: [2023-05-08 11:23:06,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +29: [2023-05-08 11:23:06,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +14: [2023-05-08 11:23:06,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +14: [2023-05-08 11:23:06,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +14: [2023-05-08 11:23:06,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +29: [2023-05-08 11:23:06,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +29: [2023-05-08 11:23:06,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +29: [2023-05-08 11:23:06,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +29: [2023-05-08 11:23:06,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +16: [2023-05-08 11:23:06,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt... +16: [2023-05-08 11:23:06,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt... +16: [2023-05-08 11:23:06,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt... +16: [2023-05-08 11:23:06,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt... +21: [2023-05-08 11:23:06,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt... +21: [2023-05-08 11:23:06,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt... +21: [2023-05-08 11:23:06,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt... +21: [2023-05-08 11:23:06,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt... +10: [2023-05-08 11:23:06,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +10: [2023-05-08 11:23:06,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +10: [2023-05-08 11:23:06,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +10: [2023-05-08 11:23:06,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +14: [2023-05-08 11:23:06,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +14: [2023-05-08 11:23:06,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +14: [2023-05-08 11:23:06,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +14: [2023-05-08 11:23:06,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +14: [2023-05-08 11:23:06,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +14: [2023-05-08 11:23:06,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +14: [2023-05-08 11:23:06,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 1: [2023-05-08 11:23:06,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 1: [2023-05-08 11:23:06,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 1: [2023-05-08 11:23:06,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. + 1: [2023-05-08 11:23:06,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt. +30: [2023-05-08 11:23:06,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt. +30: [2023-05-08 11:23:06,450] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 242 +10: [2023-05-08 11:23:06,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +10: [2023-05-08 11:23:06,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +10: [2023-05-08 11:23:06,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +10: [2023-05-08 11:23:06,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +10: [2023-05-08 11:23:06,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +10: [2023-05-08 11:23:06,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +10: [2023-05-08 11:23:06,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +10: [2023-05-08 11:23:06,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +10: [2023-05-08 11:23:06,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +17: [2023-05-08 11:23:06,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt... +17: [2023-05-08 11:23:06,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt... +17: [2023-05-08 11:23:06,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt... +17: [2023-05-08 11:23:06,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt... +20: [2023-05-08 11:23:06,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt... +20: [2023-05-08 11:23:06,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt... +20: [2023-05-08 11:23:06,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt... +20: [2023-05-08 11:23:06,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt... +22: [2023-05-08 11:23:06,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt... +22: [2023-05-08 11:23:06,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt... +22: [2023-05-08 11:23:06,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt... +22: [2023-05-08 11:23:06,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt... + 1: [2023-05-08 11:23:06,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 1: [2023-05-08 11:23:06,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +10: [2023-05-08 11:23:06,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 1: [2023-05-08 11:23:06,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 1: [2023-05-08 11:23:06,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +30: [2023-05-08 11:23:06,454] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 242 +10: [2023-05-08 11:23:06,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +10: [2023-05-08 11:23:06,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +10: [2023-05-08 11:23:06,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. +10: [2023-05-08 11:23:06,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... +10: [2023-05-08 11:23:06,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. +18: [2023-05-08 11:23:06,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt... +18: [2023-05-08 11:23:06,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt... +18: [2023-05-08 11:23:06,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt... +18: [2023-05-08 11:23:06,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt... + 1: [2023-05-08 11:23:06,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 1: [2023-05-08 11:23:06,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 1: [2023-05-08 11:23:06,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... +10: [2023-05-08 11:23:06,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 1: [2023-05-08 11:23:06,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 1: [2023-05-08 11:23:06,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 1: [2023-05-08 11:23:06,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt... + 1: [2023-05-08 11:23:06,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt. + 1: [2023-05-08 11:23:06,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt... + 1: [2023-05-08 11:23:06,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 1: [2023-05-08 11:23:06,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 1: [2023-05-08 11:23:06,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 1: [2023-05-08 11:23:06,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt. + 5: [2023-05-08 11:23:06,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt... + 5: [2023-05-08 11:23:06,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt... + 5: [2023-05-08 11:23:06,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt... + 5: [2023-05-08 11:23:06,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt... +30: [2023-05-08 11:23:06,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt... +30: [2023-05-08 11:23:06,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt... +30: [2023-05-08 11:23:06,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt... +30: [2023-05-08 11:23:06,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt... +25: [2023-05-08 11:23:06,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt... +25: [2023-05-08 11:23:06,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt... +25: [2023-05-08 11:23:06,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt... +25: [2023-05-08 11:23:06,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt... +23: [2023-05-08 11:23:06,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt... +23: [2023-05-08 11:23:06,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt... +23: [2023-05-08 11:23:06,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt... +23: [2023-05-08 11:23:06,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt... +27: [2023-05-08 11:23:06,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt... +27: [2023-05-08 11:23:06,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt... +27: [2023-05-08 11:23:06,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt... +27: [2023-05-08 11:23:06,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt... +12: [2023-05-08 11:23:06,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt... +12: [2023-05-08 11:23:06,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt... +12: [2023-05-08 11:23:06,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt... +12: [2023-05-08 11:23:06,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt... + 0: [2023-05-08 11:23:06,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt... + 0: [2023-05-08 11:23:06,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt... + 0: [2023-05-08 11:23:06,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt... + 0: [2023-05-08 11:23:06,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt... + 6: [2023-05-08 11:23:06,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt... + 6: [2023-05-08 11:23:06,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt... + 6: [2023-05-08 11:23:06,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt... + 6: [2023-05-08 11:23:06,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt... + 2: [2023-05-08 11:23:06,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt... + 2: [2023-05-08 11:23:06,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt... + 2: [2023-05-08 11:23:06,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt... + 2: [2023-05-08 11:23:06,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt... +14: [2023-05-08 11:23:06,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt... +14: [2023-05-08 11:23:06,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt... +14: [2023-05-08 11:23:06,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt... +14: [2023-05-08 11:23:06,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt... +29: [2023-05-08 11:23:06,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt... +29: [2023-05-08 11:23:06,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt... +29: [2023-05-08 11:23:06,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt... +29: [2023-05-08 11:23:06,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt... + 1: [2023-05-08 11:23:06,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt... + 1: [2023-05-08 11:23:06,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt... + 1: [2023-05-08 11:23:06,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt... + 1: [2023-05-08 11:23:06,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt... +10: [2023-05-08 11:23:06,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt... +10: [2023-05-08 11:23:06,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt... +10: [2023-05-08 11:23:06,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt... +10: [2023-05-08 11:23:06,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt... +25: [2023-05-08 11:23:06,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt. +25: [2023-05-08 11:23:06,525] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 204 +25: [2023-05-08 11:23:06,530] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 204 + 3: [2023-05-08 11:23:06,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt. + 3: [2023-05-08 11:23:06,557] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 27 + 3: [2023-05-08 11:23:06,563] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 27 + 4: [2023-05-08 11:23:06,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt. + 4: [2023-05-08 11:23:06,567] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 39 + 8: [2023-05-08 11:23:06,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt. + 8: [2023-05-08 11:23:06,571] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 69 + 3: [2023-05-08 11:23:06,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt. + 3: [2023-05-08 11:23:06,572] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 25 + 4: [2023-05-08 11:23:06,573] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 39 + 8: [2023-05-08 11:23:06,574] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 69 + 4: [2023-05-08 11:23:06,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt. + 4: [2023-05-08 11:23:06,575] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 35 + 3: [2023-05-08 11:23:06,577] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 25 + 4: [2023-05-08 11:23:06,580] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 35 + 7: [2023-05-08 11:23:06,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt. + 7: [2023-05-08 11:23:06,590] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 57 + 7: [2023-05-08 11:23:06,594] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 57 + 4: [2023-05-08 11:23:06,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt. + 4: [2023-05-08 11:23:06,602] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 37 + 4: [2023-05-08 11:23:06,606] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 37 + 7: [2023-05-08 11:23:06,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt. + 7: [2023-05-08 11:23:06,611] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 59 + 8: [2023-05-08 11:23:06,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt. + 8: [2023-05-08 11:23:06,615] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 65 + 7: [2023-05-08 11:23:06,615] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 59 + 8: [2023-05-08 11:23:06,619] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 65 + 3: [2023-05-08 11:23:06,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt. + 3: [2023-05-08 11:23:06,625] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 31 + 3: [2023-05-08 11:23:06,630] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 31 + 3: [2023-05-08 11:23:06,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt. + 3: [2023-05-08 11:23:06,632] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 29 + 3: [2023-05-08 11:23:06,637] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 29 + 7: [2023-05-08 11:23:06,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt. + 7: [2023-05-08 11:23:06,658] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 61 + 4: [2023-05-08 11:23:06,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt. + 4: [2023-05-08 11:23:06,658] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 33 +11: [2023-05-08 11:23:06,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt. +11: [2023-05-08 11:23:06,661] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 91 + 8: [2023-05-08 11:23:06,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt. + 8: [2023-05-08 11:23:06,662] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 71 + 7: [2023-05-08 11:23:06,662] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 61 + 4: [2023-05-08 11:23:06,662] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 33 +11: [2023-05-08 11:23:06,665] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 91 + 8: [2023-05-08 11:23:06,666] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 71 +26: [2023-05-08 11:23:06,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt. +26: [2023-05-08 11:23:06,667] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 215 +26: [2023-05-08 11:23:06,671] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 215 +11: [2023-05-08 11:23:06,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt. +11: [2023-05-08 11:23:06,677] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 93 +11: [2023-05-08 11:23:06,682] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 93 + 7: [2023-05-08 11:23:06,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt. + 7: [2023-05-08 11:23:06,690] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 63 + 7: [2023-05-08 11:23:06,694] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 63 +19: [2023-05-08 11:23:06,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt. +19: [2023-05-08 11:23:06,723] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 159 +19: [2023-05-08 11:23:06,727] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 159 + 8: [2023-05-08 11:23:06,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt. + 8: [2023-05-08 11:23:06,731] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 67 + 8: [2023-05-08 11:23:06,737] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 67 +11: [2023-05-08 11:23:06,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt. +11: [2023-05-08 11:23:06,740] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 89 +31: [2023-05-08 11:23:06,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt. +31: [2023-05-08 11:23:06,741] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 255 +11: [2023-05-08 11:23:06,744] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 89 +22: [2023-05-08 11:23:06,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt. +22: [2023-05-08 11:23:06,745] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 177 +31: [2023-05-08 11:23:06,746] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 255 +22: [2023-05-08 11:23:06,749] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 177 +26: [2023-05-08 11:23:06,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt. +26: [2023-05-08 11:23:06,749] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 211 +26: [2023-05-08 11:23:06,754] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 211 + 0: [2023-05-08 11:23:06,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt. + 0: [2023-05-08 11:23:06,755] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 3 + 0: [2023-05-08 11:23:06,761] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 3 +26: [2023-05-08 11:23:06,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt. +26: [2023-05-08 11:23:06,770] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 209 +21: [2023-05-08 11:23:06,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt. +21: [2023-05-08 11:23:06,771] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 175 +26: [2023-05-08 11:23:06,774] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 209 +21: [2023-05-08 11:23:06,776] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 175 +27: [2023-05-08 11:23:06,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt. +27: [2023-05-08 11:23:06,777] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 223 +31: [2023-05-08 11:23:06,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt. +22: [2023-05-08 11:23:06,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt. +31: [2023-05-08 11:23:06,779] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 249 +22: [2023-05-08 11:23:06,779] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 181 +27: [2023-05-08 11:23:06,781] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 223 +22: [2023-05-08 11:23:06,783] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 181 +31: [2023-05-08 11:23:06,784] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 249 + 5: [2023-05-08 11:23:06,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt. + 5: [2023-05-08 11:23:06,787] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 45 + 5: [2023-05-08 11:23:06,790] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 45 +25: [2023-05-08 11:23:06,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt. +25: [2023-05-08 11:23:06,791] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 201 +28: [2023-05-08 11:23:06,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt. +28: [2023-05-08 11:23:06,793] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 231 +25: [2023-05-08 11:23:06,796] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 201 +28: [2023-05-08 11:23:06,798] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 231 +12: [2023-05-08 11:23:06,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt. +12: [2023-05-08 11:23:06,799] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 101 +12: [2023-05-08 11:23:06,803] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 101 + 9: [2023-05-08 11:23:06,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt. + 2: [2023-05-08 11:23:06,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt. + 9: [2023-05-08 11:23:06,806] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 77 + 2: [2023-05-08 11:23:06,806] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 21 + 6: [2023-05-08 11:23:06,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt. + 6: [2023-05-08 11:23:06,808] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 55 +18: [2023-05-08 11:23:06,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt. +18: [2023-05-08 11:23:06,810] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 145 + 2: [2023-05-08 11:23:06,810] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 21 + 9: [2023-05-08 11:23:06,810] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 77 + 6: [2023-05-08 11:23:06,812] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 55 +18: [2023-05-08 11:23:06,814] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 145 +17: [2023-05-08 11:23:06,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt. +17: [2023-05-08 11:23:06,814] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 137 +16: [2023-05-08 11:23:06,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt. +16: [2023-05-08 11:23:06,816] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 129 +26: [2023-05-08 11:23:06,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt. +29: [2023-05-08 11:23:06,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt. +26: [2023-05-08 11:23:06,818] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 213 +29: [2023-05-08 11:23:06,818] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 233 + 5: [2023-05-08 11:23:06,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt. +17: [2023-05-08 11:23:06,819] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 137 + 5: [2023-05-08 11:23:06,820] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 43 +10: [2023-05-08 11:23:06,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt. +16: [2023-05-08 11:23:06,820] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 129 +10: [2023-05-08 11:23:06,820] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 85 +29: [2023-05-08 11:23:06,822] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 233 +26: [2023-05-08 11:23:06,822] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 213 +30: [2023-05-08 11:23:06,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt. +30: [2023-05-08 11:23:06,824] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 243 + 5: [2023-05-08 11:23:06,824] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 43 +10: [2023-05-08 11:23:06,824] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 85 +28: [2023-05-08 11:23:06,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt. +28: [2023-05-08 11:23:06,825] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 227 +31: [2023-05-08 11:23:06,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt. +31: [2023-05-08 11:23:06,826] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 251 +30: [2023-05-08 11:23:06,828] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 243 +28: [2023-05-08 11:23:06,829] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 227 +17: [2023-05-08 11:23:06,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt. +17: [2023-05-08 11:23:06,831] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 141 +31: [2023-05-08 11:23:06,831] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 251 +27: [2023-05-08 11:23:06,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt. +27: [2023-05-08 11:23:06,833] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 217 +17: [2023-05-08 11:23:06,835] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 141 +15: [2023-05-08 11:23:06,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt. +15: [2023-05-08 11:23:06,837] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 121 +27: [2023-05-08 11:23:06,837] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 217 +22: [2023-05-08 11:23:06,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt. +22: [2023-05-08 11:23:06,839] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 179 +15: [2023-05-08 11:23:06,841] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 121 +20: [2023-05-08 11:23:06,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt. +20: [2023-05-08 11:23:06,841] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 163 +22: [2023-05-08 11:23:06,843] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 179 +20: [2023-05-08 11:23:06,846] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 163 +24: [2023-05-08 11:23:06,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt. +24: [2023-05-08 11:23:06,850] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 195 +20: [2023-05-08 11:23:06,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt. +20: [2023-05-08 11:23:06,851] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 165 +30: [2023-05-08 11:23:06,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt. +30: [2023-05-08 11:23:06,853] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 247 +14: [2023-05-08 11:23:06,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt. +20: [2023-05-08 11:23:06,854] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 165 +24: [2023-05-08 11:23:06,854] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 195 +14: [2023-05-08 11:23:06,854] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 113 +15: [2023-05-08 11:23:06,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt. +15: [2023-05-08 11:23:06,855] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 127 +15: [2023-05-08 11:23:06,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt. +15: [2023-05-08 11:23:06,856] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 125 + 5: [2023-05-08 11:23:06,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt. + 5: [2023-05-08 11:23:06,856] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 41 + 5: [2023-05-08 11:23:06,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt. +25: [2023-05-08 11:23:06,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt. + 5: [2023-05-08 11:23:06,857] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 47 +30: [2023-05-08 11:23:06,857] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 247 +25: [2023-05-08 11:23:06,857] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 205 +14: [2023-05-08 11:23:06,858] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 113 +15: [2023-05-08 11:23:06,859] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 127 +15: [2023-05-08 11:23:06,860] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 125 + 5: [2023-05-08 11:23:06,861] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 41 + 5: [2023-05-08 11:23:06,862] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 47 +25: [2023-05-08 11:23:06,862] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 205 + 1: [2023-05-08 11:23:06,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt. + 1: [2023-05-08 11:23:06,863] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 13 +19: [2023-05-08 11:23:06,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt. +19: [2023-05-08 11:23:06,864] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 153 + 1: [2023-05-08 11:23:06,867] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 13 +19: [2023-05-08 11:23:06,868] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 153 +15: [2023-05-08 11:23:06,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt. +15: [2023-05-08 11:23:06,869] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 123 +10: [2023-05-08 11:23:06,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt. +28: [2023-05-08 11:23:06,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt. +28: [2023-05-08 11:23:06,872] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 229 +10: [2023-05-08 11:23:06,872] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 83 + 9: [2023-05-08 11:23:06,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt. + 9: [2023-05-08 11:23:06,872] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 73 +15: [2023-05-08 11:23:06,873] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 123 +13: [2023-05-08 11:23:06,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt. +13: [2023-05-08 11:23:06,874] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 109 +10: [2023-05-08 11:23:06,876] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 83 +12: [2023-05-08 11:23:06,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt. +12: [2023-05-08 11:23:06,877] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 103 + 9: [2023-05-08 11:23:06,877] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 73 +23: [2023-05-08 11:23:06,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt. +23: [2023-05-08 11:23:06,878] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 187 +28: [2023-05-08 11:23:06,878] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 229 +11: [2023-05-08 11:23:06,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt. +13: [2023-05-08 11:23:06,878] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 109 +11: [2023-05-08 11:23:06,878] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 95 +12: [2023-05-08 11:23:06,881] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 103 +23: [2023-05-08 11:23:06,882] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 187 +27: [2023-05-08 11:23:06,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt. +27: [2023-05-08 11:23:06,882] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 219 +11: [2023-05-08 11:23:06,882] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 95 +14: [2023-05-08 11:23:06,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt. +30: [2023-05-08 11:23:06,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt. +30: [2023-05-08 11:23:06,884] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 245 +14: [2023-05-08 11:23:06,884] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 115 +24: [2023-05-08 11:23:06,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt. +24: [2023-05-08 11:23:06,885] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 193 +22: [2023-05-08 11:23:06,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt. +22: [2023-05-08 11:23:06,886] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 183 +20: [2023-05-08 11:23:06,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt. +20: [2023-05-08 11:23:06,886] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 167 +27: [2023-05-08 11:23:06,887] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 219 +24: [2023-05-08 11:23:06,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt. +24: [2023-05-08 11:23:06,887] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 197 +21: [2023-05-08 11:23:06,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt. +30: [2023-05-08 11:23:06,888] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 245 +21: [2023-05-08 11:23:06,888] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 171 + 6: [2023-05-08 11:23:06,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt. + 6: [2023-05-08 11:23:06,889] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 53 +24: [2023-05-08 11:23:06,889] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 193 +22: [2023-05-08 11:23:06,890] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 183 +14: [2023-05-08 11:23:06,890] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 115 +20: [2023-05-08 11:23:06,890] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 167 +23: [2023-05-08 11:23:06,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt. +23: [2023-05-08 11:23:06,891] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 191 +24: [2023-05-08 11:23:06,891] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 197 +21: [2023-05-08 11:23:06,893] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 171 + 6: [2023-05-08 11:23:06,893] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 53 +18: [2023-05-08 11:23:06,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt. +18: [2023-05-08 11:23:06,895] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 151 +23: [2023-05-08 11:23:06,895] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 191 +28: [2023-05-08 11:23:06,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt. +28: [2023-05-08 11:23:06,896] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 225 +23: [2023-05-08 11:23:06,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt. +23: [2023-05-08 11:23:06,898] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 185 +18: [2023-05-08 11:23:06,899] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 151 +28: [2023-05-08 11:23:06,900] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 225 +23: [2023-05-08 11:23:06,902] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 185 +17: [2023-05-08 11:23:06,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt. + 1: [2023-05-08 11:23:06,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt. +17: [2023-05-08 11:23:06,903] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 139 + 1: [2023-05-08 11:23:06,903] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 15 +10: [2023-05-08 11:23:06,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt. + 2: [2023-05-08 11:23:06,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt. +10: [2023-05-08 11:23:06,905] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 87 + 2: [2023-05-08 11:23:06,906] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 19 +12: [2023-05-08 11:23:06,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt. +12: [2023-05-08 11:23:06,906] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 97 +14: [2023-05-08 11:23:06,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt. +23: [2023-05-08 11:23:06,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt. +23: [2023-05-08 11:23:06,908] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 189 +14: [2023-05-08 11:23:06,908] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 119 + 1: [2023-05-08 11:23:06,907] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 15 +17: [2023-05-08 11:23:06,908] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 139 + 1: [2023-05-08 11:23:06,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt. +13: [2023-05-08 11:23:06,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt. +29: [2023-05-08 11:23:06,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt. +29: [2023-05-08 11:23:06,908] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 235 +13: [2023-05-08 11:23:06,908] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 111 +25: [2023-05-08 11:23:06,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt. + 1: [2023-05-08 11:23:06,908] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 11 +25: [2023-05-08 11:23:06,909] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 203 +10: [2023-05-08 11:23:06,909] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 87 +12: [2023-05-08 11:23:06,910] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 97 + 2: [2023-05-08 11:23:06,910] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 19 + 0: [2023-05-08 11:23:06,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt. +31: [2023-05-08 11:23:06,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt. +31: [2023-05-08 11:23:06,911] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 253 + 0: [2023-05-08 11:23:06,911] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 7 + 2: [2023-05-08 11:23:06,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt. +23: [2023-05-08 11:23:06,912] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 189 + 2: [2023-05-08 11:23:06,912] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 23 +14: [2023-05-08 11:23:06,912] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 119 + 1: [2023-05-08 11:23:06,912] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 11 +13: [2023-05-08 11:23:06,913] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 111 +29: [2023-05-08 11:23:06,913] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 235 +25: [2023-05-08 11:23:06,913] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 203 +17: [2023-05-08 11:23:06,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt. +31: [2023-05-08 11:23:06,915] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 253 +17: [2023-05-08 11:23:06,915] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 143 + 1: [2023-05-08 11:23:06,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt. +29: [2023-05-08 11:23:06,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt. + 1: [2023-05-08 11:23:06,915] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 9 +29: [2023-05-08 11:23:06,916] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 237 + 2: [2023-05-08 11:23:06,916] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 23 + 0: [2023-05-08 11:23:06,917] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 7 + 1: [2023-05-08 11:23:06,919] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 9 +29: [2023-05-08 11:23:06,920] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 237 +17: [2023-05-08 11:23:06,920] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 143 +18: [2023-05-08 11:23:06,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt. +18: [2023-05-08 11:23:06,923] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 149 +20: [2023-05-08 11:23:06,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt. +20: [2023-05-08 11:23:06,923] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 161 +13: [2023-05-08 11:23:06,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt. +13: [2023-05-08 11:23:06,924] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 105 +13: [2023-05-08 11:23:06,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt. +13: [2023-05-08 11:23:06,925] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 107 + 9: [2023-05-08 11:23:06,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt. +18: [2023-05-08 11:23:06,927] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 149 +21: [2023-05-08 11:23:06,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt. + 9: [2023-05-08 11:23:06,927] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 75 +21: [2023-05-08 11:23:06,927] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 169 +20: [2023-05-08 11:23:06,927] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 161 +21: [2023-05-08 11:23:06,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt. +21: [2023-05-08 11:23:06,928] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 173 +19: [2023-05-08 11:23:06,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt. +19: [2023-05-08 11:23:06,929] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 157 +13: [2023-05-08 11:23:06,929] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 105 +13: [2023-05-08 11:23:06,930] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 107 + 9: [2023-05-08 11:23:06,931] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 75 +21: [2023-05-08 11:23:06,931] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 169 + 6: [2023-05-08 11:23:06,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt. + 6: [2023-05-08 11:23:06,932] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 51 +21: [2023-05-08 11:23:06,932] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 173 +19: [2023-05-08 11:23:06,933] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 157 +24: [2023-05-08 11:23:06,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt. +24: [2023-05-08 11:23:06,933] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 199 +16: [2023-05-08 11:23:06,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt. +16: [2023-05-08 11:23:06,936] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 135 + 6: [2023-05-08 11:23:06,937] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 51 +24: [2023-05-08 11:23:06,937] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 199 +16: [2023-05-08 11:23:06,940] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 135 +30: [2023-05-08 11:23:06,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt. +30: [2023-05-08 11:23:06,943] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 241 +10: [2023-05-08 11:23:06,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt. +10: [2023-05-08 11:23:06,945] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 81 +30: [2023-05-08 11:23:06,948] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 241 +10: [2023-05-08 11:23:06,949] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 81 +14: [2023-05-08 11:23:06,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt. +14: [2023-05-08 11:23:06,955] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 117 +16: [2023-05-08 11:23:06,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt. +16: [2023-05-08 11:23:06,956] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 131 +12: [2023-05-08 11:23:06,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt. +12: [2023-05-08 11:23:06,953] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 99 +12: [2023-05-08 11:23:06,957] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 99 +14: [2023-05-08 11:23:06,959] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 117 + 9: [2023-05-08 11:23:06,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt. + 9: [2023-05-08 11:23:06,959] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 79 +16: [2023-05-08 11:23:06,960] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 131 +16: [2023-05-08 11:23:06,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt. +16: [2023-05-08 11:23:06,962] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 133 + 9: [2023-05-08 11:23:06,963] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 79 +16: [2023-05-08 11:23:06,965] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 133 + 6: [2023-05-08 11:23:06,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt. + 6: [2023-05-08 11:23:06,967] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 49 + 6: [2023-05-08 11:23:06,972] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 49 + 0: [2023-05-08 11:23:06,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt. + 0: [2023-05-08 11:23:06,973] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 1 +19: [2023-05-08 11:23:06,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt. +19: [2023-05-08 11:23:06,976] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 155 + 0: [2023-05-08 11:23:06,977] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 1 +19: [2023-05-08 11:23:06,980] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 155 +27: [2023-05-08 11:23:06,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt. +27: [2023-05-08 11:23:06,981] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 221 +27: [2023-05-08 11:23:06,985] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 221 +18: [2023-05-08 11:23:06,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt. +18: [2023-05-08 11:23:06,988] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 147 +18: [2023-05-08 11:23:06,992] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 147 +29: [2023-05-08 11:23:07,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt. +29: [2023-05-08 11:23:07,000] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 239 +29: [2023-05-08 11:23:07,004] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 239 + 0: [2023-05-08 11:23:07,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt. + 0: [2023-05-08 11:23:07,007] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 5 + 0: [2023-05-08 11:23:07,013] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 5 +25: [2023-05-08 11:23:07,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt. +25: [2023-05-08 11:23:07,041] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 207 +25: [2023-05-08 11:23:07,046] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 207 + 2: [2023-05-08 11:23:07,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt. + 2: [2023-05-08 11:23:07,090] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 17 + 2: [2023-05-08 11:23:07,095] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 17 + 0: successfully loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b12bc4subopt at iteration 0 +31: time (ms) | load-checkpoint: 12513.64 + 0: estimated model parameters: 2.81512448 + 0: estimated model parameters without embeddings: 2.6755328 + 0: [after model, optimizer, and learning rate scheduler are built] datetime: 2023-05-08 11:23:08 + 0: > building train, validation, and test datasets ... + 0: > datasets target sizes (minimum size): + 0: train: 1 + 0: validation: 102400 + 0: test: 102400 + 0: > building train, validation, and test datasets for GPT ... + 0: > building dataset index ... + 0: reading sizes... + 0: reading pointers... + 0: reading document index... + 0: creating numpy buffer of mmap... + 0: creating memory view of numpy buffer... + 0: > finished creating indexed dataset in 0.050007 seconds + 0: number of documents: 3133972 + 0: > dataset split: + 0: train: + 0: document indices in [0, 3133972) total of 3133972 documents + 0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_doc_idx.npy + 0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_sample_idx.npy + 0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_shuffle_idx.npy + 0: loaded indexed file in 0.108 seconds + 0: total number of samples: 731002 + 0: total number of epochs: 1 + 0: > building dataset index ... + 0: reading sizes... + 0: reading pointers... + 0: reading document index... + 0: creating numpy buffer of mmap... + 0: creating memory view of numpy buffer... + 0: > finished creating indexed dataset in 0.028542 seconds + 0: number of documents: 364608 + 0: > dataset split: + 0: validation: + 0: document indices in [0, 364608) total of 364608 documents + 0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_102400ns_2048sl_1234s_doc_idx.npy + 0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_102400ns_2048sl_1234s_sample_idx.npy + 0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_102400ns_2048sl_1234s_shuffle_idx.npy + 0: loaded indexed file in 0.080 seconds + 0: total number of samples: 169955 + 0: total number of epochs: 2 + 0: > finished creating GPT datasets ... + 0: [after dataloaders are built] datetime: 2023-05-08 11:23:15 + 0: done with setup ... + 0: training ... +31: time (ms) | model-and-optimizer-setup: 29060.42 | train/valid/test-data-iterators-setup: 3926.52 + 0: [after training is done] datetime: 2023-05-08 11:23:15 +31: ----------------------------------------------------------------------------------------------------------------- +31: validation loss at the end of training for val data | lm loss value: 2.562510E+00 | lm loss PPL: 1.296833E+01 | +31: ----------------------------------------------------------------------------------------------------------------- +END 3476831: Mon 08 May 2023 11:24:57 AM EEST diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d9f8771b6619a4b9ecd23da567f292f9cb22b80b --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fa3dc73ebf2f9ef2056b432953c9ed70d58245a4f1431a169e68b594cd6e451 +size 131963479 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bd76c4bd926d0eef3a6ce3284c86ac747d195b33 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91a396d9b9f99c3b6b6dfea046b158cc690ce8db1901b030973361ce136d0139 +size 131963479 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8849f66e03930077bbd73d018d17fdfc282efb5b --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b294188951f6a6d5cf6690690202e85e644c02fd25d3b811e44b5594cb71efb +size 131963629 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..35b9d51a4063bd74bab38945ca82130b7e64b556 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f510e0675143ac105d25df43a5f9a19de9f6a00ff521d6888aee306dfb42386 +size 131963629 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3150f03ca64e3098b2170b5d492db1f7d38e8f4c --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a3a6b6fff70b4807a8c37ae473d0b12fcc452e060a842af7242f1c5bd9afe2c +size 131963501 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..41dbb5227aab0bc60cdfd5f0b49b66be9a5b8ad3 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e244b6f7fa123ce9531a85a8719b1b397bcd33a3340a01d2fba0a0cffe32028 +size 131963501 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..11e298835510058582243a356785d8b7a992a07f --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8a94e6dfabc3dab13c214febebfda06561903695da66c9f5626ca1193bdfe18 +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a389a0182fa1868afced44cc5e9c97c8c7a145b1 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d397ab5ccdebe9b7c9386176b6e348aa728de35b98949a7e8bee8d63a113cf8 +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f19c08f13fb06c11999af48ab087ff4d0f9a46f9 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:851af5394afbfac587ba63f720dd212a38bdba77b3e29195234753304b4756b3 +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dcda8df861fb4163bf52f1e34ddb1a9ba335d586 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c7c7d1eb13dcafd0327c59163c08ff22ac5f5b5ded38c9b3a66c5a63a9f3a7e +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..39e32b3beed9d6c66e0947287f1f985aadfd9dab --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03b49a5e41cc71b8e8a57a775d783ce173597c31b1583e5ab83d85a246ae27c1 +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2cbbbe0a1ba9c37356aa7c78f55992a191e45fbe --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f79a628056b317133d659228776911c42adaf90a4d0cd6235610243a721d7d60 +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ed8721aa9db17d4949410ab925fff0457afdaa3c --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdd204dec71e55117b86e3a555867791392e2ae15cfeb32864f17d92e41a538a +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fc6738988269ac52c9a3769ac6dd6a2cc9d68cb8 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6dca3efd5609a5f0999a008c9c16f09f84ba084effe6a2ad5c8e27c02e61a3f +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..98d367b200490f3cd376d88c268bfd4def3bada9 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0233f4b0ebc6a6ebaa802f5655ea45a53010e4404d3d9ba8e14ea7db9c1245c3 +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6bbf287c2789db76db077ae91ab60848ffdf4dd --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d016804c7d56b1fe048c60b6d153d61c677ce1f3155f799fb1369f924b46d55 +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..92db692b1811260c130027b6669d00a532f5c9d1 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb9028158f328b7e0ba8c5f547ac25adc424637ddd20e8cd5e94ec9290edb643 +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..021bc8c5895ef6fb2803630da532a89f2569d123 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:beb82b8dc74fc503b71dfeb0c08d43956d059dd06c0cd2cf74a8d7e6119f924e +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ac9e3f16bfed81e48aed9cd2729956cb38dad7a --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:73924f800810cf80f9c3405e269c3a29598280180711c1757098c3dea5a43924 +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2311405f5f52823b1687d66ebd81529847e18ef2 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e561ae3b9fbea3871e830de957de35e6c3e8fab268f2322b698a97ae9389867 +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c58eedbd8d860fcaabc366832c92305c9d8aa6f --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9269861d5cb175159f63941db33daefff850e146005a7222514cae5e578c8d69 +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e7ef4efe3d7eeb09c89df965a9801d8393adf7b3 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55deab6e5fb823562e6880b3cdc34862f11050c98c3d8265cb3277b2614a5564 +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..645960a6cc7e9f96c532d6b194eeb671ca36513d --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eddca8088d3622feae70c6abd59e4ee7e383e0cff929910f8e4d9999e7159364 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..efa650e1af3941844c0c21c6c6e6a67e7a0984f9 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3666e26f481737bebb0420dbc23ea5b1036f7a8251db645dc32f02699aa50de +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ac883f2d5debc13c4fadb866fe501f6d7bb8e283 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c787e1b55dfd48d3c91339c73ab5da79af641c7ae37a97283e521758aba61081 +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a0b1dbea02127c939b172bd33b3efa67c0d59573 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:309cbc0c8f8bc1a49cff4e3171199b2b593bccf37fb44d711af9c0fb2f75cfec +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..227a8af2d3c47ebd8b15579d13df84782a2e2298 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8093a55a5ec750cdd4d358c5edd4aca99d04729486d1d7c2f625feb6e3070f6 +size 131963629 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4a78309c59bbab013cadecee17db53f7a5b161ce --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3753667d15fbb97179d9a81d863a5df6f365f9707e7a02b2b7f04a964af62594 +size 131963629 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2eacf07148089af1c232db838d92a7f86840a966 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f7075aacb977fb397fc2fb2f6c269b3a782852e3c709751f6e2187957dd8aa8 +size 131963437 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c502585b2045cd92ff7c60fed6a9a413ca30b48c --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec510c2373c2f4357ab8ec7c67d10157c3800242a37de04db81d4aa389cebffc +size 131963437 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fc3ad9c5cb78cd660f8c1c1406eb889b663bfafa --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a19d70a48be77eb30789d4a91d4aef20ee832fab23218d09ca131cbf4b89e1c +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..104e8174bb35b8b0ac56ac596efeddd060318e13 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:145e921e0ed49ee53d5377951f93cfcef7807fd233e233968fcff536afadc23a +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..597fdda192de447569c47538bdbe3319584d5f8b --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54cd77c563ce8d3d72c1d4592cb4fe3dbc7870725728de3bb5c778eab6254d6e +size 131963629 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..af41e5cb4c752a06c313e77c4c39b00f804a3f5b --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60e67bed7577745029f9ca37e4f37cf1c3688c68ae6aa406f97c085cecef7c58 +size 131963629 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..510d788e0ec013ef066bf7348c9199ccee3660e8 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c321e5dc3f559b355b80308f5f6db63f82591de374700cb7ac33d20b09f2a7e +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..63cc0ca48d8de06fc2800ef3eb7a07347dec5ebe --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c95bc448362f57a738c555663f060ddafdfe109467b61c081016122eeb92a177 +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ffdfdf532f4ec228300be0d30975f30288f161b1 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:921b5c35cf0b3f5989e74643d853fe382feaae31e8ded7cc8da1ab42f56c044d +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b71367ece8e6bf70997d306c3105cc29ab68759f --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28f15685e16ebdeb0c631746dfb88a005a953be116d4030eb26f958b6c910e95 +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..673f7e68de332e36999b3f6011b189b485d8b9df --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2362829fbc4aff3b4f90a0aed76e6da7c3a616351e65b8a8da3572337163ece3 +size 131963501 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8adbe4d9f50942def0efe6dec977e5e64be993c4 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5180ce8e4957c5edbf1b1418834e932b5e31e135357fa2eb6fc4f0cc85b009e6 +size 131963501 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b62a85ede0fd093761764b2f5f72daf32469fed8 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f37409d204c81c6a893e0889ddfbbf14d689f5e89c14b3b7688fdd8bd29f6067 +size 131963693 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3cac0a91686e183a484cf46c40d209044aeb106b --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33b72861c4a61584e8fdd48f85583f3237d28626dad7ff11fb65f03cb30182f2 +size 131963693 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0c7922f36dc024ec1493a7b624742d26cfb3b85a --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1ffdeeda0e93acd465b65aecc45ffdad44e996ef75e2aaf47584a792b724cf3 +size 131963437 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2cd388c68b6b00c6630ed0fe0cdf34eccf7c657a --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed772f0dac18f03e977b587f6a0f42e35011578692c3190c457b40063f26b084 +size 131963437 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c9b2ae7eb239a44bf66068261b5365d0ccf01d1 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cadbf12079b060489359ac97786c96fb8ce841e2a0d26486f09a98f99a31d6b +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..23511973e7ddd6172796ea57f22b0807fc61e26f --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9106fc45d47d729d25dce75d95c67466c0e770227bc140d835efdf1b1fc9e47b +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe8232650c4adae025c994890e5e1b58c0b5facb --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e39d0001b396a853295725a8c7fe42e706f4df81501cec1f687d68f575452095 +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..88867f89864c8fa8101abadb8db1cf8ddd0a1f4b --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61ff0cedbd3ca382e86038d5ca62bfd15d978c382311f5710a7b03bb2612e25f +size 131963565 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ecc5454a9dff49e502bf4fb65a63c5422c2ef1f5 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7115f004b1bcf8f036029842c49f64ff1c97c5902e97614648253dea8204202 +size 131963629 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8851f5499e86475e8c369b55397bf212a402c7ee --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c46194d933e5bad08dca0e076bdc13e723c921d8682937b5b9d85ecb4bf4602a +size 131963629 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..17fe85184213fd044d2ed8c3f0bce9400ebc4448 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab525ac99b96f5753f10f6449c70caa6b125971f311a799d0db302c53c75918c +size 131963501 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c9dc8bef0f1287af0afb2128fb6b6515846d315c --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:768cb894eb5dc5661679493c1f7a3c5d9c9724eafad5f334e7b18f0c782e7033 +size 131963501 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1408b1a724a85ae6c4eef937bb776da18ab5eb5d --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15579c5f7b40f6fa27ef2bbc48fa3d6ee8c404bc4b246f4d7eab1a8e13fb51a2 +size 131963629 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..92b6aa566e9fd4d69654363bbe09aa24fe3ce207 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b657f15812fa26995b166b9a813c353ce366a9c007cf27b8af09ae168eac13ab +size 131963629 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f1b9b15dbbf6fc5b08d5aea68c81cc51016536ed --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6eca54bb6b85ecc90ad71c7b6fb3b744a960798c9359852429639d888b126c2 +size 131963501 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..98829bd7ec0c70eea82dd9c8a1e5e826306d9d9d --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd50c5af4c43667bd258a000d90e6eec372b558828cb2b70331dadef867255a8 +size 131963501 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3747fb9eac187989015de02821be6fbd718d8972 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a957e4ddb29e81306165e0a269ccba0c842ee928c6084ad7a8ca4cbb0003ad77 +size 131963693 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..50909fdc4c22d8c1672429b22c724b9d43e6498e --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bb2976d5435552968535495d4705f8fca081dba19efc88768f7f9a63658733b +size 131963693 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..29ad640252e1562432f1918572676bffbbf91e31 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b66ee817a54e428b1ee7d74ab9111083401d49a6499abf41cacccad5a170b54 +size 131963437 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9ef5133d46dfa10e7ad63cbb8015dc95b0576854 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a527dd2efc49bed0894809370461ab8341a2be1cbffa5324b58c935ef107e906 +size 131963437 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..04a58a898fcf1a8913c4261ad69e5a554b34a9f6 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:383690eb590f4b3c5192ca91ed35779b4f243c332802d2f03733f326d947bb28 +size 131963437 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..361f4a97e1bc43f43e6a4c314b5b9af7d5c53908 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ca1a8daed08094207d01287f22ebb0e91d330b4389dd7ec3e006f10f3f86871 +size 131963437 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..93a21b8debb851a09b27ec271cf3987d2979035f --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c73ae27857203ec31c9bd6a192eec2825cdc672d80493a9baed14a896bae82b +size 131963682 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..adb37485d28a6cbf06e7c1fb4dd420ff39626782 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a44729a642548ce17f978384eca103e21cad1e028453acbc534bd4bc2d7bc85e +size 131963682 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..99e3e1aa425fa287fd65aa916aa86600b6e2c4a3 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72059e0a22db2c9fd55a942f59dc40b68543a15ac3143b0c1a9d197feb40ab27 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0a8f5b1ffe95b9dea11b6dda5616ff142517f5a6 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f76fdb74b0444480a552e13310e51ca0f257b57dfb630b4331d5e1d9c0c85c3 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb51d5fe46d011493b7e83c8c638ac795b73f4d6 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e01715d414bcf846df2e2af6a94ec8c88ccb1db555bed896bdba9a05f2a4d957 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eaabb3e9a36f3d8fd37692ef482e8d461ac51bab --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:040b23ef1ca84a2bc3bfcfe6068815fc07f6e4737fa9f729e350516db2e37eba +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c59ca8c8c83a8df354e29902c6d8f23ee7499a35 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9418bc8eebb32595b3aed466e267447afbb42bcdea784051dbec6b34849bcebe +size 131963426 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0a10748c324aa3298486bd63e880147bbbab5b0e --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f8e69aaa26b949c39901fb9d6cc0b4c02282ec6963d90af89deda62929a9648 +size 131963426 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3782424ca5d929093e19a5816bf14112a748d6b3 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88c3805c5fdb29c4878b1e547a5b4afbb43c834280b9b1d5f4178a540fb1fb13 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8e5edc9801b2b159827492a7a8e4d246e4a8bc7a --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d10059859d45d8422ef224a0c09d5c3e722a540ca7730e327c8bf47b5800d31 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a006acc4e72731ee651d89e61b21f40b33e6c870 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90eb159dfc5c3a3fcf920c12619936a13a65988d0cf675ae08d8277603c9972f +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..71908c75c70e783caa2292fdfcd6ed87732502bd --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0639d568973c6db3336cae76729f6ad52ad5fc71529157db62308173476e4a92 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c0c810d9d07ab558f5cc089198d098aa49626789 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d3ae63bd2a5e24d7bf5720ffcc59351ebbc5153dba78ebeb976a577f734f461 +size 131963426 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9866126c6fe668327e056baba98b0686f7c7947b --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de8256ea680960d250973c9868575481fc48cdabd7ce9397f490f475873f877d +size 131963426 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ef168181630d075b4b90ffae1b93f3c69f1d60da --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81c557f24063d30a89d518896f09971defa7c19dde85acbd5b1160e86de4fe09 +size 131963682 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c04d35bfc5022d2ef22553aa0ca5f6bc846bfd08 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:522f096f139aef80028e519faef8e2c75618e5d2d7aa0f698f83103548f8b1d1 +size 131963682 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2ccc9f2f6fcb58f0a79b859aeab900d0fd5b8db --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c55bc4b013273389751bbda70dbc03ae0e94147b72cbc886d3f46068dadd511e +size 131963479 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0f5b7092403308e83710218533040d00f7d67652 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36236e448e31c7408431c7c756a941de7818c29f60270fb39b0cb391ddc8478e +size 131963479 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..df4dca5c992bdbec04df57d7becb72e39e83496c --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fea2569eb093e889329d44b5cdb7a21da44eaf9166b8d4d7df6926bcc32fccce +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2cc7e23881d415e4d6203ac4d2d7b2fac1216e79 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:517a84f192a692dad7565b4633be5496c7b8ca47c7e1ce169b4871eaa043f3a0 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..41f0ea1da796bf37f339571148f28fe2d1eea3b7 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a985083e6c4affb5a4ec98770d650aa6361cfc8f149d8848a2840dfc36dce5a +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..465b4143a6078916e8268501f9c301f9dffdf19d --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca6cdeffa63a1dad506cf4840ce030b364016a108c526b39fec7b5665118a7ff +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a872bdae0f1ac139c3cd7f4feb02c71b263acd31 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a47ee1f0912b23370df48bc28e0feeef005f442ce61c3a4ca1d6f98d67aa9be +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c087b8435e630af5e24bfabcbb912317b2582ba --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cfbc89388500d416c8074025f39776876441dc3aefa7271222a4e51d27de5519 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c521550ab38aadd9db49f767909e4c5cdd6b7ad --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07393d85dd5d0621150b4228bb973de5cef618bc06238714c7488eb5b278e717 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4a00083eb9e93d1fc3048d78919c8086c61e7950 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74bcdbe4a919d729c4f0fe230a580d7f0593e8c5efb811fc58aaddddc15be552 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0949fab27823a1eafb422a628ab4be3f291a53d9 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b49783e9237662e4e6cac5d479d741a5897b44078d7820c200b0bfc7ee24f05 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c1ddd97992d5a55edd5a4658171eed232feb1405 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f27a235777a9647bcad9471dd83eea114d6dee731e59bcd0f470f935da06f3f9 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..36f86540cc0924b1de1d12005d94cbe8cc49885e --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e0cdcd2741dba616e452c0c49bb4540ac578a64e723d47056da88a44118d7e1 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eecb3929c0290515327b4e9cb0e04afa284e3b76 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f54fc7badbeac350f2f95e86a6bdf13716f79215b817cb39b0f8dde25244ede +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a13d3c33128251fae68b75ad196b623cc2a59422 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67b428aea550ede01dcfc0bbecb1096560a67f524a0353d407d8f90c6580eba2 +size 131963682 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7d58310c8deaf758bc70e2f6d54132292fc52c3e --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75a043fbff45eed0862d329d5f5dbf7dee8708d49b54a360560df4532894e878 +size 131963682 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f421aa29d5fe93fb151ed30900d622752909803 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3ffddd1b8ce485e99da5d6a1f78807f53989cb6034c3b7e4a72bc6c821b3535 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eec3a074aa3277498576211924b746df664545d8 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:475bb021ab3ab4c91032efe5bb87ad59bb9145b683bce86357f7426153a6ec66 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..832be99d8d9f26e835539460bd905f2f05ffd072 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:850e5698b17e593b779235789261574a09f66df2588715ff84780d5d0970d772 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d2cfd2ae7ae771cd21cb012198ae2aee898b949 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f71bd52e3f1afa058022e14e70e196954f312bfaa4ab9d836b7fcd17f368d77a +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9213c594c734c590afb40e569db2a18061371a83 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d261b62ca13d2d3fc582187ccd71b266bd705b08f095e3def4f75cf604e1727b +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..90c977448678dfd7032a807d8a87759b40a5cc1a --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4a2c12b8ba1a0ea5324420f65a3995f56a16fb9105de10f2ae88ab2a1a3121d +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dd55618738d50801608031ea1d1ddab7940a494f --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cf211ff2844704a2c3c56de9a797ad2dad5f3fad1e74e0716c792e8ae0fb658 +size 131963543 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..61074af4ddcc022a57c8573eaec933a751b3f7da --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49fb3534ad73f5b9570347054cde72a39300188d07c591050cff4cd2bf6bfcc1 +size 131963543 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..462a89d191773cbf0c5589c7a6d213bdf22a3335 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72a7f27772852b83dd6300ff055a6b9b8fb1b59d1106888b45887af34bbd078e +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d35b0818dfa642cf023aad07521935685d92f28f --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e515a1af81a1d88d88434a867f1ea86aa6d4cfb34f44b918f6d43be8dc59353 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b6d03f28d05d29672cd39b5a26557df6f57586e --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f5187a9ee8ee3983629197b46946e97d225bef89cc056c82b9beecd0af7291d +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7e9766a808afd4c61e464b3f7787f06e630ad60d --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84117386edaf2d3b1416c3497d2cb88ecb80007c1a8c2dad46d435e1fe677f27 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b581da2c5fb2ffba85056430e085a0b83d29e03 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:154f3bd4f0ec9c36cbd248af3d031544a9eea9f43781aff51772cdc2f22bcaa8 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa9d11ef46b3130cfcb0a3e7206082e49372e7b7 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0526f76be6cbe7dfaf1c0d71aa91a816f519c2a3dcf351190f0f88c530674b1 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2c9e29a3b3b26bdc3c53b7da560640b4aa807222 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afa9f44268dd21a3f8b12083e61003fe2897283ae53131708c72a76dc6f930fe +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e0e84e90f87707150750db4ab46b0fa24066491c --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:618e2c2b8ef2d5f830396dc1b1ecb70449c509964b24ec8ed2f0608e3494fb07 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a51557e7f024d8f963e7c76f392be493b641fd6 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4217740357362c08c0748e812ed23313c69b5c15a10def4191bad31ac9aa7a8 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c33a430f3f04b97dbcdbaa56e155c1dcbf01e255 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9cb079a1350f7b03b4903a5a6f14ff32e12b22051ff6342f8d8631d1f040510 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ac3fbc312087dab5a8f3b8470231f1d685990d66 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c99f12188c53ca4b9cf77ce95da3135329d316c6af40e85bcab19636f0943a3f +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..537cbd6ecc2ad9d5ddab93f4b6114eab6fc4ed2e --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ada9ffc7c36ea10555fa5e68bab4a3da35344556719ee1e384a3a82b01c83228 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..50841b058bbdd6c5ea9b394ae7d7e0bee15cc5cc --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7d00c026e966b1c2ca8cdc035c330e44dbd55646251e9c37c466ea0a64e62cc +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c8d1428592edc9fba1a43efb6d756f3febfbd2db --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:47b506e7d6609ba99d65c3ad47dbe7f2c1d0695ca4ced1377d98d15a535c1c85 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..48810514d4ad356603eb72daf4aef210d6a175d5 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84bf08be5c6d2a443f7202512298e14883953c8580b21ebb90aa6cd0f0c72430 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fae52dbb55982a9a8865699d80e66060cb880d1d --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbc7db75203f2e6d0471193fa2f5fec3dbbfbaee0713d8dbf4c9135ad1f442ef +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7cd0ad541ca6f0e9cf2d878e0d96290a1f965b69 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca5cb5c086a29ea6067328e84df1fcaff2a205ce1185c69c9eb41841b044d0d8 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a1308195acc99740d8004b8af0fb1eebd6f2d65e --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9baf0572d56d11cd0cee7729aada503f319bdfa907142be58d3c3badf468d110 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..86c57434227e87a3f22a94fbf862b1a9ed551354 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53fbfc33ceeecab5fb5dcf2812be4de69b1b7bf682441ebb34864a13b591ac8e +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb67e35872564d201e93290f2cd3ef919ac0ca35 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:badc20b955709f70d2c7f37d88eba14cafc106a6e65d895cba2a356c5bbdb813 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4059a3fa5ef9fa56dd55400b67dd183eb2c80e9c --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e2dd896b82d156a9a69638743daf97f137e009dcf4d6ae82bb0c56c6955c414 +size 131963479 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9627b9b36b51978e349e4279201b2c3dfc3cfeff --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d373d247403be886baae3302fe9af807342a0fe1fdc2982e769388c700b05ef2 +size 131963479 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ebff56d6a1da3618701db46682a536a5f1bd5434 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f066aa38d00bdb571aad7b37d57c43667c412a44dd38c22dad02417cb15a051d +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..51b77503735225875151fe21fc55a0fdd7f2a1f3 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bdcc7a2654b55a4f36e0d664ac8889684b411f47f9ab8e8ba23c3fd7734b3b2 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b68061408b238a7459ab87b2936b930ad913083 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cea3f8324c83aeed571129219beb270f89e284ba45dc126d7a0527a6f1c01159 +size 131963746 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..71918da83f7809369d98aab75cf155972aca3f61 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad4a03771e717e052490e339f42e148455fa2b07d459f92eef7f8c71b6d7f5b1 +size 131963746 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..192b14438b1f9d961d988597d2c6205bb38cefab --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fd4bc3873d35981d929978e16d625766e5a9057991725567b340c2b37baba0e +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0724fbf31c6b094455a0d6b3e33d3e694e730b91 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a8fbd6b6fc69a2a51ab724eb9f1597fdd1d3c17f9b8209eaef7a693c747cb15 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..06110bece6ed72c2f125e4838396274a1a3a4186 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3141eccb0dae261e594b572ea04eb877ec4c477bed4f2a8a61788ddd10e25d3 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5c63f038e29da81335addf1f4d7fc3c0bbd1db8 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fef586b5f3eb0db1c643b9730e8a003a86bedebf87ffada2c4de420d0f6588ba +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..51a3c74853a1fba535fb14f025ef0a7735d5ca8f --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abd9adf27be9ad3470da6c10c17307eb5fc338c1634a9cec124208b2eff0e890 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f8e963bceb2a46430dfb38c29fcc629b68a2738 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5743cc506af5eb8b3738a522a069abeebb8db38947824c93fc6907559629e4be +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e846946fda11bbc0a2e2f02ccbec9d6565cf869b --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1aa8557337014c01e50f65260b3d6f863fdde8e79d154037b776bc3269af6d46 +size 131963426 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b2872621bc5b34752b0fc2364df663864d7a61d --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08160fb989b3f5716d1b7eb836f9102d9b66568a81b8ebc439f1b19f1c570b1e +size 131963426 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ea1fe00bdae59707e15263632fb33435d9b51281 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6454a24689f56bc028faef4db6522fe11694cd9aea383a6e0029cd4d3e70ea9e +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..230fdf84d78fbca5d73ed1edd890ef2b0997fc62 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29efad53b117f33537cefafc5ebda641b1d6d120b46ae025f6b5b5a12db8c417 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..29f7f9b63da7a67038bffde4d34271bfa902a602 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22f75f1d9f6ff932dbc493a144a4199cd3b495523eb0597a37a4092ab3cc18b8 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0112bae8ea71d3626bb403a577e48c9fe0622570 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b0ff1749e47d85a9d2ba78f1a318707c850981c7b5c73c8c170546f70effd3f +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8fa0b68e3fba655d0659db3dd8f01a839607d277 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a85a9565aef2d29c1c44862edacc357408e84f8dd85812483f02b718db947757 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cce44d52d0073992711aebd5a73d8fd6410ff6cd --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6d55abdd759f5c658307939a3839e59f7c2b0c73a504e662f41f6e4cfc17395 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d27f86436a11b28f1d5b1a92a3911c8d82f3d85d --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bf2a17af348997755f2f921b8208f89a64e3899fe84805d86425a64c5d07645 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d561c6ac50a4c323ab9c8b2a9a29d4dc6233d42 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3576fd7287bd89a129cbb84dea9d09c73d043259490d5db10b9535677883e36d +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1bd19d2411541219e8c2ba83010f55ad96207f74 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:897269457059ff66595fa01a998cf433c84a96fadb0fb905c09c7a19a89d7ce7 +size 131963607 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b87230655805423abead6a6590050787f67dd8d8 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:100df18701262cdd67966855810788e672941e5c4493edd4de3cd8e569990419 +size 131963607 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f2ed51905302f4d0fe1adca7c4efd16ae232d6e --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2db76eeed91f2de0741682bab5a3cecd059c17a1cc6506bd256c3910f508755e +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cbc1ec2fef0840a639920aa582c49ac89a2f954d --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7890236cb86f522410712457fe5dd6cc18007d295c2695565e65f3ae59639495 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..121a2d04e047863e34b499afafd013e49ea69ab6 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5853b0bb8b0a57f8722c123f5e054c3c09297138ba7e50b2bd555866c737085 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6dce8471f206fc1746ad552503f750ba29951040 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad96f335978a3bbc43594310792255c2d5b56515a892c61464bef14029511a3c +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..11313ad985b86e32b6f2626fc9b3a6f924e1d2f3 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03ae46a9238a335a7f7a687d23200634a8ba49a2c80a766c5ec0680c168f2ce6 +size 131963426 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4d1af09e046b81d65d6d2496d5005713194106cc --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38d268c59159e0e237ee3f8da3ec0f1103a50cf8f737e0f194ba43df104f16f3 +size 131963426 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e161687aa5c57b5e2e227c4048f5e009b0eb9c67 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a4b36e7a5755495ea5d22631bcb245513365a4fb0b258eaa4186f4fc77bcb9e +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7937a66bbb67910fea3a31e3135169b53769afc0 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc16f45529ea4fea73bea83f5c8394d05d54cc261bb2bdb484217b7782a04ccb +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a8407bf1eb689f08d76994620857c55380f1d7df --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4921dc29c90f1d8134fd6711da3a548d74596c0065fc23db4868248b29e17ef0 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..560efebff5c23d20229b592813e417c72bfb2be5 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c60b38f5e50f8e4239015fb1ecb5fe8e9b95528a4a1dd0431fda350c6403bc99 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2af575fd1e682eb5d1ae38ffbc85f82527629c54 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85a3b851bd8c1b3dc4ab5c22580403fc9d310d05fef334496a3c75ad2fb3e1e5 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ba2936e13a9cf471a1ff05b1d1b3f9babfdccfa3 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d5aeb4ed6e1a37202e902f028b8eea08a893a1ff1183a473754e20ef187310c +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..692f042ef292262744d0d4943ec9c170ac1e2971 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53475943a54da664d991ae25bad951ed787ab1440bf494d2f7edb177bd44ddff +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5476bbe429a4f5ca81f77d4fac76c328b2d21fb9 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b36f0acc8cd058b403ab47359641f6dfd7c35d3139a68245311da2bee904e94 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8044cc619e959715e37fa1fc958a694fa460f127 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2ae348d31ca0d3b5e4f7454db4a2df8727d41b534a2c76585174ee8563a03ca +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ca24e72e4dd802af01b65aef6c7d5620e98293fa --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30628b09d4e812a24af9e3a16ee0e27674352a7a2b9dab2822e39da4a3946c5a +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dd5ada7167193c37a4db1fef56f895743ce3e71b --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7f0051fe4962914bdccc624128718fef55a3f5a93aef8fa0cd3291e1f5bf2a8 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..947590643c6ffabd67416e0bf6ad2fcdb58b68e4 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f277896bbdcea7de421dee73e220c3a1d488cf0a4327f35c7d57ca4763fb315 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..028c855c3237cca140fe0b135b76e435ba4d8dec --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e05ff5ad8066c01a47651baa6988d8d76d8d436a3965e2faf1d0b513c8cc227d +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..319c9aea9dc3377f92582e7e86c55194518faa62 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5289a068c057cfc4e0222bef02c0063fd33d420dd44059dc029b1fe7f47cd9b +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..67fbb9e854f041613d4ae02283913ad520e6ec26 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e23a622d04a2360705a143e53f52c353bd679f797d2318ea77e111cfbd065392 +size 131963543 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a6e02aa925ef494485832b17d3f8e866e3047905 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd98aca6c581a57e73a0e9645a34401cfae427e6edaba3c27a34eb9f00fc1bf5 +size 131963543 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..14ec443fd96ed969472c5290bb5225ba6430318a --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a7cf966d11ca7ebfb41f2d13fa0a6510b34cf3257369885118fa41388e86ab4 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a3ff891a505b5939ba573b58b367d6d0f447f1d0 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c0071dba8a3338b1300baa8a0e35b8691d216d0ad9baddebf1609518295a4aa +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3127feb183bcc901abe54f1573af3a6346b8aba7 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bcd1f49103f870de5e467516dbbbe469211285549b323a6562590dd91ccb597 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f309803039fde275f96dca34b6d06d8b8f0e3b46 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6b7ba3086f436f5673fd9ab4327d1081987cbf9c4a0f74cd3c7c4f95ec541eb +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4095b4ec35a3977b4e0c5b8a8c04960692fdc8d7 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d3d3ccc283637b5ca23dd8ae8bf61f44e5bd9f8b0390bbcc4eaf60e389a27d4 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..39cd03e86e242e56c5f9fb837ca7c02994c10c5a --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4f417b712a175c59827d3dfcedf7d56ac5ff60337708b7b47abd0d7a5f6c533 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0f5bbc723c299b38aa5aa1eeb87688c4021a6ead --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93d3a63656eaba05c5e433e8411e1da93b91354b41d25222892dbb9b2d5b079f +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..559dc73449570318240671bdbf31ecc2d27bc446 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d07bf3b06293ae07586946898442fa0a59ac9ecdfd8f76ecd11a326a0758016c +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d0169bff0ad8b12af395ad9c98f2f3e71c4c572b --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:296b27776b3fed8def9e050ed1ab26e2fb254ae4b3051f074e2391e5b4e326d1 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b0cc4282050b85d14a98c6e8868dc353b4941c76 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9745a89cc9f31da81d24a8bf00de36a720b01d7eeb64fb734ad8198b113ce5b1 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..50d136fc56bab36144c5c801c4fc74279e978646 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e96523ea2e44fc1a434fa86236d023326b38304e703695ad2536c4757a099b26 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf208a748527fdb4541ece9e564312e3db3bc4a0 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9063c4e7a0e132b983c4352a9a03240ae001e485603b42b3212325c95671a594 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..785bb4e2f4750e05bb8d7f162e8d66f67d4a9798 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:852a333df2d137a0c54dcf1127d0b6707ee4ace11ca473813ad82f58094a3820 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f728316bd8d80283237c37fb9b3505846ab7d5a2 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e816e9cd727797414f5c39ea97c5dd5b72fb5b8f60343de4e75383e5138807f +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c44caefc563bc80f437907699a658b14b66b2e2c --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ea1c0145d6662b6e2ebef1de0a96a46a8969c0566a1b2ee5373694c1d823cd1 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e7506de9c13a72350111e968aa98357c334c5a94 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e047679207929ef775ac3707f635ddeb48dc20bbc0ff91d0d378380e7f56e638 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b097517874111cf9157fabd3a276de6afa7dc8c --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef0c772e0ae02f0b512adab38d9ac3cc88eb75898f195c229768eae55d8085e9 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..437813c5cc9cfa681f8a76c594f2f61837986371 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9f1381f3278e60854cd77ddb2f8beff09a6dc0e9f5701560eb87d9cd292189d +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..41ecdae7425b74f243a7c8a544b4a7aeeec1c29c --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:482368a8f1dc57bc5f29e6063d6f4070d05a135ba8ba2447cbe8832f86b62a13 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..56f35e4868ac613f55797f5b324f100e7d1e3f7f --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d4b086c77d94f0b4f2b98ed078d092ec4ec6fc4dbcb207fb1b899645b1f367a +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f0361eff2c233de7fddc3da1e27f2da1cd16fa4e --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f2de00806bdd87a8ed5189216447554d69ad68720b59eb5e6f6907bc4aba91b +size 131963415 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..650af1bdd765f168dad7730897514e12b31d7aeb --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2204a5007feb1a99ed1ebe74d4117cf7f6bd783b81208a720c6882d6ae5e7f3f +size 131963415 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d4dec33b616744ae00c57fe75dc5dc41f9e40bdf --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a57665aa4e82f2c3d7a731ff55023fce25236aa51f80b770735975765a75066 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8be08b49e7bbae3e011ba9b6d8928606274c6af3 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d580a68b1f053d2bc02324422c0f9435094a2e6775690657d2dc9113bc7b5834 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..79d403157c5c9afab2e976e94e9d2b4caacaf99e --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5036708e95b516401b19d6ed329d50ee915045a52f200136aef67d7e14dc141 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fbf51077bdbe3506df270b2f8dee8eb25b44e2b8 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c621c8beaeda557384a7a9cbab08a4fe5e93efac3e1018152a3d4fb4a4e9a9cd +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6acbedb2729680518ac941ec1d88ae75d96339c --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:533e136471b7166c87b2ec7c7a14d65ede054aaf4a124c033eb4a7ef6a0aa2ad +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e05aa33a011a8a19bd5cf697854bf53ae29f899 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8dbb6ff840defcd6414ba5a48190acf6ba8a7f0ed55409844f4b661650d78468 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3082c3f45a83c0448f02399fa8521da13969cb5e --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ede84a385965e51fe8d829fe6134aef047237e33a2b2bf0a795d60f9870eb168 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..020539f70395663772ba3ea69c3cef8458185dac --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63f60dee3683dff46cd1c11a8f2f01d7a39e0977fc68d9bd08b7119f8378be49 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9204fc4a9766c4243d9a87f58659023a1e155f4b --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7acd281af17590701b654135fe01594c9c3c82f7517aaba80c78099c36b85155 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ba4a41d51712449c59dd6c0fd30f72b58d4aa424 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0838941e82521cdd1bab8e0fcb84668c5f37c29edceec7395fceae24fd209502 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8472f603ba6924ad16fdb6ec15a7986a01c28584 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e3156ae669c6f87e9dfb8a65ab8c729c3e7244b873cf2a3dd1822ba85003d1d +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fabee83e8b793e5d8821d2f7cd6a1c876abec184 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:951ff141015549f59cc89f8fc9ed052093a882b8ea026818e237bcf025aa1592 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..876abb894bb71a8fedfeab374f4d031f865693b8 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80b4b0a17a5fe43c22e83cf8db777aa163568239077153ada27bfa4db68c29fd +size 131963682 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b11b71f68502f4d915720f97e282587c5f2c03ce --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca56ec610d8c280f46db505ab8c8483bb43471a2ba7c4481387c4f95aab18cce +size 131963682 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..18f8dad78d5ad91e30ac1340bcfb406bff9a1d84 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56549d544efc0dd128dac91b590f3bccee3c76f2ef483069ec9191969622c1ce +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec861a8c5fdc7314a0d0f0afe0c631ac8514c1ba --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a02caf5bbe4ad84887766b772bbe8553f6cbc7574bc30399eecc09eb405adfa +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..75bcafd4f72aaedb4cd59519fc758789dc9a47ae --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bd9abfe91f799d299665fbf9d892122a44d163d7b536392bfd5f9ac56663d96 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ab27789782289a85ca1d1fc0e0320a5e62f2d1d --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ad51a9ede7d8785378af5011e34d22b5704ab45da27eca6d480a63829ec2cb9 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7ca6ca552a6c020274e3fb7aebdd71e96fe01087 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83681b277b31fdac34ec60ce71f139ffc3df794fb24b3eb30985377fbd1dec25 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9128fc6c206d8e8acff167e442be96ddcc06e397 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f982f823b065ada1dcd320b6c773f0bc6f7b8c20b468c378946d52a4c03837ff +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..760f7d1ae490389ebeb6fdcca4aabf4834a51120 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b73744fff2f42103e9123bfdeb0e58278f73ea18b9d92dbfbd3f4af949d83fe +size 131963543 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..176a1b738626947c5c4d28121f9a97c96f927cdc --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35ccca213b2181f083b634398a57d0cedcf290e23c258c9c5df72e4bb7833d02 +size 131963543 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b7049c897a2db9eb0cfe03ca8577c3b31d2edae6 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a3980f7c7b876874da7fe5c296d24bf58d7c825e1944be260af73d6d21031b5 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cabb35ff6332e9bd67d06955c5ca02dbbc90c2cd --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f6ae53a68c00ac73dfdfa3a2edb45a54fa7cb2d27b9b1826ffeac8a867cb1fd +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1fae77e18a93eacab4c6cffcdaa8d4add0561942 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8a375aa79c2766bc215ba56e3abee49d3507092b6be1a9b700120758a700400 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fd0af73c84f46ee471ea9d81a630a0eba028c49d --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81fc2b744e1800b6b0cda134fce87f3a76ad1aba2e5981e42f65c06b226b7948 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1571026de81c2b32bb54e8dded2744c04b884414 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b4f426398866f4e0ea5baaade29315ba13c4679fd4f08170c68cdf09c118e18 +size 131963426 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..020ec85051c5714cf30aac0493ef0b7c541a9fbf --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ace6c7e193c8bdcce3b57f5bcfa1c5e7de65571c06e2495067440bc9e6067cc8 +size 131963426 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..efa319cd91b824034de62d0c4ced898de58c98e8 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8d8ef3935feb4f71bbc4abf3754b8b142475964b90cf35cf7f438321c376b7c +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e527c21daa9b39ba7844afcbb7242062e9ffacb6 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22111fe1829cbb103a2fb75cab5abbf49b40bb06b37bf6d91e9d631153491a23 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f9ef85b7e911deaf6e1609ae66c0f779d511bbce --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9880712bae8d140d51d3c79c8585de97360695ebb7245a95dcd5c4e1f2707f32 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..be9ad2912f5d220e25a2fb476e26bc28d44ee6e9 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a6e2a4083fcd2c880ccac869c5831036b3e0c4b4f080c7ac9303aa3ad5e28d0 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f0f7b28e9c8610efc94d7220ceb832f7fab642f7 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5fcdd2f2e7275a9aa2db4be889aecbc54ec3caee1f673337edd106eb279303a +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8157da4e9cefea53d175633f70ceb24159ebbf6d --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10b79c8d2d6521990e20957b95c9df1fc7073b45d24e2a4bcb2650fa4befbafe +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ef27dfd91d290635c27316db5dd83fbeb27f1bd --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c62d94b55ddb41c3449e21a941d8ae74dfdb0680eaac24447d8ebbdeb9743421 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..60e4174d19be61a0d12b8dc239dfc375e4eb8f5a --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c52d7165e5384084ac9845199f81d68a19f4ad137d1bef1b15a468bd55a8c5d +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5357b81609eca0ebba401e473ee4feca5de27a3d --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ffa77dd16050ada65753b9d6f7bad4c362c748fd932dbe116eae71166b100fb +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a22028a6634ceb37e2db0efea4db6363ab307c72 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cac01164de6b34d2539522735f06b270b7fcd91e464d3d26655979b106a992f +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6c0e0a5ddaf39cf199c97dd36156cf9e770379b5 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a5172837bea80af45c3a0f0871b8cb1e3d0be1ce0f149a0db79918851b2cd95 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c31f656d3307f1eb9d9889966167599bbe4f1bf0 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbc7d767de96e5873a3186588a323c5daa859d210899810527f11189a89a5b4c +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a4d5f9a2296c3b2cf3f54b6444757314346c7b78 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2787d767d08ac9934ec472eb07ba7ef07575f05b05e78054e27a7ad23cc75121 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..33f1abbb50ba1f61817c48af4be1abdd45276706 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:382e204058df9b8a7efd5509c901ce4555ab60dadc7d0689db3a5adfc23a0588 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f4b1dbe44ae7501dc2dc52427e63df61587f8c5 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:316c094b137a01f7fac765b46485b51f300a5e188450a6ca353993003f7e5166 +size 131963479 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..08898b0efea01222a2e2c364dc340e3bbaf0f658 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc14e23fa7a9cbce71059d2df0cebe76b93d43e9619687db8fdd88f5b5cee6e6 +size 131963479 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..15de8d95954e3c513450347f0ff4286557f640b1 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d50124e4ca37def435b835242620ba1149dc22aad918f2b36d182d630a08e98 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ce351d931c04286b6935b1a29285fae2a279eba8 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f6c5ef23ddc473d0e8b724f4d93bd176bb7e8e4e600476c31e335337edad6ec +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e86fde15c975571d79e4012d7703ff62dde9a874 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c02e3ee0be2a37b569a536043811140c2eb944dee3c1974a102e149dc5d03c3 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..190f379a70ab0eb16e4d4d5730203baf964ebd8b --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5de8f20ea97cfb430c1628d2997fc3fe8d5034e2dd0c164cf63235af42526ad3 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..de7b8c260cc4e6fcc66663ec5c98550c2e013a70 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48d02971f0dec23692ad49fbed5de306898f515d0578bdaa6cca509d1a3f275f +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e8bcf0c132c00f01b5765216b28a8c33e1c3f87 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfa6a46ba3e5cccb91586d9a84b7cb897748131371e3181914dee417310f7a71 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..154e2d78bbb1a83fd8a435e7f7dc63bfc91ea495 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3047a45861c13c9ae5e75fef0e6fbb5ef1dfeb1b5570ccaa66a41eae40a0028 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0be494fd326ac9e654f6336cffc5d7f0f33e8ee2 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54dfcc1b9cd664b4b228aa19bbaef81af43ff3336f658058a28fb64e00ec6822 +size 131963618 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2545a82b37983ba21443f0cb7e54b89a39b696da --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22d48588775d8577e805d6596fffe5017c7dcd9c0375ac41761b4e493a896788 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c9a63c253a9afc56755b951942be36b400b8eb35 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ed82a5f0d42773562abcff1652a72090a383da81cacbe6fcf664dc846086d94 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0475a0168983db626804e51ce0bda57a33ea3c17 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae666dcb6d30f7c075a5f13eb676e91408dd032d94164fe2c0c4dff147b6a56b +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7439d3d51ddf7d34fb112df69495ba15e7d2e6d3 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe596c1770794594951b9454049c6ea39501ddac784a219d84bd27ec0dea2225 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f954558b1e46c3a51379a31818d34c7cc260661 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9fa87bef710233d8330f3b319cf16efa33466a1bf5d7a768c4795ac8d056993 +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec1a3167e809c0a09f16810e461282e8edb733c5 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6ddabfd83ba59c6e75082900f29cea986dfa3158224a59a93be1da194801d0e +size 131963490 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0ea9db0bde97a907e5ebe4f7dd9333f0234a5a44 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e25d398f695f29cb22485c8f359b0be1af21dc914bb0552b1d0f9ae41d73a9f +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..452ab9bdee7b33989426a7687ccc06ea84f78ee5 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85461b3ba2a8976fef0755ea6de544b9047e065811254c176c3c0eab940d5090 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e50df81cdde9a5e7754df52f59359e1e3c459ba9 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70bbb24ffc7e9a6334265ab142c748616355c174fac9f1ed2ce87370c0115aa0 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a65ed16c92475f6f93b41ed7d3ef0a6a486c93ab --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5891669f9e9447fea77725d46288e8653e9e814006de3ef94c31d5a4d4b25d69 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd0c0d82a0ab042782f4763ab4d331bc7fdabce8 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c26e98c37c95ed6d33a9dd9dcb3783fff1508d1b4b0842171e1059e814ff8a61 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f75c90b3b19ab98bfcc3b216e863fb59e2875444 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4baa8ebb888f2864f3cc5a99c25af72714ac306776d34c500d61e6c23572dc88 +size 131963554 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bd99b257cd3fc15c4bec98ab007608e0c4ea8c64 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fc7546548bf85c67e2eae5e170aa690f95839e4cc069a4ddbaf3135dae5d694 +size 131963543 diff --git a/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5d2b14dd17e3d15aa74bf4a37f0278a1befe3f4c --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9a064e676ef9a0fbe643d5804b46dc8e96b365c27dce437ef065938f7a3c971 +size 131963543 diff --git a/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2567b81da52f31b79d959aaacd40d07444d6fe7e --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ad418110923bbceb47a0d7395ea2314f6c04fe5d1364c0b90b81f2e045ad95b +size 139592963 diff --git a/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..314dc25810d89487c0e2d7b5daf80d7ad8bfa64a --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_01-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35d4072a06420c96187e742c83a7413a04059052e6d814a75fe8bd793ab14c31 +size 139592963 diff --git a/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5952f0edad5ca07f6c6ddbc6c9e58b7c6df6c4a8 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3de7d93f6078355710e7629c15680c44eccfa98a102e63a898930fe145211ff6 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b7cfc5e1e7950b13bf5106cdc5637e65cb09f14 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_03-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d85df366eade5d3a439bedbd46de122ce8f65b6ad0c6426eb322843f3bde76d9 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5123a13fa54ab1e97b2fe4fa66ef0effa3eb2b8d --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6362c1dfcffe86ff3d59459d60f2ea170376d7e3b4228717ccae9be5a28ec035 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a925ffac6ba698e01cfde299970f65623379ea75 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_04-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e82171743245c8b8f31821ee94ccc68c9d19ab3f3ae8d6d7e604210c383281a0 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c94cff5cf35c8d13a4c6123daa972c5bb4f95f0 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37e94d9cb4d94790c5e00009b99d68b4ddac19ab9cda8c991ad28f4745094223 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb987441803a7cc594146b967515f1d3793656d7 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_05-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:653a1b63dee50e2afdb38105afa4fe6fa7c904b7ddd49d2bd7d6e880b70afd1b +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..90a041cfe39a6a2be9f2b240c2d84792e27c8055 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0639a554f2db3eadad0c4b5fa4ce04442fe5e384975374a2a71f8f3a63411d73 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1bb6af4153e6a70407b6cfb20a39031cdd2488e --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_06-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fb028456c35217cae6945eec32a5298b99bbf40027e1af87f092c128a9bf4f9 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bba5e70b2250f6d0b4885d796bbaddc2d2778dd7 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:346647e4f8f488efcc3505d273df05b1fa12685c21be81911b6b95238f9ad1a9 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d0fad9430efbf58c8ffc7b7669310aca21e4a15 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_07-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d54713ebfd1ac66c0b4febe3e56ccb9e7f7b7ae3b596f4969bae905c836bd8f2 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f52d3c8741b2dd29b1dd71da231e356353393cc --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3baa578130f1edde6fbf576ca88b9061d135147330ca2f8adb1c8a46d870975b +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1bfb8a0f3ac387d96526b004e5cd3f13760a476 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_08-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53dd06b388f7b644780087efdb3717700472cda1f8f73ecd63cd90e8c52fd016 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..210da58efecb4ab064e7a126367cc526a21751f8 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:865cfd9aa7cab07bd677ed2ffe88e4f7476dc0004ebfcb2ba6a8767b04b16eab +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..248a0be513880f3807a3b52b2f1a5e31f69f4883 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_09-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9af2407f238be9f6384774da8a6953fe1b878f981b4b71124e0bf89e09c50d86 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e4e3ab9605dd01c902481aa03df70fd63f49d630 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8659369d184e3552e3a69e9e814764827a8889e7204988e272d535a6685ba34c +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e0cc1fb7146766c9cfdfc2dadf61b0a613c0a33c --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_10-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90dda8bd69425e878af0a0551cdb6e2d23aa73f6e0598a34078da50a84c05e9f +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c62bd409e449f7a2a3cf5a217ca6f9f5daff354d --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32cec83e4543750759cb38386b08d97459af5084f8ecf2bb1882477d77d8ba6f +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..af35885672e9df9f78b63a86a856f84f9b4e113e --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_11-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:026ab09286f8081b900b144578610cbc9c474aae8c42237f2200a481e6e89160 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6cd072e18dcc471d890a4e1f196c6e3963607b0d --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e887132c1519bfb97a5c888753eac7c95fdcc2223b58d4df5edd552b075d8de +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e82aa1178854adf1e6d4ac19bf12a4a366c5ebcc --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_12-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3639c9bb181c3d55ad75e9779e63b3558347980f14b84fdd71773bf3d49b0dc +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bc77bb5d4813617253e7621f76256d4844887bf7 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af106796f79cf81d41c2c226a2064a58b982ee526c8c279fdc65dbde81933ede +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..99ae209cdd5c00b989f16013e7445fe9f0c54b65 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_13-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f63652bd369a64487d695a26a7a5e437cc6b7cfb9d44baf39d7a632733f4155b +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c3e7710c54f90f11abf153ddd43a66b8e287d12 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:316625a0ef450845a4c04ce2675fac78728c081bf60107f56bb91e7dd133ba19 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b6734e77d48bf99a68c0d0db44dba1424ebbb98 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_14-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dec2048a615baaa5abf28674f49153a23c9cd0fa375e0243f258607fd612a84 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..044619da2d73ef302a840164a941b361497eae3f --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c27a059532ca31960c2df8f58e942ddf8316c5fdfb8e545caa19a5fda209dfd4 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..53fa6f50ccb44e0ea938712a108d3ba8cddabfbe --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_15-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45340846ef3defd5cf344240c4b66009ea939daa97ac6d186b26d4ffd2d93e73 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..59d59113c4c6e6e8c4fde752c90e7469af6f7a0e --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c908b9954ec421048e1d39eac8e2ddb0d13536da17ffe7335c65b6ef491034bc +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..39ed7c10fbc46e38415f6b0cc610f4b444bb33b7 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_16-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39b92372ff5225f4aba781bbe9113217aa969ec1da5209988fa930d6a8cd127d +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..41c27ba3c07c98016f698edb15970cf187606fd4 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fac2d02322f0fc85b9b857ccdaec4e81a31bfb7c1f59a326fae2483381ab1b6 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cffb7b12dc99523e69345caee9684b9a6c2f28c9 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_17-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:264d37d8bf2b126949f30c7ddc7f034c5d44e6fa0bab3c3f756e1e7ba61b00ed +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..89b7181abc18ab55db85afa3da408d50d653dafb --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:40064417eecac93987549e8742f6b786feac927d18e8b2abc2f5b0831b4fdbaa +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6239a5de3ef841f4d3a9b09b1a4febe280a5be6e --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_18-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9effd364ec4fd8d5d084f14e923b3909ebf3264e7064eaca4efc41257edc75d6 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ca8cc19242eec3808faaeea44f51c4dc5a0704c3 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44db7d7827ffb01506fdf95d7f1d18982643bb2c512c906b168b985fcf239e4e +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..645a3d763256d88d7dc124861f1ce9dcb9c5d0c7 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_19-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cd8e70b6347c2acb920fd42c11a81cb1d0f05fdc8e39b7ee2d61ef38eb2e4b6 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..84f6b6c93d68549ce1fd809f501dd5542616a35c --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89df928286474789d82eb9d6f5061b48f7f5bc8881da7ecf562203f81752ae39 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b25d9b1f00882940070036394991d5d8c2ad7fdb --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_20-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ee4bef0e3a5bc4a41aa17058f6644f88dabff9d882848d6dfe6764fa9fc95f6 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..25cf5ed78a28f2772c5640d595dd5fc5fef6580c --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5cd8fd831ecb0310a779e8491681792ad7be67c9cb5d7de616551da7fecd4909 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..de04e41dbfe5b8d46a82ec6f68267e415a5236eb --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_21-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ed8e692d26cb56a216a8c7a147ccf9df3950688b779df29e27f752f2ddfe798 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a38839ff7aa847461917e73fd301a89af86f19b --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_22-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4a1d5493c34ad1fe2d722d78c85609385611cf825854929c432ff6da2bd4685 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..42a956c70654e56af5374783cdf86b75be34fc4c --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_22-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c639b1271382fbf613d9f92d5a15350a8efa10f674e777359f889ebb96ac09b3 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d0b376ccfdc78e153c7cc41df9dbd1d89a7dbcfc --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59958055e79cf4a9a032dcc74e965617be0b842f2c3fd069e054521a336eeded +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f11ed06649945fcfff636b0a4b989abdb7888c8d --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_23-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d35d2b84f21c49cbf330b6eff8a14a067bad0ea859d92301d8b7e3c847aa5135 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..22e839e6b263c1f78f758185f491a8f0d487d25b --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_24-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd2cb00645e7c3e2e6eae38cfbdb2be78fca60b6a3fa19c11ab5aaccf9a29424 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..42422f4ab809bb0e626f05c62b195795f442b6ec --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_24-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9ec00d86bebb2a1092efa14b4fb0ae53e7d162b699cf38c329d4a5471600d09 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..382a9da801c9f6da9e269e7cea4c24eb5890896e --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_25-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:387f9d1bd2cb92a03570abf3cdc0b64eb18b9d014bc24fb3b12401d3b86f47d1 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a2dab195efd36b0ccd815c54325b0f29e39dbaca --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_25-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4338f104f399cab5fd46097fd86a78402f3fc55f455d787e2ef0eb785d4c8613 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..518ddf73a4de60ac404ca9a82d2d320f2678dd0c --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_26-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9ed2e75c70bbcad8ff216f73e5964a3a788ddc6941a352ebb5a058ccc0b6b24 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..709489ccc86e41a595ee78faee94831efc184294 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_26-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c3f0ad323b921f281d9c3b0b4187f775ba75de31ef75f7815d9a57de605065e +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..29bb313bb340fe0c29dc9769f93c63eea50f2b55 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_27-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ad3d01933683fb7f543c38391d11e65401dd457616af78b8f886d866bf2b18d +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..55b7cda32de286bd4f9a104b5b2b58d2e90e7456 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_27-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c254f2bfe0a139c2d5e2941cc442dbb461b507b90f528a972698f1f0c2cf4794 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..41efcbd5d6af3209cce389ee6e256f7d63d98281 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_28-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7eef8226d22c2863054cd093c33c1a3f887aba4dece634140c800a3f15df742 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1877f08358031c0b9101f970fe725cfe21dd059 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_28-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b377239d4908d7a1b4fcbb43019253b8867ba0d64f06bad4804c663476a81011 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ca9cfff2c96b5b35194d54df38bcbe1b8c102839 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_29-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bca12cb83ffa9bbc3e605a8d2546f8b4f31ee82ab0442ea69bef574a37d67b4b +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..92ff6792a01933158e3bb62ede618616b8a2e85c --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_29-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd07a7f272b691c461cf30faac181b347f351375832250c5176b465b10235e17 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b41f570566f4bdd2365c17a7a9ab9e198bf4895d --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_30-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89136870ee508b306668dfa3ecff6663a5408e077e807c5bb61faaaea76af07a +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f8b2d684a235ff5b4c2723d6d76cf5e4a5b9c22e --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_30-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae1ff97dbffb92d3541fbe8dca7262ca63df154ad1b4e6c46d44b8946c3ad7d5 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3254600012d66867d175abe41b702ce6b08cac74 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_31-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f866118a39db011841d3bcb6f0ad311b25862efd85ee515997a45270e0af1c8d +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f33a0a977153c012f9b546d19d7f19d522bbaa6 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_31-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83a3ca14c2425e91d2d05cf11bfb9dd6891176690a29682d472eec06204f8984 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..df35a60191c0671c73dd21e1b0098d751870859e --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_32-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77103f2b66be871cbdd9601a178eb6d8edaa284a91443c202da568309756dee2 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4ca451c21deb3a2f2f262a9b63a48ec26e93df3c --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_32-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56bfd828a9f8f14e13a7ca60abec6560bb985f543e6a14778c95945f3dc0e0ed +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad2879ca8f829ecaaac0675a342f061ac1223763 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_33-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6eb3692dd09a3829fcdb06b517fdb799e25b7db215cafece3df050f56761c52 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5341493aea9e53ffef3807d560c4d5f1ab7c0a03 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_33-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7916a8e725ca5131cf7e659099c24684fdc6ca6a801e33aebbdd0895d7ddc0ee +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e8d359086b235c6f2d90b103861e79ae1b10f921 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_34-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30a8b37265a3fd4160ef8edab136aa48c0bfcf003d4a6d97a13c62505a18a178 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a18db6be6b20c17ccd78abb6fd3a497358eb8313 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_34-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d78b57abbf1081091f820a7d08e3e9623f01b3b7083d7b2b778d0ce1ff6a588 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1e3e80dc14122124633a8ff265e263932e8fb2b5 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_35-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fda7525dc488f39202ea84d887eb22013af60b80d7c8f218a5bbfbb3f01bd347 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ed8f9fea01f72c055b9924abe18a1d515ba8f003 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_35-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6624d468008b4ea76c9bef7671fb9a2137c7f1f7f8049ea24d3e2bae506ff73f +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..89b48188c1aeabafe555a544941401f4b8162f7a --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_36-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41cfa1e0d133f2a30a17960f6fb0a9d4d3147ae784886e08f92b5118fd604fb1 +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..132a560b61c16c85ae47767a8082b58c05f8cb00 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_36-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e7ed5f536a886d11a456562ed0a20f7ad69e0066c984183781f695e72111f4a +size 78696195 diff --git a/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..023221095d988330d449d3a513e07b3894a3e189 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_38-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:550664a0e9d743994c65d2011629b35854709be5ba99e8a11bd02b8452b14146 +size 11459 diff --git a/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt b/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e0ebe67d6e29a20427623f821a6783557d8b1756 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/layer_38-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59fab063e87ab7e8ba71a35e05bc92c24a029241001e60a5dd50a68a107f6050 +size 11459 diff --git a/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt b/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..62c6c362859c6c93e68dd5f03a5667bd13e9e6a4 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:951838759383ae93bfce37c224c44adb5702e5912cb73dd62c4139886a0fce41 +size 49971 diff --git a/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt b/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b54cd24fda4e27f9ad0f59088f44a912c1774dd6 --- /dev/null +++ b/4b284b12bc4subopt/global_step109871/mp_rank_01_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e02bce75d9a94d7d91fe4dcdacbb4eac095d8b4fa037d7a73b5a8929b381bc9d +size 49971 diff --git a/4b284b12bc4subopt/latest b/4b284b12bc4subopt/latest new file mode 100644 index 0000000000000000000000000000000000000000..de86a0304b28cf22aa8648efad63f619017b150e --- /dev/null +++ b/4b284b12bc4subopt/latest @@ -0,0 +1 @@ +global_step109871 \ No newline at end of file diff --git a/4b284b12bc4subopt/sbatch_4b284b12bc4subopt.sh b/4b284b12bc4subopt/sbatch_4b284b12bc4subopt.sh new file mode 100644 index 0000000000000000000000000000000000000000..f8f28ab2d063541aba2f31765b915b7081e1bcf4 --- /dev/null +++ b/4b284b12bc4subopt/sbatch_4b284b12bc4subopt.sh @@ -0,0 +1,166 @@ +#!/bin/bash +#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901 +#SBATCH --nodes=32 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=40 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 48:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + +VARIANT=4b284b12bc4opt + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" + +TRAIN_DATA_PATH=train12b.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_12B_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + +PP_SIZE=1 +TP_SIZE=2 + +MICRO_BATCH_SIZE=2 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_2980M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=10000 + +# Tokens: 84_000_000_000 +# -> Samples: 41_015_625.0 +# TRAIN_SAMPLES=41_015_625 +# TOKENS: 115_208_849_730.55898 +# Samples: 56_254_321.1574995 +TRAIN_SAMPLES=56_254_321 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.95 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 562_543 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1000 \ + --eval-iters 100 \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/4b284b12bc4subopt/tensorboard_4b284b12bc4subopt/events.out.tfevents.1683211293.nid006741.54329.0 b/4b284b12bc4subopt/tensorboard_4b284b12bc4subopt/events.out.tfevents.1683211293.nid006741.54329.0 new file mode 100644 index 0000000000000000000000000000000000000000..4d962c04107ea96eb9fc836bc8f24bd75e21428d --- /dev/null +++ b/4b284b12bc4subopt/tensorboard_4b284b12bc4subopt/events.out.tfevents.1683211293.nid006741.54329.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e677b8511dbd6a736e19ecaa2a5788519e30fb80310f63cc4f7fc09a1677812c +size 161318426 diff --git a/4b284b12bc4subopt/tensorboard_4b284b12bc4subopt/events.out.tfevents.1683384108.nid005063.95501.0 b/4b284b12bc4subopt/tensorboard_4b284b12bc4subopt/events.out.tfevents.1683384108.nid005063.95501.0 new file mode 100644 index 0000000000000000000000000000000000000000..e4b833936d77603c640eaecf900521e04ba6ad11 --- /dev/null +++ b/4b284b12bc4subopt/tensorboard_4b284b12bc4subopt/events.out.tfevents.1683384108.nid005063.95501.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0f99c5c525a05d1d605e101c2c48ae237f9f337c1b8c823338c99018ea74353 +size 53701312 diff --git a/4b284b12bc4subopt/tensorboard_4b284b12bc4subopt/events.out.tfevents.1683441640.nid006741.14962.0 b/4b284b12bc4subopt/tensorboard_4b284b12bc4subopt/events.out.tfevents.1683441640.nid006741.14962.0 new file mode 100644 index 0000000000000000000000000000000000000000..4eaea801398075ff8153e2ef3f5c46c0c299682e --- /dev/null +++ b/4b284b12bc4subopt/tensorboard_4b284b12bc4subopt/events.out.tfevents.1683441640.nid006741.14962.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c160d9f86658b619ea6a2721117ac8f1f5c9617f7a40121e32db0359ea5e77ba +size 21493 diff --git a/4b284b12bc4subopt/tensorboard_4b284b12bc4subopt/events.out.tfevents.1683441961.nid006741.20247.0 b/4b284b12bc4subopt/tensorboard_4b284b12bc4subopt/events.out.tfevents.1683441961.nid006741.20247.0 new file mode 100644 index 0000000000000000000000000000000000000000..48419daa206be571c0d5ef90d4d85ed797ad0930 --- /dev/null +++ b/4b284b12bc4subopt/tensorboard_4b284b12bc4subopt/events.out.tfevents.1683441961.nid006741.20247.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67b0f06424b98c025ce249066235afecc3cd98e85e7e7c4c30518a95b51a6509 +size 16424 diff --git a/4b284b12bc4subopt/tensorboard_4b284b12bc4subopt/events.out.tfevents.1683442246.nid007109.91332.0 b/4b284b12bc4subopt/tensorboard_4b284b12bc4subopt/events.out.tfevents.1683442246.nid007109.91332.0 new file mode 100644 index 0000000000000000000000000000000000000000..2b00c7a8cda533e488ef927961f3033b0e6faa00 --- /dev/null +++ b/4b284b12bc4subopt/tensorboard_4b284b12bc4subopt/events.out.tfevents.1683442246.nid007109.91332.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b64df18d0a2ed65711a48f803ed7e2d7f625cb59c475c602293e707db8dc1f2e +size 16424 diff --git a/4b284b12bc4subopt/tensorboard_4b284b12bc4subopt/events.out.tfevents.1683442533.nid006741.29749.0 b/4b284b12bc4subopt/tensorboard_4b284b12bc4subopt/events.out.tfevents.1683442533.nid006741.29749.0 new file mode 100644 index 0000000000000000000000000000000000000000..0082f7cb70efd5c55c47bcbc8fcd7ac0457353b7 --- /dev/null +++ b/4b284b12bc4subopt/tensorboard_4b284b12bc4subopt/events.out.tfevents.1683442533.nid006741.29749.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfe7988d3ea00339ca14dc441f4a067beb42ce221469a0e6c7288a12513613fd +size 16424 diff --git a/4b284b12bc4subopt/tensorboard_4b284b12bc4suboptval/events.out.tfevents.1683534125.nid006037.26099.0 b/4b284b12bc4subopt/tensorboard_4b284b12bc4suboptval/events.out.tfevents.1683534125.nid006037.26099.0 new file mode 100644 index 0000000000000000000000000000000000000000..fa39354e20c2155a6f47cf7e4eb69fe1052c8a73 --- /dev/null +++ b/4b284b12bc4subopt/tensorboard_4b284b12bc4suboptval/events.out.tfevents.1683534125.nid006037.26099.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88eae4c213acbcf43e5e6d275d1a4e17768e48e2a8bceed7358bcb03a86b7555 +size 980 diff --git a/4b284b6bc4opt2/3484963.err b/4b284b6bc4opt2/3484963.err new file mode 100644 index 0000000000000000000000000000000000000000..567b7db8f2628cc8a30a3fdec7199f00dbfb5291 --- /dev/null +++ b/4b284b6bc4opt2/3484963.err @@ -0,0 +1,3641 @@ +14: 2023-05-09 16:41:11.351652: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-09 16:41:11.351666: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-09 16:41:11.351724: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-09 16:41:11.351731: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-09 16:41:11.351752: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-09 16:41:11.351755: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-09 16:41:11.351775: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-09 16:41:11.351785: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-09 16:41:11.352316: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-09 16:41:11.352324: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-09 16:41:11.352334: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-09 16:41:11.352344: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-09 16:41:11.352346: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-09 16:41:11.352353: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-09 16:41:11.352441: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-09 16:41:11.352508: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-09 16:41:11.352495: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-09 16:41:11.352522: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-09 16:41:11.352543: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: 2023-05-09 16:41:11.352318: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-09 16:41:11.352331: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-09 16:41:11.352310: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: 2023-05-09 16:41:11.352575: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-09 16:41:11.352582: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-09 16:41:11.352592: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: 2023-05-09 16:41:11.352635: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-09 16:41:11.352647: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-09 16:41:11.352667: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-09 16:41:11.352693: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-09 16:41:11.352706: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-09 16:41:11.352346: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-09 16:41:11.352372: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-09 16:41:11.352699: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-09 16:41:11.352709: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-09 16:41:11.352735: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: 2023-05-09 16:41:11.352390: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-09 16:41:11.352401: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-09 16:41:11.352407: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: 2023-05-09 16:41:11.352529: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-09 16:41:11.352532: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-09 16:41:11.352576: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: 2023-05-09 16:41:11.352814: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-09 16:41:11.352824: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-09 16:41:11.352830: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: 2023-05-09 16:41:11.352692: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-09 16:41:11.352738: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-09 16:41:11.352752: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: 2023-05-09 16:41:11.352819: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-09 16:41:11.352822: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-09 16:41:11.352871: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-09 16:41:11.352609: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-09 16:41:11.352609: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-09 16:41:11.352552: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-09 16:41:11.352561: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-09 16:41:11.352766: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-09 16:41:11.352782: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-09 16:41:11.352627: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-09 16:41:11.352608: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-09 16:41:11.352666: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: 2023-05-09 16:41:11.352910: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-09 16:41:11.352917: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-09 16:41:11.352940: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-09 16:41:11.352601: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-09 16:41:11.352604: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-09 16:41:11.352760: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-09 16:41:11.352784: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-09 16:41:11.352760: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: 2023-05-09 16:41:11.352564: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-09 16:41:11.352714: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-09 16:41:11.352726: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-09 16:41:11.352792: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-09 16:41:11.352797: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-09 16:41:11.352810: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-09 16:41:11.352622: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-09 16:41:11.352630: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-09 16:41:11.352842: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-09 16:41:11.352856: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-09 16:41:11.352572: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-09 16:41:11.352578: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-09 16:41:11.352740: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-09 16:41:11.352764: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-09 16:41:11.352766: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-09 16:41:11.352882: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-09 16:41:11.353041: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-09 16:41:11.353068: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-09 16:41:11.353085: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: 2023-05-09 16:41:11.352646: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-09 16:41:11.353157: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-09 16:41:11.353170: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-09 16:41:11.353162: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: 2023-05-09 16:41:11.352873: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-09 16:41:11.352885: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-09 16:41:11.352889: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: 2023-05-09 16:41:11.352860: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-09 16:41:11.352870: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-09 16:41:11.352774: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-09 16:41:11.352785: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-09 16:41:11.353065: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-09 16:41:11.353065: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-09 16:41:11.353062: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: 2023-05-09 16:41:11.353205: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-09 16:41:11.353217: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-09 16:41:11.353232: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: 2023-05-09 16:41:11.352802: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-09 16:41:11.352815: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-09 16:41:11.352810: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: 2023-05-09 16:41:11.352964: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-09 16:41:11.352977: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-09 16:41:11.352984: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-09 16:41:11.352950: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-09 16:41:11.352968: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-09 16:41:11.352890: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-09 16:41:11.353129: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-09 16:41:11.353142: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-09 16:41:11.352965: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-09 16:41:11.353116: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-09 16:41:11.353129: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-09 16:41:11.353154: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: 2023-05-09 16:41:11.352939: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-09 16:41:11.353308: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-09 16:41:11.353326: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-09 16:41:11.353322: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: 2023-05-09 16:41:11.353154: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-09 16:41:11.353164: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-09 16:41:11.353174: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: 2023-05-09 16:41:11.353135: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-09 16:41:11.353132: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-09 16:41:11.353142: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-09 16:41:11.353123: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-09 16:41:11.352985: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-09 16:41:11.353000: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-09 16:41:11.353279: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-09 16:41:11.353209: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-09 16:41:11.353165: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-09 16:41:11.353190: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-09 16:41:11.353289: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-09 16:41:11.353346: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-09 16:41:11.353360: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-09 16:41:11.353221: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-09 16:41:11.353120: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-09 16:41:11.353161: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-09 16:41:11.353294: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-09 16:41:11.353385: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-09 16:41:11.353191: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-09 16:41:11.353197: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-09 16:41:11.353345: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-09 16:41:11.353362: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-09 16:41:11.353397: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-09 16:41:11.353431: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-09 16:41:11.353244: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-09 16:41:11.353253: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-09 16:41:11.353210: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-09 16:41:11.353195: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-09 16:41:11.353266: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-09 16:41:11.353221: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-09 16:41:11.353223: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-09 16:41:11.353235: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: 2023-05-09 16:41:11.353306: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-09 16:41:11.353300: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-09 16:41:11.353246: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-09 16:41:11.353334: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-09 16:41:11.353356: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-09 16:41:11.353381: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: 2023-05-09 16:41:11.353271: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-09 16:41:11.353296: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-09 16:41:11.353299: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: 2023-05-09 16:41:11.353557: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-09 16:41:11.353562: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-09 16:41:11.353570: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: 2023-05-09 16:41:11.353482: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-09 16:41:11.353497: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-09 16:41:11.353513: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-09 16:41:11.353385: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-09 16:41:11.353350: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-09 16:41:11.353583: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-09 16:41:11.353558: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-09 16:41:11.353503: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-09 16:41:11.353525: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-09 16:41:11.353701: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: 2023-05-09 16:41:11.353356: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-09 16:41:11.353419: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-09 16:41:11.353385: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-09 16:41:11.353389: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-09 16:41:11.353622: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-09 16:41:11.353622: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-09 16:41:11.353622: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-09 16:41:11.353525: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-09 16:41:11.353537: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-09 16:41:11.353713: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-09 16:41:11.353702: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-09 16:41:11.353420: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-09 16:41:11.353429: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-09 16:41:11.353444: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-09 16:41:11.353549: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-09 16:41:11.353561: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-09 16:41:11.353576: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: 2023-05-09 16:41:11.353759: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-09 16:41:11.353509: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-09 16:41:11.354217: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-09 16:41:11.354231: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-09 16:41:11.354253: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-09 16:41:11.354262: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-09 16:41:11.354246: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-09 16:41:11.354258: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-09 16:41:11.354265: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-09 16:41:11.354289: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-09 16:41:11.354299: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-09 16:41:11.354284: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-09 16:41:11.354295: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-09 16:41:11.354305: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: 2023-05-09 16:41:11.354305: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-09 16:41:11.354288: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-09 16:41:11.354300: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-09 16:41:11.354295: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: 2023-05-09 16:41:11.354361: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-09 16:41:11.354368: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-09 16:41:11.354396: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-09 16:41:11.354308: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-09 16:41:11.354328: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-09 16:41:11.354320: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-09 16:41:11.354323: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-09 16:41:11.354332: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-09 16:41:11.354334: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-09 16:41:11.354336: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-09 16:41:11.354402: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-09 16:41:11.354391: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-09 16:41:11.354340: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-09 16:41:11.354350: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-09 16:41:11.354353: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-09 16:41:11.354412: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-09 16:41:11.354409: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-09 16:41:11.354412: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-09 16:41:11.354468: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-09 16:41:11.354480: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-09 16:41:11.354482: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: 2023-05-09 16:41:11.354485: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-09 16:41:11.354487: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-09 16:41:11.354502: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-09 16:41:11.354491: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-09 16:41:11.354509: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-09 16:41:11.354467: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-09 16:41:11.354500: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-09 16:41:11.354519: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-09 16:41:11.354523: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-09 16:41:11.354529: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: 2023-05-09 16:41:11.354502: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-09 16:41:11.354558: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-09 16:41:11.354571: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-09 16:41:11.354565: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: 2023-05-09 16:41:11.354610: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-09 16:41:11.354613: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-09 16:41:11.354578: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-09 16:41:11.354583: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-09 16:41:11.354595: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-09 16:41:11.354599: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-09 16:41:11.354611: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-09 16:41:27.871643: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:41:27.871949: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-09 16:41:27.871707: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:41:27.871967: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-09 16:41:27.871681: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:41:27.871976: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-09 16:41:27.871730: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:41:27.871999: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-09 16:41:27.871743: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:41:27.872008: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-09 16:41:27.871754: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:41:27.871998: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-09 16:41:27.871772: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:41:27.872014: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-09 16:41:27.871762: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:41:27.872015: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:41:27.872829: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-09 16:41:27.872808: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-09 16:41:27.872826: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-09 16:41:27.872852: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-09 16:41:27.872859: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-09 16:41:27.872853: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-09 16:41:27.872851: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-09 16:41:27.872863: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-09 16:41:27.872870: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-09 16:41:27.872870: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-09 16:41:27.872881: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-09 16:41:27.872880: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-09 16:41:27.872880: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-09 16:41:27.872889: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-09 16:41:27.872887: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-09 16:41:27.872892: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-09 16:41:27.877440: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:41:27.877481: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:41:27.877493: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:41:27.877512: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:41:27.877518: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:41:27.877526: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:41:27.877531: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:41:27.877538: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:41:27.877599: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:41:27.877967: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-09 16:41:27.877972: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-09 16:41:27.877622: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:41:27.878002: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-09 16:41:27.878005: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-09 16:41:27.878006: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-09 16:41:27.878020: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-09 16:41:27.878022: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-09 16:41:27.877630: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:41:27.878025: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-09 16:41:27.877771: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:41:27.877640: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:41:27.877637: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:41:27.877651: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:41:27.877784: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:41:27.877656: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:41:27.877587: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:41:27.877673: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:41:27.877660: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-09 16:41:27.877696: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:41:27.877803: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:41:27.877679: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:41:27.877918: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:41:27.877648: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:41:27.877668: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-09 16:41:27.877715: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:41:27.877817: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-09 16:41:27.877730: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:41:27.877693: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:41:27.877662: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:41:27.877718: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:41:27.877660: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-09 16:41:27.877770: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:41:27.877838: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:41:27.877699: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:41:27.877933: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:41:27.877669: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:41:27.877748: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:41:27.878220: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-09 16:41:27.877764: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:41:27.877840: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-09 16:41:27.877779: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:41:27.877705: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:41:27.877946: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:41:27.877667: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:41:27.877755: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:41:27.878262: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-09 16:41:27.878270: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-09 16:41:27.878287: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-09 16:41:27.878304: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-09 16:41:27.878301: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-09 16:41:27.877775: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:41:27.877854: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-09 16:41:27.877773: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-09 16:41:27.877704: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:41:27.877684: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:41:27.877976: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:41:27.877699: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:41:27.877713: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:41:27.878310: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-09 16:41:27.877793: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:41:27.877826: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-09 16:41:27.877800: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:41:27.877708: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:41:27.877982: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:41:27.877706: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:41:27.877765: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:41:27.878318: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-09 16:41:27.877784: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:41:27.878529: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-09 16:41:27.877774: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-09 16:41:27.877725: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:41:27.878306: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-09 16:41:27.877982: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:41:27.877743: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:41:27.877775: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-09 16:41:27.877735: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:41:27.878549: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-09 16:41:27.877800: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-09 16:41:27.877749: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:41:27.878327: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-09 16:41:27.878344: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-09 16:41:27.878358: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-09 16:41:27.878004: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:41:27.877804: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-09 16:41:27.878384: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-09 16:41:27.878560: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-09 16:41:27.877818: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-09 16:41:27.877753: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:41:27.878367: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-09 16:41:27.878374: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-09 16:41:27.878381: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-09 16:41:27.878389: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-09 16:41:27.878003: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-09 16:41:27.878410: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-09 16:41:27.878414: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-09 16:41:27.877828: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-09 16:41:27.877764: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:41:27.878572: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-09 16:41:27.878589: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-09 16:41:27.878564: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-09 16:41:27.878429: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-09 16:41:27.878430: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-09 16:41:27.878578: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-09 16:41:27.878572: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-09 16:41:27.878567: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-09 16:41:27.877784: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:41:27.878600: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-09 16:41:27.878610: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-09 16:41:27.878602: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-09 16:41:27.878615: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-09 16:41:27.878591: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-09 16:41:27.878601: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-09 16:41:27.878589: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-09 16:41:27.877792: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:41:27.878617: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-09 16:41:27.878618: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-09 16:41:27.878627: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-09 16:41:27.878628: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-09 16:41:27.878666: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-09 16:41:27.878701: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-09 16:41:27.878606: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-09 16:41:27.878624: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-09 16:41:27.878593: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-09 16:41:27.877800: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-09 16:41:27.878440: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-09 16:41:27.878442: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-09 16:41:27.878612: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-09 16:41:27.878621: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-09 16:41:27.878623: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-09 16:41:27.878643: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-09 16:41:27.878670: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-09 16:41:27.878672: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-09 16:41:27.878663: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-09 16:41:27.878700: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-09 16:41:27.878710: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-09 16:41:27.878714: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-09 16:41:27.878719: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-09 16:41:27.878732: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-09 16:41:27.878754: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-09 16:41:27.878447: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-09 16:41:27.878626: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-09 16:41:27.878684: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-09 16:41:27.878688: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-09 16:41:27.878776: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-09 16:41:27.878702: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-09 16:41:27.878707: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-09 16:41:27.878710: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-09 16:41:27.878794: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-09 16:41:27.878805: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-09 16:41:27.878485: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:41:27.878817: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-09 16:41:27.878509: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-09 16:41:27.878518: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-09 16:41:27.878513: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-09 16:41:27.878632: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-09 16:41:27.878523: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-09 16:41:27.878532: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-09 16:41:27.878530: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-09 16:41:27.878978: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-09 16:41:27.878986: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-09 16:41:27.879008: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-09 16:41:27.879012: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-09 16:41:27.879016: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-09 16:41:27.879026: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-09 16:41:27.879029: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-09 16:41:27.879037: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-09 16:41:27.878950: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:41:27.878981: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:41:27.879000: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:41:27.879026: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:41:27.879052: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:41:27.879089: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:41:27.879078: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:41:27.879077: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:41:27.879551: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-09 16:41:27.879570: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-09 16:41:27.879580: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-09 16:41:27.879587: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-09 16:41:27.879591: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-09 16:41:27.879597: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-09 16:41:27.879598: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-09 16:41:27.879605: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-09 16:41:27.879752: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:41:27.879782: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:41:27.879796: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:41:27.879805: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:41:27.879830: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:41:27.879821: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:41:27.879846: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:41:27.879833: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:41:27.880531: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-09 16:41:27.880550: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-09 16:41:27.880561: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-09 16:41:27.880569: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-09 16:41:27.880581: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-09 16:41:27.880584: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-09 16:41:27.880587: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-09 16:41:27.880594: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-09 16:41:27.880092: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:41:27.880243: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-09 16:41:27.880124: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-09 16:41:27.880154: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-09 16:41:27.880188: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-09 16:41:27.880206: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:41:27.880260: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:41:27.880181: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-09 16:41:27.880215: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:41:27.880280: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-09 16:41:27.880167: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:41:27.880291: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-09 16:41:27.880205: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:41:27.880308: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:41:27.880229: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:41:27.880316: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:41:27.880254: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:41:27.880322: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:41:27.880305: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:41:27.880326: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:41:27.880324: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:41:27.880355: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:41:27.880312: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:41:27.880356: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:41:27.880831: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-09 16:41:27.880838: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-09 16:41:27.880845: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-09 16:41:27.880798: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-09 16:41:27.880816: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-09 16:41:27.880851: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-09 16:41:27.880857: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-09 16:41:27.880863: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-09 16:41:27.880835: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-09 16:41:27.880832: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-09 16:41:27.880863: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-09 16:41:27.880891: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-09 16:41:27.880855: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-09 16:41:27.880852: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-09 16:41:27.880858: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-09 16:41:27.880883: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-09 16:41:27.880866: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-09 16:41:27.880913: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-09 16:41:27.880917: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-09 16:41:27.880929: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-09 16:41:27.880953: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-09 16:41:27.880953: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-09 16:41:27.880958: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-09 16:41:27.880965: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-09 16:41:27.880842: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:41:27.880872: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:41:27.880922: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:41:27.880940: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:41:27.880955: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:41:27.880945: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:41:27.880902: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:41:27.880950: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:41:27.881552: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-09 16:41:27.881575: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-09 16:41:27.881584: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-09 16:41:27.881597: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-09 16:41:27.881260: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:41:27.881293: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:41:27.881311: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:41:27.881355: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:41:27.881373: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:41:27.881377: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:41:27.881392: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:41:27.881379: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:41:27.881610: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-09 16:41:27.881608: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-09 16:41:27.881622: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-09 16:41:27.881621: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-09 16:41:27.882008: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-09 16:41:27.882024: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-09 16:41:27.882055: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-09 16:41:27.882072: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-09 16:41:27.881567: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:41:27.882091: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-09 16:41:27.882100: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-09 16:41:27.882117: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-09 16:41:27.881599: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:41:27.882132: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-09 16:41:27.881631: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-09 16:41:27.881671: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-09 16:41:27.881647: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-09 16:41:27.881674: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-09 16:41:27.881661: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-09 16:41:27.881699: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-09 16:41:27.882350: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-09 16:41:27.882388: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-09 16:41:27.882402: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-09 16:41:27.882423: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-09 16:41:27.882429: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-09 16:41:27.882446: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-09 16:41:27.882459: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-09 16:41:27.882465: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-09 16:41:27.882034: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:41:27.882050: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:41:27.882088: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:41:27.882112: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:41:27.882099: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:41:27.882124: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:41:27.882255: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:41:27.882129: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:41:27.882155: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:41:27.882281: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:41:27.882305: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:41:27.882319: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:41:27.882834: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-09 16:41:27.882829: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-09 16:41:27.882488: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:41:27.882336: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-09 16:41:27.882525: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:41:27.882354: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-09 16:41:27.882537: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:41:27.882343: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:41:27.882871: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-09 16:41:27.882878: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-09 16:41:27.882542: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:41:27.882358: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:41:27.882909: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-09 16:41:27.882888: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-09 16:41:27.882911: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-09 16:41:27.882918: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-09 16:41:27.882548: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:41:27.882473: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:41:27.882903: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-09 16:41:27.882922: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-09 16:41:27.882944: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-09 16:41:27.882955: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-09 16:41:27.882959: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-09 16:41:27.882507: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:41:27.882968: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-09 16:41:27.882965: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-09 16:41:27.882518: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:41:27.882975: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-09 16:41:27.882546: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:41:27.882570: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-09 16:41:27.882466: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:41:27.882579: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:41:27.882587: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-09 16:41:27.882505: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:41:27.882607: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-09 16:41:27.882534: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-09 16:41:27.882576: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:41:27.883129: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-09 16:41:27.882625: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:41:27.883150: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-09 16:41:27.883157: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-09 16:41:27.882586: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-09 16:41:27.882602: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-09 16:41:27.882602: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:41:27.883171: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-09 16:41:27.883174: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-09 16:41:27.883180: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-09 16:41:27.883185: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-09 16:41:27.883187: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-09 16:41:27.883144: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-09 16:41:27.883179: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-09 16:41:27.883190: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-09 16:41:27.883224: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-09 16:41:27.883228: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-09 16:41:27.883230: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-09 16:41:27.883254: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-09 16:41:27.883282: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-09 16:41:27.882984: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:41:27.883014: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:41:27.883035: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:41:27.883057: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:41:27.883073: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:41:27.883077: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:41:27.883096: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-09 16:41:27.883220: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:41:27.883057: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-09 16:41:27.883247: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-09 16:41:27.883264: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-09 16:41:27.883274: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:41:27.883712: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-09 16:41:27.883289: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-09 16:41:27.883196: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-09 16:41:27.883288: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-09 16:41:27.883304: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-09 16:41:27.883307: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-09 16:41:27.883230: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-09 16:41:27.883274: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:41:27.883745: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-09 16:41:27.883305: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:41:27.883766: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-09 16:41:27.883780: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-09 16:41:27.883285: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:41:27.883796: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-09 16:41:27.883795: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-09 16:41:27.883810: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-09 16:41:27.883311: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:41:27.883822: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-09 16:41:27.883321: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-09 16:41:27.883332: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-09 16:41:27.883818: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-09 16:41:27.883842: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-09 16:41:27.883844: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-09 16:41:27.883497: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-09 16:41:27.883857: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-09 16:41:27.883864: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-09 16:41:27.883520: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-09 16:41:27.883876: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-09 16:41:27.883876: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-09 16:41:27.883883: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-09 16:41:27.883535: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-09 16:41:27.883852: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-09 16:41:27.883873: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-09 16:41:27.883543: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-09 16:41:27.883550: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-09 16:41:27.883558: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-09 16:41:27.883522: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-09 16:41:27.883881: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-09 16:41:27.883577: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-09 16:41:27.883892: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-09 16:41:27.883901: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-09 16:41:27.883903: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-09 16:41:27.884030: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-09 16:41:27.883914: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-09 16:41:27.883920: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-09 16:41:27.884056: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-09 16:41:27.884058: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-09 16:41:27.884071: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-09 16:41:27.884074: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-09 16:41:27.884081: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-09 16:41:27.884084: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-09 16:41:27.884092: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-09 16:41:27.883568: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:41:27.883540: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:41:27.883610: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:41:27.883609: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:41:27.883573: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:41:27.883637: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:41:27.883582: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:41:27.883655: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:41:27.883597: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:41:27.883663: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:41:27.883609: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:41:27.883661: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:41:27.883617: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:41:27.883685: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:41:27.883623: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:41:27.884309: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-09 16:41:27.883638: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:41:27.884306: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-09 16:41:27.884325: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-09 16:41:27.884327: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-09 16:41:27.884333: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-09 16:41:27.884345: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-09 16:41:27.884353: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-09 16:41:27.884356: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-09 16:41:27.884364: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-09 16:41:27.884346: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-09 16:41:27.884354: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-09 16:41:27.884357: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-09 16:41:27.884361: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-09 16:41:27.884372: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-09 16:41:27.884378: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-09 16:41:27.884375: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-09 16:41:27.882564: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-09 16:41:27.882561: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-09 16:41:27.882554: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-09 16:41:27.882987: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-09 16:41:27.883007: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-09 16:41:27.883024: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-09 16:41:27.883036: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-09 16:41:27.883045: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-09 16:41:27.883050: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-09 16:41:27.883054: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-09 16:41:27.883056: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-09 16:42:01.084037: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:42:01.084065: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:42:01.084087: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:42:01.084101: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:42:01.084114: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:42:01.084115: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:42:01.084129: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:42:01.084139: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:42:01.085915: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:42:01.085944: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:42:01.085990: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:42:01.085938: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:42:01.086012: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:42:01.085956: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:42:01.086020: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:42:01.085966: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:42:01.086034: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:42:01.085965: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:42:01.086033: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:42:01.085973: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:42:01.086057: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:42:01.085971: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:42:01.086062: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:42:01.086070: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:42:01.086502: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:42:01.086528: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:42:01.086552: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:42:01.086564: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:42:01.086563: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:42:01.086574: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:42:01.086592: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:42:01.086597: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:42:01.087057: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:42:01.087058: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:42:01.087068: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-09 16:42:01.087037: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:42:01.087059: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:42:01.087065: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-09 16:42:01.087055: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:42:01.087066: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-09 16:42:01.087074: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:42:01.087068: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:42:01.087094: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-09 16:42:01.087076: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:42:01.087094: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-09 16:42:01.087099: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-09 16:42:01.087101: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-09 16:42:01.087079: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:42:01.087103: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-09 16:42:01.087104: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-09 16:42:01.087102: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-09 16:42:01.087070: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:42:01.087161: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-09 16:42:01.087187: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-09 16:42:01.087095: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-09 16:42:01.087207: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:42:01.087301: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:42:01.087423: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:42:01.087302: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:42:01.087301: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:42:01.087425: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:42:01.087301: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:42:01.087430: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:42:01.087302: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:42:01.087430: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:42:01.087302: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:42:01.087430: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:42:01.087307: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:42:01.087434: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:42:01.087439: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-09 16:42:01.087309: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:42:01.087322: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-09 16:42:01.087439: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-09 16:42:01.087436: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-09 16:42:01.087323: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-09 16:42:01.087324: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-09 16:42:01.087326: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-09 16:42:01.087436: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-09 16:42:01.087446: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-09 16:42:01.087327: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-09 16:42:01.087325: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-09 16:42:01.087326: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-09 16:42:01.087449: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-09 16:42:01.087450: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-09 16:42:01.087449: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-09 16:42:01.087327: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-09 16:42:01.087453: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-09 16:42:01.087453: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-09 16:42:01.087882: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:42:01.087889: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:42:01.087887: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:42:01.087891: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:42:01.087890: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:42:01.087890: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:42:01.087895: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:42:01.087898: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-09 16:42:01.087890: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-09 16:42:01.087903: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-09 16:42:01.087908: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-09 16:42:01.087908: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-09 16:42:01.087912: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-09 16:42:01.087910: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-09 16:42:01.087911: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-09 16:42:01.087914: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-09 16:42:01.088381: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-09 16:42:01.088445: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-09 16:42:01.088384: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-09 16:42:01.088383: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-09 16:42:01.088465: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-09 16:42:01.088384: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-09 16:42:01.088479: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-09 16:42:01.088389: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-09 16:42:01.088485: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-09 16:42:01.088389: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-09 16:42:01.088497: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-09 16:42:01.088391: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-09 16:42:01.088511: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-09 16:42:01.088393: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-09 16:42:01.088399: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-09 16:42:01.088521: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-09 16:42:01.088401: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-09 16:42:01.088400: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-09 16:42:01.088405: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-09 16:42:01.088529: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-09 16:42:01.088406: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-09 16:42:01.088407: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-09 16:42:01.088408: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-09 16:42:01.088404: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-09 16:42:01.088608: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:42:01.088644: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:42:01.088665: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:42:01.088689: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:42:01.088694: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:42:01.088700: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:42:01.088713: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:42:01.088730: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:42:01.089291: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-09 16:42:01.089432: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:42:01.089323: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-09 16:42:01.089452: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:42:01.089333: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-09 16:42:01.089458: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:42:01.089353: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-09 16:42:01.089471: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:42:01.089385: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-09 16:42:01.089475: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:42:01.089408: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:42:01.089392: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-09 16:42:01.089480: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:42:01.089399: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-09 16:42:01.089487: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:42:01.089449: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:42:01.089467: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:42:01.089493: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:42:01.089510: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:42:01.089531: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:42:01.089536: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:42:01.089546: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:42:01.089559: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-09 16:42:01.089642: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:42:01.089664: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-09 16:42:01.089661: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-09 16:42:01.089676: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:42:01.089744: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:42:01.089682: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-09 16:42:01.089690: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:42:01.089692: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-09 16:42:01.089696: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:42:01.089703: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-09 16:42:01.089706: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:42:01.089762: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:42:01.089707: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-09 16:42:01.089713: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:42:01.089772: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:42:01.089715: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-09 16:42:01.089719: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:42:01.089785: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:42:01.089728: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:42:01.089782: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:42:01.089730: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:42:01.089790: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-09 16:42:01.089745: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:42:01.089800: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:42:01.089844: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-09 16:42:01.089764: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-09 16:42:01.089772: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-09 16:42:01.089784: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-09 16:42:01.089798: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-09 16:42:01.089802: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-09 16:42:01.089798: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-09 16:42:01.089815: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-09 16:42:01.089993: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-09 16:42:01.090014: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-09 16:42:01.090033: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-09 16:42:01.090042: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-09 16:42:01.090054: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-09 16:42:01.090057: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-09 16:42:01.090052: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-09 16:42:01.090061: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:42:01.090149: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-09 16:42:01.090029: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:42:01.090167: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:42:01.090179: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:42:01.090188: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-09 16:42:01.090047: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:42:01.090199: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-09 16:42:01.090053: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:42:01.090205: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-09 16:42:01.090063: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:42:01.090213: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-09 16:42:01.090068: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:42:01.090219: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:42:01.090331: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-09 16:42:01.090070: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-09 16:42:01.090175: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:42:01.090311: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-09 16:42:01.090179: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:42:01.090347: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:42:01.090361: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:42:01.090332: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:42:01.090376: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:42:01.090351: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:42:01.090379: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:42:01.090361: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:42:01.090384: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:42:01.090372: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:42:01.090385: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:42:01.090370: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:42:01.090394: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:42:01.090380: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-09 16:42:01.090456: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:42:01.090381: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-09 16:42:01.090473: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-09 16:42:01.090485: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-09 16:42:01.090485: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-09 16:42:01.090503: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-09 16:42:01.090504: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-09 16:42:01.090521: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-09 16:42:01.090528: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:42:01.090628: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:42:01.090638: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:42:01.090666: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-09 16:42:01.090802: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:42:01.090670: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-09 16:42:01.090801: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:42:01.090682: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-09 16:42:01.090804: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:42:01.090704: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-09 16:42:01.090801: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:42:01.090700: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-09 16:42:01.090805: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:42:01.090706: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-09 16:42:01.090807: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-09 16:42:01.090810: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:42:01.090955: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-09 16:42:01.090810: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-09 16:42:01.090822: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-09 16:42:01.090822: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-09 16:42:01.090822: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-09 16:42:01.090823: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-09 16:42:01.090958: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-09 16:42:01.090824: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-09 16:42:01.090824: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-09 16:42:01.090825: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-09 16:42:01.090959: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:42:01.090971: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-09 16:42:01.090826: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-09 16:42:01.090960: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:42:01.090961: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:42:01.090962: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:42:01.090966: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:42:01.090966: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-09 16:42:01.090983: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-09 16:42:01.090984: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-09 16:42:01.090986: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-09 16:42:01.090986: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-09 16:42:01.090987: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-09 16:42:01.090989: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-09 16:42:01.090989: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-09 16:42:01.091075: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:42:01.091075: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:42:01.091225: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:42:01.091077: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:42:01.091079: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:42:01.091207: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:42:01.091080: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-09 16:42:01.091180: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:42:01.091215: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:42:01.091226: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:42:01.091079: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:42:01.091208: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:42:01.091234: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:42:01.091082: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-09 16:42:01.091179: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:42:01.091208: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-09 16:42:01.091263: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:42:01.091218: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:42:01.091234: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:42:01.091242: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-09 16:42:01.091088: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:42:01.091097: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-09 16:42:01.091183: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:42:01.091209: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-09 16:42:01.091263: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:42:01.091221: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:42:01.091231: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-09 16:42:01.091241: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-09 16:42:01.091237: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:42:01.091097: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-09 16:42:01.091098: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-09 16:42:01.091099: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-09 16:42:01.091182: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:42:01.091213: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-09 16:42:01.091262: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:42:01.091222: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:42:01.091249: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-09 16:42:01.091100: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-09 16:42:01.091100: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-09 16:42:01.091101: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-09 16:42:01.091184: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:42:01.091215: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-09 16:42:01.091264: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:42:01.091226: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:42:01.091239: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-09 16:42:01.091105: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-09 16:42:01.091181: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:42:01.091217: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-09 16:42:01.091266: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:42:01.091222: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:42:01.091241: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-09 16:42:01.091359: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-09 16:42:01.091187: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:42:01.091216: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-09 16:42:01.091226: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-09 16:42:01.091267: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:42:01.091223: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:42:01.091239: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-09 16:42:01.091253: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-09 16:42:01.091361: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-09 16:42:01.091189: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-09 16:42:01.091202: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-09 16:42:01.091227: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-09 16:42:01.091227: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-09 16:42:01.091228: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-09 16:42:01.091266: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:42:01.091225: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-09 16:42:01.091241: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-09 16:42:01.091259: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-09 16:42:01.091262: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-09 16:42:01.091263: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-09 16:42:01.091361: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-09 16:42:01.091201: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-09 16:42:01.091201: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-09 16:42:01.091203: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-09 16:42:01.091230: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-09 16:42:01.091231: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-09 16:42:01.091233: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-09 16:42:01.091268: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-09 16:42:01.091287: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-09 16:42:01.091243: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-09 16:42:01.091248: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-09 16:42:01.091249: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-09 16:42:01.091263: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-09 16:42:01.091364: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-09 16:42:01.091360: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-09 16:42:01.091203: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-09 16:42:01.091205: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-09 16:42:01.091204: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-09 16:42:01.091234: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-09 16:42:01.091286: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-09 16:42:01.091287: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-09 16:42:01.091288: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-09 16:42:01.091250: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-09 16:42:01.091251: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-09 16:42:01.091251: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-09 16:42:01.091363: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-09 16:42:01.091203: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-09 16:42:01.091291: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-09 16:42:01.091289: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-09 16:42:01.091292: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-09 16:42:01.091364: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-09 16:42:01.091293: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-09 16:42:01.091366: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-09 16:42:01.091368: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-09 16:42:01.091367: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:42:01.091512: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-09 16:42:01.091370: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-09 16:42:01.091375: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-09 16:42:01.091369: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-09 16:42:01.091377: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-09 16:42:01.091380: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-09 16:42:01.091383: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-09 16:42:01.091373: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-09 16:42:01.091381: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-09 16:42:01.091381: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-09 16:42:01.091382: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-09 16:42:01.091375: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:42:01.091515: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-09 16:42:01.091384: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-09 16:42:01.091371: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:42:01.091517: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:42:01.091568: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-09 16:42:01.091371: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-09 16:42:01.091383: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-09 16:42:01.091521: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-09 16:42:01.091383: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-09 16:42:01.091384: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-09 16:42:01.091383: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-09 16:42:01.091523: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:42:01.091528: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-09 16:42:01.091390: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-09 16:42:01.091389: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-09 16:42:01.091391: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-09 16:42:01.091528: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-09 16:42:01.091537: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-09 16:42:01.091392: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-09 16:42:01.091527: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:42:01.091572: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:42:01.091526: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:42:01.091574: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:42:01.091528: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:42:01.091540: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-09 16:42:01.091575: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-09 16:42:01.091541: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-09 16:42:01.091549: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-09 16:42:01.091550: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-09 16:42:01.091574: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:42:01.091586: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-09 16:42:01.091551: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-09 16:42:01.091575: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:42:01.091576: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:42:01.091575: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:42:01.091587: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-09 16:42:01.091694: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-09 16:42:01.091591: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-09 16:42:01.091592: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-09 16:42:01.091593: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-09 16:42:01.091599: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-09 16:42:01.091600: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-09 16:42:01.091601: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-09 16:42:01.091696: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:42:01.091694: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:42:01.091698: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:42:01.091699: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-09 16:42:01.091790: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:42:01.091702: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:42:01.091703: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-09 16:42:01.091790: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:42:01.091704: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:42:01.091713: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-09 16:42:01.091790: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:42:01.091714: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-09 16:42:01.091714: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-09 16:42:01.091715: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-09 16:42:01.091792: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:42:01.091716: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-09 16:42:01.091718: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-09 16:42:01.091719: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-09 16:42:01.091795: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-09 16:42:01.091716: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-09 16:42:01.091791: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-09 16:42:01.091796: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-09 16:42:01.091796: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-09 16:42:01.091814: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-09 16:42:01.091815: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-09 16:42:01.091815: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-09 16:42:01.091817: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-09 16:42:01.091817: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-09 16:42:01.091819: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-09 16:42:01.091816: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-09 16:42:01.091820: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-09 16:42:01.092251: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:42:01.092253: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:42:01.092252: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:42:01.092256: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:42:01.092255: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:42:01.092256: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:42:01.092260: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:42:01.092263: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-09 16:42:01.092272: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-09 16:42:01.092272: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-09 16:42:01.092274: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-09 16:42:01.092275: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-09 16:42:01.092273: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-09 16:42:01.092275: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-09 16:42:01.092276: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-09 16:42:01.092276: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-09 16:42:01.101633: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-09 16:42:01.101556: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-09 16:42:01.101650: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-09 16:42:01.101591: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-09 16:42:01.101660: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-09 16:42:01.101604: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-09 16:42:01.101674: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-09 16:42:01.101613: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-09 16:42:01.101678: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-09 16:42:01.101632: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-09 16:42:01.101684: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-09 16:42:01.101653: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-09 16:42:01.101694: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-09 16:42:01.101848: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-09 16:42:01.101794: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-09 16:42:01.101851: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-09 16:42:01.102642: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-09 16:42:01.102647: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-09 16:42:01.102644: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-09 16:42:01.102660: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-09 16:42:01.102650: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-09 16:42:01.102643: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-09 16:42:01.102643: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-09 16:42:01.102647: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-09 16:42:01.102665: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-09 16:42:01.102672: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-09 16:42:01.102673: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-09 16:42:01.102674: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-09 16:42:01.102841: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-09 16:42:01.102858: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-09 16:42:01.102675: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-09 16:42:01.102675: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-09 16:42:01.103139: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:42:01.103163: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:42:01.103198: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:42:01.103211: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:42:01.103228: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:42:01.103228: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:42:01.103240: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:42:01.103419: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:42:01.104027: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:42:01.104063: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:42:01.104078: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:42:01.104094: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:42:01.104117: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:42:01.104123: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:42:01.104318: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:42:01.104321: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:42:01.104473: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:42:01.104481: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:42:01.104490: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-09 16:42:01.104485: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:42:01.104496: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-09 16:42:01.104489: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:42:01.104488: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:42:01.104489: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:42:01.104490: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:42:01.104497: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-09 16:42:01.104509: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-09 16:42:01.104513: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-09 16:42:01.104514: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-09 16:42:01.104516: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-09 16:42:01.104516: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-09 16:42:01.104519: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-09 16:42:01.104789: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-09 16:42:01.104836: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-09 16:42:01.104876: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-09 16:42:01.104881: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-09 16:42:01.104903: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-09 16:42:01.104914: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-09 16:42:01.104926: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-09 16:42:01.104928: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:42:01.105632: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:42:01.105667: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:42:01.105682: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:42:01.105699: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:42:01.105721: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:42:01.105731: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:42:01.105747: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:42:01.105978: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-09 16:42:01.106128: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-09 16:42:01.106143: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-09 16:42:01.106143: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-09 16:42:01.106154: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-09 16:42:01.106146: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-09 16:42:01.106147: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-09 16:42:01.106148: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-09 16:42:01.106143: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-09 16:42:01.106155: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-09 16:42:01.106172: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-09 16:42:01.106173: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-09 16:42:01.106175: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-09 16:42:01.106176: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-09 16:42:01.106177: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-09 16:42:01.106180: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-09 16:42:01.106180: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-09 16:42:01.106318: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:42:01.106352: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:42:01.106372: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:42:01.106389: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:42:01.106404: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:42:01.106418: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:42:01.106433: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:42:01.106503: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:42:01.106815: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:42:01.106852: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:42:01.106874: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:42:01.106886: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:42:01.106900: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:42:01.106903: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:42:01.106921: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:42:01.106927: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:42:01.106981: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:42:01.106996: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-09 16:42:01.106947: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-09 16:42:01.106974: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:42:01.106995: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:42:01.106998: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:42:01.106999: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:42:01.107004: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:42:01.107006: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:42:01.106981: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:42:01.107022: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-09 16:42:01.107023: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-09 16:42:01.107004: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:42:01.106995: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:42:01.107014: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-09 16:42:01.107029: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:42:01.107012: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:42:01.107012: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:42:01.107022: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-09 16:42:01.107043: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:42:01.107019: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:42:01.107027: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-09 16:42:01.107029: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-09 16:42:01.107035: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-09 16:42:01.107055: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:42:01.107021: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-09 16:42:01.107036: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-09 16:42:01.107061: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:42:01.107204: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-09 16:42:01.107071: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:42:01.107224: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-09 16:42:01.107247: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:42:01.107637: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:42:01.107641: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:42:01.107642: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:42:01.107643: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:42:01.107643: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:42:01.107645: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:42:01.107649: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:42:01.107646: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-09 16:42:01.107667: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-09 16:42:01.107667: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-09 16:42:01.107669: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-09 16:42:01.107671: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-09 16:42:01.107671: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-09 16:42:01.107673: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-09 16:42:01.107672: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-09 16:42:01.107673: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-09 16:42:01.107834: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:42:01.107848: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-09 16:42:01.107843: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:42:01.107844: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:42:01.107852: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:42:01.107850: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:42:01.107852: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:42:01.107853: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:42:01.107858: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-09 16:42:01.107868: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-09 16:42:01.107875: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-09 16:42:01.107876: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-09 16:42:01.107877: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-09 16:42:01.107878: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-09 16:42:01.107879: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-09 16:42:01.107876: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-09 16:42:01.108256: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-09 16:42:01.108464: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-09 16:42:01.108464: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-09 16:42:01.108466: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-09 16:42:01.108467: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:42:01.108472: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-09 16:42:01.108469: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-09 16:42:01.108468: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-09 16:42:01.108472: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-09 16:42:01.108464: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-09 16:42:01.108493: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-09 16:42:01.108493: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-09 16:42:01.108494: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-09 16:42:01.108495: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-09 16:42:01.108496: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-09 16:42:01.108497: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-09 16:42:01.108498: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-09 16:42:01.108502: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-09 16:42:01.108487: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:42:01.108504: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:42:01.108515: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:42:01.108514: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:42:01.108509: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:42:01.108626: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:42:01.108635: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:42:01.109320: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:42:01.109324: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:42:01.109326: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:42:01.109327: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:42:01.109327: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:42:01.109330: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:42:01.109331: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:42:01.109338: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-09 16:42:01.109339: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-09 16:42:01.109332: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-09 16:42:01.109342: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-09 16:42:01.109345: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-09 16:42:01.109344: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-09 16:42:01.109347: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-09 16:42:01.109349: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-09 16:42:01.109356: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-09 16:42:01.089502: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-09 16:42:01.090929: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-09 16:42:01.090933: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-09 16:42:01.090934: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-09 16:42:01.090934: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-09 16:42:01.090938: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-09 16:42:01.090937: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-09 16:42:01.090938: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-09 16:42:01.090943: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-09 16:42:01.090947: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-09 16:42:01.090947: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-09 16:42:01.090953: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-09 16:42:01.090952: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-09 16:42:01.090954: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-09 16:42:01.090955: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-09 16:42:01.090956: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-09 16:42:01.090957: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-09 16:42:01.102786: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-09 16:42:01.102802: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-09 16:42:01.102868: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-09 16:42:01.102871: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-09 16:42:01.102878: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-09 16:42:01.102891: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-09 16:42:01.102896: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-09 16:42:01.102885: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-09 16:42:01.102899: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-09 16:42:01.102900: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-09 16:42:01.102914: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-09 16:42:01.102899: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-09 16:42:01.102902: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-09 16:42:01.102929: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-09 16:42:01.102931: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-09 16:42:01.102931: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-09 16:42:01.105230: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:42:01.105233: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:42:01.105234: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:42:01.105235: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:42:01.105236: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:42:01.105240: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:42:01.105238: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:42:01.105242: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-09 16:42:01.105259: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-09 16:42:01.105258: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-09 16:42:01.105259: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-09 16:42:01.105262: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-09 16:42:01.105262: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-09 16:42:01.105264: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-09 16:42:01.105261: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-09 16:42:01.105264: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-09 16:42:01.108255: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:42:01.108263: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:42:01.108264: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:42:01.108266: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:42:01.108278: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-09 16:42:01.108278: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-09 16:42:01.108268: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:42:01.108272: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:42:01.108287: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-09 16:42:01.108276: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-09 16:42:01.108297: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-09 16:42:01.108298: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-09 16:42:01.108301: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-09 16:42:01.108302: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-09 16:42:01.108304: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: Successfully preprocessed all matching files. + 0: Detected CUDA files, patching ldflags + 0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... + 0: Building extension module scaled_upper_triang_masked_softmax_cuda... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module scaled_upper_triang_masked_softmax_cuda... + 0: Successfully preprocessed all matching files. + 0: Detected CUDA files, patching ldflags + 0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... + 0: Building extension module scaled_masked_softmax_cuda... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module scaled_masked_softmax_cuda... + 0: Successfully preprocessed all matching files. + 0: Detected CUDA files, patching ldflags + 0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... + 0: Building extension module fused_mix_prec_layer_norm_cuda... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module fused_mix_prec_layer_norm_cuda... + 8: Successfully preprocessed all matching files. + 8: Successfully preprocessed all matching files. +15: Successfully preprocessed all matching files. +15: Successfully preprocessed all matching files. +19: Successfully preprocessed all matching files. +19: Successfully preprocessed all matching files. +16: Successfully preprocessed all matching files. +16: Successfully preprocessed all matching files. +31: Successfully preprocessed all matching files. +31: Successfully preprocessed all matching files. + 1: Successfully preprocessed all matching files. + 1: Successfully preprocessed all matching files. +25: Successfully preprocessed all matching files. +25: Successfully preprocessed all matching files. +27: Successfully preprocessed all matching files. +27: Successfully preprocessed all matching files. +27: Successfully preprocessed all matching files. +16: Successfully preprocessed all matching files. + 0: Successfully preprocessed all matching files. +25: Successfully preprocessed all matching files. +15: Successfully preprocessed all matching files. +25: Successfully preprocessed all matching files. + 1: Successfully preprocessed all matching files. +23: Successfully preprocessed all matching files. +28: Successfully preprocessed all matching files. +29: Successfully preprocessed all matching files. +26: Successfully preprocessed all matching files. +26: Successfully preprocessed all matching files. +26: Successfully preprocessed all matching files. +26: Successfully preprocessed all matching files. +29: Successfully preprocessed all matching files. +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: + 3: + 3: + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: + 2: + 2: + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: + 4: + 4: + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: + 5: + 5: +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: + 7: + 7: + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: + 6: + 6: +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: + 9: + 9: +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: +11: +11: +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: +10: +10: +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: +12: +12: +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: +13: +13: +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: +14: +14: +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: +17: +17: +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: +18: +18: +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: +20: +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: +21: +21: +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: +22: +22: +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: +24: +24: +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: +30: +30: + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... + 0: Building extension module utils... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module utils... +10: Loading extension module utils... +15: Loading extension module utils... +22: Loading extension module utils... +24: Loading extension module utils... +25: Loading extension module utils... + 2: Loading extension module utils... + 7: Loading extension module utils... +12: Loading extension module utils... +29: Loading extension module utils... +31: Loading extension module utils... +18: Loading extension module utils... +19: Loading extension module utils... +21: Loading extension module utils... +20: Loading extension module utils... +24: Loading extension module utils... + 4: Loading extension module utils... + 5: Loading extension module utils... +28: Loading extension module utils... + 6: Loading extension module utils... +13: Loading extension module utils... + 9: Loading extension module utils... +27: Loading extension module utils... +16: Loading extension module utils... +17: Loading extension module utils... +14: Loading extension module utils... +11: Loading extension module utils... +12: Loading extension module utils... +26: Loading extension module utils... +22: Loading extension module utils... + 5: Loading extension module utils... + 7: Loading extension module utils... +28: Loading extension module utils... +25: Loading extension module utils... +10: Loading extension module utils... +30: Loading extension module utils... + 2: Loading extension module utils... +15: Loading extension module utils... +29: Loading extension module utils... +20: Loading extension module utils... + 4: Loading extension module utils... +31: Loading extension module utils... +19: Loading extension module utils... +21: Loading extension module utils... +24: Loading extension module utils... +17: Loading extension module utils... +13: Loading extension module utils... +11: Loading extension module utils... +27: Loading extension module utils... + 6: Loading extension module utils... +26: Loading extension module utils... + 7: Loading extension module utils... +16: Loading extension module utils... +10: Loading extension module utils... +15: Loading extension module utils... +22: Loading extension module utils... +18: Loading extension module utils... + 9: Loading extension module utils... +30: Loading extension module utils... +14: Loading extension module utils... +25: Loading extension module utils... +20: Loading extension module utils... +24: Loading extension module utils... + 5: Loading extension module utils... +28: Loading extension module utils... + 2: Loading extension module utils... + 7: Loading extension module utils... +29: Loading extension module utils... +18: Loading extension module utils... +21: Loading extension module utils... + 4: Loading extension module utils... +12: Loading extension module utils... +16: Loading extension module utils... +31: Loading extension module utils... +15: Loading extension module utils... +11: Loading extension module utils... +26: Loading extension module utils... +19: Loading extension module utils... + 5: Loading extension module utils... + 6: Loading extension module utils... +29: Loading extension module utils... +27: Loading extension module utils... +30: Loading extension module utils... +17: Loading extension module utils... +14: Loading extension module utils... +13: Loading extension module utils... +25: Loading extension module utils... +22: Loading extension module utils... + 9: Loading extension module utils... +10: Loading extension module utils... +28: Loading extension module utils... +21: Loading extension module utils... +20: Loading extension module utils... + 2: Loading extension module utils... + 4: Loading extension module utils... +19: Loading extension module utils... +16: Loading extension module utils... + 6: Loading extension module utils... +12: Loading extension module utils... +13: Loading extension module utils... +11: Loading extension module utils... +31: Loading extension module utils... +30: Loading extension module utils... +17: Loading extension module utils... +18: Loading extension module utils... + 9: Loading extension module utils... +27: Loading extension module utils... +26: Loading extension module utils... +14: Loading extension module utils... + 0: Loading extension module utils... + 0: Loading extension module utils... + 0: Loading extension module utils... + 3: Loading extension module utils... + 8: Loading extension module utils... + 3: Loading extension module utils... + 8: Loading extension module utils... + 3: Loading extension module utils... +23: Loading extension module utils... + 3: Loading extension module utils... + 8: Loading extension module utils... +23: Loading extension module utils... + 8: Loading extension module utils... + 1: Loading extension module utils... +23: Loading extension module utils... + 1: Loading extension module utils... +23: Loading extension module utils... + 1: Loading extension module utils... + 1: Loading extension module utils... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +29: +29: Loading extension module utils... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +28: +28: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +25: +25: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +25: +25: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... + 7: + 7: Loading extension module utils... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +19: +19: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils... + 3: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 3: + 3: Loading extension module utils...Loading extension module utils... + 3: + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: +12: +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +17: +17: Loading extension module utils... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +26: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step... +20: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 4: + 4: Loading extension module utils...Loading extension module utils... + 4: + 5: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +23: +23: Loading extension module utils...Loading extension module utils... +23: + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +21: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +21: +21: Loading extension module utils...Loading extension module utils... +21: +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +12: +12: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... +10: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +20: +20: Loading extension module utils...Loading extension module utils... +20: +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +24: +24: Loading extension module utils...Loading extension module utils... +24: +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step... +20: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... + 6: + 6: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: +11: +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... + 0: Building extension module utils... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module utils... + 0: Loading extension module utils... + 0: Loading extension module utils... + 0: Loading extension module utils... +12: Loading extension module utils... + 3: Loading extension module utils... + 5: Loading extension module utils... +21: Loading extension module utils... +12: Loading extension module utils... + 3: Loading extension module utils... +11: Loading extension module utils... +17: Loading extension module utils... +12: Loading extension module utils... + 5: Loading extension module utils... +30: Loading extension module utils... +21: Loading extension module utils... +20: Loading extension module utils... +13: Loading extension module utils... +21: Loading extension module utils... + 3: Loading extension module utils... +11: Loading extension module utils... +17: Loading extension module utils... + 5: Loading extension module utils... +14: Loading extension module utils... +30: Loading extension module utils... + 4: Loading extension module utils... +12: Loading extension module utils... +20: Loading extension module utils... +13: Loading extension module utils... + 3: Loading extension module utils... +21: Loading extension module utils... +11: Loading extension module utils... +17: Loading extension module utils... + 5: Loading extension module utils... +14: Loading extension module utils... +30: Loading extension module utils... + 4: Loading extension module utils... +20: Loading extension module utils... +13: Loading extension module utils... +11: Loading extension module utils... + 4: Loading extension module utils... +13: Loading extension module utils... +17: Loading extension module utils... +14: Loading extension module utils... +30: Loading extension module utils... + 4: Loading extension module utils... +20: Loading extension module utils... + 2: Loading extension module utils... +14: Loading extension module utils... + 2: Loading extension module utils... + 9: Loading extension module utils... + 2: Loading extension module utils... + 7: Loading extension module utils... + 9: Loading extension module utils... + 1: Loading extension module utils... + 8: Loading extension module utils... + 7: Loading extension module utils... + 9: Loading extension module utils... + 1: Loading extension module utils... + 2: Loading extension module utils... + 8: Loading extension module utils... +19: Loading extension module utils... + 7: Loading extension module utils... + 9: Loading extension module utils... + 1: Loading extension module utils... + 8: Loading extension module utils... +19: Loading extension module utils... + 7: Loading extension module utils... + 1: Loading extension module utils... +16: Loading extension module utils... +23: Loading extension module utils... + 8: Loading extension module utils... +19: Loading extension module utils... +16: Loading extension module utils... +10: Loading extension module utils... +23: Loading extension module utils... +19: Loading extension module utils... +10: Loading extension module utils... +23: Loading extension module utils... +16: Loading extension module utils... +29: Loading extension module utils... +10: Loading extension module utils... +28: Loading extension module utils... +23: Loading extension module utils... +15: Loading extension module utils... +16: Loading extension module utils... +25: Loading extension module utils... +26: Loading extension module utils... +31: Loading extension module utils... +29: Loading extension module utils... +10: Loading extension module utils... +28: Loading extension module utils... +15: Loading extension module utils... +25: Loading extension module utils... + 6: Loading extension module utils... +26: Loading extension module utils... +31: Loading extension module utils... +22: Loading extension module utils... +18: Loading extension module utils... +29: Loading extension module utils... +28: Loading extension module utils... +24: Loading extension module utils... +15: Loading extension module utils... +25: Loading extension module utils... + 6: Loading extension module utils... +26: Loading extension module utils... +31: Loading extension module utils... +22: Loading extension module utils... +27: Loading extension module utils... +25: Loading extension module utils... +18: Loading extension module utils... +29: Loading extension module utils... +28: Loading extension module utils... +24: Loading extension module utils... +15: Loading extension module utils... + 6: Loading extension module utils... +26: Loading extension module utils... +31: Loading extension module utils... +22: Loading extension module utils... +27: Loading extension module utils... +18: Loading extension module utils... +24: Loading extension module utils... + 6: Loading extension module utils... +22: Loading extension module utils... +27: Loading extension module utils... +18: Loading extension module utils... +24: Loading extension module utils... +27: Loading extension module utils... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +12: +12: Loading extension module utils...Loading extension module utils... +12: +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 0: + 0: + 0: Loading extension module utils...Loading extension module utils...Loading extension module utils... + 0: + 0: + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +20: +20: Loading extension module utils...Loading extension module utils... +20: +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +20: +20: Loading extension module utils...Loading extension module utils... +20: +13: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +13: +13: Loading extension module utils...Loading extension module utils... +13: + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +30: +30: Loading extension module utils...Loading extension module utils... +30: +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 1: + 1: Loading extension module utils...Loading extension module utils... + 1: + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +14: +14: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 8: + 8: Loading extension module utils...Loading extension module utils... + 8: + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... + 7: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 7: + 7: Loading extension module utils... + 7: Loading extension module utils... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +25: +25: +25: Loading extension module utils...Loading extension module utils...Loading extension module utils... +25: +25: +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +22: +22: Loading extension module utils...Loading extension module utils... +22: + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +31: +31: Loading extension module utils...Loading extension module utils... +31: +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/utils.py:349: UserWarning: Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings + 0: warnings.warn("Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings") diff --git a/4b284b6bc4opt2/3484963.out b/4b284b6bc4opt2/3484963.out new file mode 100644 index 0000000000000000000000000000000000000000..5adeab1d295b9ac845f944d501f0c70e9c90d58c --- /dev/null +++ b/4b284b6bc4opt2/3484963.out @@ -0,0 +1,40510 @@ +Model parameters: d_model 2560 ffw_size 10240 kv_size 128 n_heads 20 n_layers 34 +Megatron-DeepSpeed/pretrain_gpt.py --tensor-model-parallel-size 2 --pipeline-model-parallel-size 1 --num-layers 34 --hidden-size 2560 --num-attention-heads 20 --kv-channels 128 --ffn-hidden-size 10240 --seq-length 2048 --max-position-embeddings 2048 --micro-batch-size 2 --global-batch-size 1024 --train-samples 1 --vocab-file gpt2/vocab.json --merge-file gpt2/merges.txt --clip-grad 1.0 --kill-switch-path kill-switch-4b284b6bc4opt2val --bf16 --optimizer adam --adam-beta1 0.9 --adam-beta2 0.999 --adam-eps 1e-8 --lr 2e-4 --min-lr 2e-5 --lr-decay-style cosine --lr-decay-samples 1 --lr-warmup-samples 0 --clip-grad 1.0 --weight-decay 1e-1 --override-lr-scheduler --reset-progress --no-load-optim --log-interval 10 --save-interval 1000 --eval-interval 1 --eval-iters 100 --eval-only true --tensorboard-dir tensorboard_4b284b6bc4opt2val --tensorboard-queue-size 5 --log-timers-to-tensorboard --log-batch-size-to-tensorboard --log-validation-ppl-to-tensorboard --save lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2 --load lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2 --train-weighted-split-paths-path train1b5.txt --valid-weighted-split-paths-path val.txt --data-impl mmap --deepspeed --deepspeed_config ds_configs/3484963.json --zero-stage 0 +START 3484963: Tue 09 May 2023 04:40:02 PM EEST + 0: + 0: + 0: ======================= ROCm System Management Interface ======================= + 0: ================================= Concise Info ================================= + 0: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 0: 0 46.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 2 48.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 4 41.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 5 51.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 6 36.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 7 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: ================================================================================ + 0: ============================= End of ROCm SMI Log ============================== +21: +21: +21: ======================= ROCm System Management Interface ======================= +21: ================================= Concise Info ================================= +21: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +21: 0 39.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 1 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 2 40.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 4 43.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 5 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 6 37.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: ================================================================================ +21: ============================= End of ROCm SMI Log ============================== + 4: + 4: + 4: ======================= ROCm System Management Interface ======================= + 4: ================================= Concise Info ================================= + 4: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 4: 0 44.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 2 44.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 4 43.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 6 40.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: ================================================================================ + 4: ============================= End of ROCm SMI Log ============================== + 3: + 3: + 3: ======================= ROCm System Management Interface ======================= + 3: ================================= Concise Info ================================= + 3: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 3: 0 42.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 2 34.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 4 43.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 5 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 6 39.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: ================================================================================ + 3: ============================= End of ROCm SMI Log ============================== +10: +10: +10: ======================= ROCm System Management Interface ======================= +10: ================================= Concise Info ================================= +10: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +10: 0 43.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 2 39.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 4 43.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 5 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 6 41.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: ================================================================================ +10: ============================= End of ROCm SMI Log ============================== + 2: + 2: + 2: ======================= ROCm System Management Interface ======================= + 2: ================================= Concise Info ================================= + 2: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 2: 0 45.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 2 37.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 4 42.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 6 37.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: ================================================================================ + 2: ============================= End of ROCm SMI Log ============================== +28: +28: +28: ======================= ROCm System Management Interface ======================= +28: ================================= Concise Info ================================= +28: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +28: 0 42.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 2 41.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 4 41.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 6 38.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: ================================================================================ +28: ============================= End of ROCm SMI Log ============================== + 9: + 9: + 9: ======================= ROCm System Management Interface ======================= + 9: ================================= Concise Info ================================= + 9: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 9: 0 46.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 2 40.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 4 40.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 6 40.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: ================================================================================ + 9: ============================= End of ROCm SMI Log ============================== +30: +30: +30: ======================= ROCm System Management Interface ======================= +30: ================================= Concise Info ================================= +30: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +30: 0 43.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 2 41.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 4 40.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 6 37.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: ================================================================================ +30: ============================= End of ROCm SMI Log ============================== +22: +22: +22: ======================= ROCm System Management Interface ======================= +22: ================================= Concise Info ================================= +22: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +22: 0 41.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 2 42.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 4 43.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 5 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 6 38.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: ================================================================================ +22: ============================= End of ROCm SMI Log ============================== +20: +20: +20: ======================= ROCm System Management Interface ======================= +20: ================================= Concise Info ================================= +20: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +20: 0 39.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 2 40.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 4 42.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 6 37.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: ================================================================================ +20: ============================= End of ROCm SMI Log ============================== +19: +19: +19: ======================= ROCm System Management Interface ======================= +19: ================================= Concise Info ================================= +19: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +19: 0 44.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 2 38.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 3 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 4 43.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 6 37.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 7 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: ================================================================================ +19: ============================= End of ROCm SMI Log ============================== +27: +27: +27: ======================= ROCm System Management Interface ======================= +27: ================================= Concise Info ================================= +27: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +27: 0 44.0c 97.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 1 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 2 35.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 4 39.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 6 35.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 7 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: ================================================================================ +27: ============================= End of ROCm SMI Log ============================== + 1: + 1: + 1: ======================= ROCm System Management Interface ======================= + 1: ================================= Concise Info ================================= + 1: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 1: 0 39.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 2 42.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 4 42.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 6 37.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: ================================================================================ + 1: ============================= End of ROCm SMI Log ============================== +16: +16: +16: ======================= ROCm System Management Interface ======================= +16: ================================= Concise Info ================================= +16: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +16: 0 45.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 2 41.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 3 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 4 43.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 6 41.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: ================================================================================ +16: ============================= End of ROCm SMI Log ============================== +15: +15: +15: ======================= ROCm System Management Interface ======================= +15: ================================= Concise Info ================================= +15: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +15: 0 43.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 2 39.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 4 44.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 5 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 6 40.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: ================================================================================ +15: ============================= End of ROCm SMI Log ============================== +25: +25: +25: ======================= ROCm System Management Interface ======================= +25: ================================= Concise Info ================================= +25: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +25: 0 42.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 2 41.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 3 38.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 4 43.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 6 41.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: ================================================================================ +25: ============================= End of ROCm SMI Log ============================== +13: +13: +13: ======================= ROCm System Management Interface ======================= +13: ================================= Concise Info ================================= +13: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +13: 0 43.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 1 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 2 38.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 4 42.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 5 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 6 41.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: ================================================================================ +13: ============================= End of ROCm SMI Log ============================== +29: +29: +29: ======================= ROCm System Management Interface ======================= +29: ================================= Concise Info ================================= +29: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +29: 0 44.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 2 41.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 3 38.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 4 44.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 6 42.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: ================================================================================ +29: ============================= End of ROCm SMI Log ============================== +14: +14: +14: ======================= ROCm System Management Interface ======================= +14: ================================= Concise Info ================================= +14: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +14: 0 44.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 2 37.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 3 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 4 43.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 6 42.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: ================================================================================ +14: ============================= End of ROCm SMI Log ============================== +31: +31: +31: ======================= ROCm System Management Interface ======================= +31: ================================= Concise Info ================================= +31: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +31: 0 44.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 2 39.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 3 38.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 4 40.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 6 41.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 7 39.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: ================================================================================ +31: ============================= End of ROCm SMI Log ============================== +12: +12: +12: ======================= ROCm System Management Interface ======================= +12: ================================= Concise Info ================================= +12: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +12: 0 47.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 2 39.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 4 43.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 6 38.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: ================================================================================ +12: ============================= End of ROCm SMI Log ============================== +23: +23: +23: ======================= ROCm System Management Interface ======================= +23: ================================= Concise Info ================================= +23: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +23: 0 48.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 2 40.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 4 40.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 6 40.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 7 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: ================================================================================ +23: ============================= End of ROCm SMI Log ============================== + 8: + 8: + 8: ======================= ROCm System Management Interface ======================= + 8: ================================= Concise Info ================================= + 8: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 8: 0 44.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 2 38.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 3 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 4 40.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 6 38.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: ================================================================================ + 8: ============================= End of ROCm SMI Log ============================== + 5: + 5: + 5: ======================= ROCm System Management Interface ======================= + 5: ================================= Concise Info ================================= + 5: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 5: 0 44.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 2 40.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 4 37.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 5 39.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 6 40.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 7 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: ================================================================================ + 5: ============================= End of ROCm SMI Log ============================== +18: +18: +18: ======================= ROCm System Management Interface ======================= +18: ================================= Concise Info ================================= +18: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +18: 0 47.0c 97.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 2 40.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 3 38.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 4 42.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 5 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 6 42.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: ================================================================================ +18: ============================= End of ROCm SMI Log ============================== +24: +24: +24: ======================= ROCm System Management Interface ======================= +24: ================================= Concise Info ================================= +24: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +24: 0 43.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 2 40.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 4 41.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 6 37.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: ================================================================================ +24: ============================= End of ROCm SMI Log ============================== +11: +11: +11: ======================= ROCm System Management Interface ======================= +11: ================================= Concise Info ================================= +11: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +11: 0 46.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 2 42.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 4 43.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 5 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 6 39.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: ================================================================================ +11: ============================= End of ROCm SMI Log ============================== +26: +26: +26: ======================= ROCm System Management Interface ======================= +26: ================================= Concise Info ================================= +26: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +26: 0 45.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 1 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 2 42.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 4 40.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 6 39.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: ================================================================================ +26: ============================= End of ROCm SMI Log ============================== +17: +17: +17: ======================= ROCm System Management Interface ======================= +17: ================================= Concise Info ================================= +17: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +17: 0 48.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 1 51.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 2 39.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 4 39.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 6 43.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: ================================================================================ +17: ============================= End of ROCm SMI Log ============================== + 6: + 6: + 6: ======================= ROCm System Management Interface ======================= + 6: ================================= Concise Info ================================= + 6: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 6: 0 38.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 2 38.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 4 39.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 5 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 6 41.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: ================================================================================ + 6: ============================= End of ROCm SMI Log ============================== + 7: + 7: + 7: ======================= ROCm System Management Interface ======================= + 7: ================================= Concise Info ================================= + 7: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 7: 0 36.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 2 39.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 4 42.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 5 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 6 38.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 7 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: ================================================================================ + 7: ============================= End of ROCm SMI Log ============================== +29: Launching on nid005243 (29/32), master nid005047 port 9999, GPUs 8, CUDA: True +28: Launching on nid005242 (28/32), master nid005047 port 9999, GPUs 8, CUDA: True +27: Launching on nid005241 (27/32), master nid005047 port 9999, GPUs 8, CUDA: True +20: Launching on nid005067 (20/32), master nid005047 port 9999, GPUs 8, CUDA: True +11: Launching on nid005058 (11/32), master nid005047 port 9999, GPUs 8, CUDA: True +31: Launching on nid005245 (31/32), master nid005047 port 9999, GPUs 8, CUDA: True +21: Launching on nid005068 (21/32), master nid005047 port 9999, GPUs 8, CUDA: True +22: Launching on nid005069 (22/32), master nid005047 port 9999, GPUs 8, CUDA: True + 3: Launching on nid005050 (3/32), master nid005047 port 9999, GPUs 8, CUDA: True +10: Launching on nid005057 (10/32), master nid005047 port 9999, GPUs 8, CUDA: True +16: Launching on nid005063 (16/32), master nid005047 port 9999, GPUs 8, CUDA: True + 4: Launching on nid005051 (4/32), master nid005047 port 9999, GPUs 8, CUDA: True +30: Launching on nid005244 (30/32), master nid005047 port 9999, GPUs 8, CUDA: True + 9: Launching on nid005056 (9/32), master nid005047 port 9999, GPUs 8, CUDA: True +13: Launching on nid005060 (13/32), master nid005047 port 9999, GPUs 8, CUDA: True + 0: Launching on nid005047 (0/32), master nid005047 port 9999, GPUs 8, CUDA: True + 2: Launching on nid005049 (2/32), master nid005047 port 9999, GPUs 8, CUDA: True +19: Launching on nid005066 (19/32), master nid005047 port 9999, GPUs 8, CUDA: True +14: Launching on nid005061 (14/32), master nid005047 port 9999, GPUs 8, CUDA: True +15: Launching on nid005062 (15/32), master nid005047 port 9999, GPUs 8, CUDA: True + 1: Launching on nid005048 (1/32), master nid005047 port 9999, GPUs 8, CUDA: True +12: Launching on nid005059 (12/32), master nid005047 port 9999, GPUs 8, CUDA: True +25: Launching on nid005072 (25/32), master nid005047 port 9999, GPUs 8, CUDA: True + 5: Launching on nid005052 (5/32), master nid005047 port 9999, GPUs 8, CUDA: True + 8: Launching on nid005055 (8/32), master nid005047 port 9999, GPUs 8, CUDA: True +26: Launching on nid005240 (26/32), master nid005047 port 9999, GPUs 8, CUDA: True +18: Launching on nid005065 (18/32), master nid005047 port 9999, GPUs 8, CUDA: True + 7: Launching on nid005054 (7/32), master nid005047 port 9999, GPUs 8, CUDA: True +17: Launching on nid005064 (17/32), master nid005047 port 9999, GPUs 8, CUDA: True +23: Launching on nid005070 (23/32), master nid005047 port 9999, GPUs 8, CUDA: True +24: Launching on nid005071 (24/32), master nid005047 port 9999, GPUs 8, CUDA: True + 6: Launching on nid005053 (6/32), master nid005047 port 9999, GPUs 8, CUDA: True + 0: using world size: 256, data-parallel-size: 128, tensor-model-parallel size: 2, pipeline-model-parallel size: 1 + 0: accumulate and all-reduce gradients in fp32 for bfloat16 data type. + 0: using torch.bfloat16 for parameters ... + 0: ------------------------ arguments ------------------------ + 0: abort_on_unmet_fused_kernel_constraints ......... False + 0: accumulate_allreduce_grads_in_fp32 .............. True + 0: adam_beta1 ...................................... 0.9 + 0: adam_beta2 ...................................... 0.999 + 0: adam_eps ........................................ 1e-08 + 0: adlr_autoresume ................................. False + 0: adlr_autoresume_interval ........................ 1000 + 0: apply_query_key_layer_scaling ................... True + 0: apply_residual_connection_post_layernorm ........ False + 0: attention_dropout ............................... 0.1 + 0: attention_softmax_in_fp32 ....................... False + 0: bert_binary_head ................................ True + 0: bert_load ....................................... None + 0: bf16 ............................................ True + 0: bias_dropout_fusion ............................. True + 0: bias_gelu_fusion ................................ True + 0: biencoder_projection_dim ........................ 0 + 0: biencoder_shared_query_context_model ............ False + 0: block_data_path ................................. None + 0: checkpoint_activations .......................... False + 0: checkpoint_in_cpu ............................... False + 0: checkpoint_num_layers ........................... 1 + 0: clip_grad ....................................... 1.0 + 0: codecarbon_dir .................................. None + 0: consumed_train_samples .......................... 0 + 0: consumed_train_tokens ........................... 0 + 0: consumed_valid_samples .......................... 0 + 0: contigious_checkpointing ........................ False + 0: cpu_optimizer ................................... False + 0: cpu_torch_adam .................................. False + 0: curriculum_learning ............................. False + 0: data_impl ....................................... mmap + 0: data_parallel_size .............................. 128 + 0: data_path ....................................... None + 0: dataloader_type ................................. single + 0: DDP_impl ........................................ local + 0: decoder_seq_length .............................. None + 0: deepscale ....................................... False + 0: deepscale_config ................................ None + 0: deepspeed ....................................... True + 0: deepspeed_activation_checkpointing .............. False + 0: deepspeed_config ................................ ds_configs/3484963.json + 0: deepspeed_mpi ................................... False + 0: distribute_checkpointed_activations ............. False + 0: distributed_backend ............................. nccl + 0: embed_layernorm ................................. False + 0: embedding_path .................................. None + 0: encoder_seq_length .............................. 2048 + 0: eod_mask_loss ................................... False + 0: eval_interval ................................... 1 + 0: eval_iters ...................................... 100 + 0: eval_only ....................................... True + 0: evidence_data_path .............................. None + 0: exit_duration_in_mins ........................... None + 0: exit_interval ................................... None + 0: ffn_hidden_size ................................. 10240 + 0: finetune ........................................ False + 0: fp16 ............................................ False + 0: fp16_lm_cross_entropy ........................... False + 0: fp32_residual_connection ........................ False + 0: gigaflos_no_embeds .............................. 0 + 0: global_batch_size ............................... 1024 + 0: glu_activation .................................. None + 0: hidden_dropout .................................. 0.1 + 0: hidden_size ..................................... 2560 + 0: hysteresis ...................................... 2 + 0: ict_head_size ................................... None + 0: ict_load ........................................ None + 0: img_dim ......................................... 224 + 0: indexer_batch_size .............................. 128 + 0: indexer_log_interval ............................ 1000 + 0: inference ....................................... False + 0: init_method_std ................................. 0.02 + 0: init_method_xavier_uniform ...................... False + 0: initial_loss_scale .............................. 4294967296 + 0: kill_switch_path ................................ kill-switch-4b284b6bc4opt2val + 0: kv_channels ..................................... 128 + 0: layer_norm_fusion ............................... True + 0: layernorm_epsilon ............................... 1e-05 + 0: lazy_mpu_init ................................... None + 0: load ............................................ lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2 + 0: local_rank ...................................... None + 0: log_batch_size_to_tensorboard ................... True + 0: log_interval .................................... 10 + 0: log_learning_rate_to_tensorboard ................ True + 0: log_level ....................................... None + 0: log_level_replica ............................... None + 0: log_loss_scale_to_tensorboard ................... True + 0: log_num_zeros_in_grad ........................... False + 0: log_params_norm ................................. False + 0: log_path ........................................ None + 0: log_timers_to_tensorboard ....................... True + 0: log_validation_ppl_to_tensorboard ............... True + 0: loss_on_targets_only ............................ False + 0: loss_scale ...................................... None + 0: loss_scale_window ............................... 1000 + 0: lr .............................................. 0.0002 + 0: lr_decay_iters .................................. None + 0: lr_decay_samples ................................ 1 + 0: lr_decay_style .................................. cosine + 0: lr_decay_tokens ................................. None + 0: lr_warmup_fraction .............................. None + 0: lr_warmup_iters ................................. 0 + 0: lr_warmup_samples ............................... 0 + 0: make_vocab_size_divisible_by .................... 128 + 0: mask_prob ....................................... 0.15 + 0: masked_softmax_fusion ........................... True + 0: max_position_embeddings ......................... 2048 + 0: mean_noise_span_length .......................... None + 0: memory_centric_tiled_linear ..................... False + 0: merge_file ...................................... gpt2/merges.txt + 0: micro_batch_size ................................ 2 + 0: min_loss_scale .................................. 1.0 + 0: min_lr .......................................... 2e-05 + 0: mmap_warmup ..................................... False + 0: no_load_optim ................................... True + 0: no_load_rng ..................................... None + 0: no_save_optim ................................... None + 0: no_save_rng ..................................... None + 0: noise_density ................................... None + 0: num_attention_heads ............................. 20 + 0: num_channels .................................... 3 + 0: num_classes ..................................... 1000 + 0: num_layers ...................................... 34 + 0: num_layers_per_virtual_pipeline_stage ........... None + 0: num_workers ..................................... 2 + 0: onnx_safe ....................................... None + 0: openai_gelu ..................................... False + 0: optimizer ....................................... adam + 0: optimizer_fusion ................................ True + 0: override_lr_scheduler ........................... True + 0: pad_vocab_size_to ............................... None + 0: params_dtype .................................... torch.bfloat16 + 0: partition_activations ........................... False + 0: patch_dim ....................................... 16 + 0: pipeline_model_parallel_size .................... 1 + 0: position_embedding_type ......................... PositionEmbeddingType.absolute + 0: pp_partition_method ............................. None + 0: profile_backward ................................ False + 0: query_in_block_prob ............................. 0.1 + 0: rampup_batch_size ............................... None + 0: rank ............................................ 0 + 0: remote_device ................................... none + 0: reset_attention_mask ............................ False + 0: reset_position_ids .............................. False + 0: reset_progress .................................. True + 0: retriever_report_topk_accuracies ................ [] + 0: retriever_score_scaling ......................... False + 0: retriever_seq_length ............................ 256 + 0: reweight_loss_based_on_position_frequency ....... False + 0: sample_rate ..................................... 1.0 + 0: save ............................................ lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2 + 0: save_interval ................................... 1000 + 0: scatter_gather_tensors_in_pipeline .............. True + 0: scattered_embeddings ............................ False + 0: seed ............................................ 1234 + 0: seq_length ...................................... 2048 + 0: sgd_momentum .................................... 0.9 + 0: short_seq_prob .................................. 0.1 + 0: skip_train_iteration_range ...................... None + 0: split ........................................... None + 0: split_transformers .............................. False + 0: sync_tp_duplicated_parameters ................... False + 0: synchronize_each_layer .......................... False + 0: tensor_model_parallel_size ...................... 2 + 0: tensorboard_dir ................................. tensorboard_4b284b6bc4opt2val + 0: tensorboard_log_interval ........................ 1 + 0: tensorboard_queue_size .......................... 5 + 0: test_weighted_split_paths ....................... None + 0: test_weighted_split_paths_path .................. None + 0: tile_factor ..................................... 1 + 0: titles_data_path ................................ None + 0: tokenizer_name_or_path .......................... None + 0: tokenizer_type .................................. GPT2BPETokenizer + 0: train_iters ..................................... None + 0: train_samples ................................... 1 + 0: train_tokens .................................... None + 0: train_weighted_split_names ...................... ['train'] + 0: train_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document']] + 0: train_weighted_split_paths_path ................. None + 0: train_weighted_split_splits ..................... [['0:1']] + 0: train_weighted_split_weights .................... [['1.0']] + 0: universal_checkpoint ............................ False + 0: use_bnb_optimizer ............................... False + 0: use_checkpoint_lr_scheduler ..................... False + 0: use_contiguous_buffers_in_ddp ................... True + 0: use_cpu_initialization .......................... None + 0: use_one_sent_docs ............................... False + 0: use_pin_memory .................................. False + 0: valid_num_workers ............................... 2 + 0: valid_weighted_split_names ...................... ['validation'] + 0: valid_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document']] + 0: valid_weighted_split_paths_path ................. None + 0: valid_weighted_split_splits ..................... [['0:1']] + 0: valid_weighted_split_weights .................... [['1.0']] + 0: virtual_pipeline_model_parallel_size ............ None + 0: vocab_extra_ids ................................. 0 + 0: vocab_file ...................................... gpt2/vocab.json + 0: weight_decay .................................... 0.1 + 0: world_size ...................................... 256 + 0: zero_allgather_bucket_size ...................... 0.0 + 0: zero_contigious_gradients ....................... False + 0: zero_reduce_bucket_size ......................... 0.0 + 0: zero_reduce_scatter ............................. False + 0: zero_stage ...................................... 0 + 0: -------------------- end of arguments --------------------- + 0: setting number of micro-batches to constant 4 + 0: > building GPT2BPETokenizer tokenizer ... + 0: > padded vocab (size: 50257) with 175 dummy tokens (new size: 50432) + 0: DeepSpeed general environment info: + 0: torch install path ............... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch'] + 0: torch version .................... 1.13.0+rocm5.2 + 0: torch cuda version ............... None + 0: torch hip version ................ 5.2.21151-afdc89f8 + 0: nvcc version ..................... None + 0: deepspeed install path ........... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/deepspeed'] + 0: deepspeed info ................... 0.7.5, unknown, unknown + 0: deepspeed wheel compiled w. ...... torch 1.13, hip 5.1 + 0: **** Git info for Megatron: git_hash=unknown git_branch=unknown **** + 0: > initializing torch distributed ... + 0: [2023-05-09 16:43:25,203] [INFO] [comm.py:633:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +31: > setting tensorboard ... + 0: > initializing tensor model parallel with size 2 + 0: > initializing pipeline model parallel with size 1 + 0: > setting random seeds to 1234 ... + 0: > initializing model parallel cuda seeds on global rank 0, model parallel rank 0, and data parallel rank 0 with model parallel seed: 3952 and data parallel seed: 1234 + 0: > compiling dataset index builder ... + 0: make: Entering directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' + 0: make: Nothing to be done for 'default'. + 0: make: Leaving directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' + 0: >>> done with dataset index builder. Compilation time: 0.110 seconds + 0: > compiling and loading fused kernels ... + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.cpp [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.hip [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] + 0: Total number of unsupported CUDA function calls: 0 + 0: + 0: + 0: Total number of replaced kernel launches: 87 + 0: [1/1] c++ scaled_upper_triang_masked_softmax_hip.o scaled_upper_triang_masked_softmax_hip.cuda.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/opt/rocm/lib -lamdhip64 -o scaled_upper_triang_masked_softmax_cuda.so + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.cpp [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.hip [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] + 0: Total number of unsupported CUDA function calls: 0 + 0: + 0: + 0: Total number of replaced kernel launches: 63 + 0: ninja: no work to do. + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda_kernel.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_hip_kernel.hip [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] + 0: Total number of unsupported CUDA function calls: 0 + 0: + 0: + 0: Total number of replaced kernel launches: 67 + 0: [1/1] c++ layer_norm_cuda.o layer_norm_hip_kernel.cuda.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/opt/rocm/lib -lamdhip64 -o fused_mix_prec_layer_norm_cuda.so + 0: >>> done with compiling and loading fused kernels. Compilation time: 21.604 seconds + 0: time to initialize megatron (seconds): 27.466 + 0: [after megatron is initialized] datetime: 2023-05-09 16:43:55 + 0: building GPT model ... + 0: [2023-05-09 16:43:55,607] [INFO] [utils.py:827:see_memory_usage] Before Building Model + 0: [2023-05-09 16:43:55,607] [INFO] [utils.py:828:see_memory_usage] MA 0.0 GB Max_MA 0.0 GB CA 0.0 GB Max_CA 0 GB + 0: [2023-05-09 16:43:55,608] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 38.03 GB, percent = 7.6% + 0: SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None + 0: Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=0, model=1): 1, ProcessCoord(pipe=0, data=1, model=0): 2, ProcessCoord(pipe=0, data=1, model=1): 3, ProcessCoord(pipe=0, data=2, model=0): 4, ProcessCoord(pipe=0, data=2, model=1): 5, ProcessCoord(pipe=0, data=3, model=0): 6, ProcessCoord(pipe=0, data=3, model=1): 7, ProcessCoord(pipe=0, data=4, model=0): 8, ProcessCoord(pipe=0, data=4, model=1): 9, ProcessCoord(pipe=0, data=5, model=0): 10, ProcessCoord(pipe=0, data=5, model=1): 11, ProcessCoord(pipe=0, data=6, model=0): 12, ProcessCoord(pipe=0, data=6, model=1): 13, ProcessCoord(pipe=0, data=7, model=0): 14, ProcessCoord(pipe=0, data=7, model=1): 15, ProcessCoord(pipe=0, data=8, model=0): 16, ProcessCoord(pipe=0, data=8, model=1): 17, ProcessCoord(pipe=0, data=9, model=0): 18, ProcessCoord(pipe=0, data=9, model=1): 19, ProcessCoord(pipe=0, data=10, model=0): 20, ProcessCoord(pipe=0, data=10, model=1): 21, ProcessCoord(pipe=0, data=11, model=0): 22, ProcessCoord(pipe=0, data + 0: =11, model=1): 23, ProcessCoord(pipe=0, data=12, model=0): 24, ProcessCoord(pipe=0, data=12, model=1): 25, ProcessCoord(pipe=0, data=13, model=0): 26, ProcessCoord(pipe=0, data=13, model=1): 27, ProcessCoord(pipe=0, data=14, model=0): 28, ProcessCoord(pipe=0, data=14, model=1): 29, ProcessCoord(pipe=0, data=15, model=0): 30, ProcessCoord(pipe=0, data=15, model=1): 31, ProcessCoord(pipe=0, data=16, model=0): 32, ProcessCoord(pipe=0, data=16, model=1): 33, ProcessCoord(pipe=0, data=17, model=0): 34, ProcessCoord(pipe=0, data=17, model=1): 35, ProcessCoord(pipe=0, data=18, model=0): 36, ProcessCoord(pipe=0, data=18, model=1): 37, ProcessCoord(pipe=0, data=19, model=0): 38, ProcessCoord(pipe=0, data=19, model=1): 39, ProcessCoord(pipe=0, data=20, model=0): 40, ProcessCoord(pipe=0, data=20, model=1): 41, ProcessCoord(pipe=0, data=21, model=0): 42, ProcessCoord(pipe=0, data=21, model=1): 43, ProcessCoord(pipe=0, data=22, model=0): 44, ProcessCoord(pipe=0, data=22, model=1): 45, ProcessCoord(pipe=0, data=23, model=0 + 0: ): 46, ProcessCoord(pipe=0, data=23, model=1): 47, ProcessCoord(pipe=0, data=24, model=0): 48, ProcessCoord(pipe=0, data=24, model=1): 49, ProcessCoord(pipe=0, data=25, model=0): 50, ProcessCoord(pipe=0, data=25, model=1): 51, ProcessCoord(pipe=0, data=26, model=0): 52, ProcessCoord(pipe=0, data=26, model=1): 53, ProcessCoord(pipe=0, data=27, model=0): 54, ProcessCoord(pipe=0, data=27, model=1): 55, ProcessCoord(pipe=0, data=28, model=0): 56, ProcessCoord(pipe=0, data=28, model=1): 57, ProcessCoord(pipe=0, data=29, model=0): 58, ProcessCoord(pipe=0, data=29, model=1): 59, ProcessCoord(pipe=0, data=30, model=0): 60, ProcessCoord(pipe=0, data=30, model=1): 61, ProcessCoord(pipe=0, data=31, model=0): 62, ProcessCoord(pipe=0, data=31, model=1): 63, ProcessCoord(pipe=0, data=32, model=0): 64, ProcessCoord(pipe=0, data=32, model=1): 65, ProcessCoord(pipe=0, data=33, model=0): 66, ProcessCoord(pipe=0, data=33, model=1): 67, ProcessCoord(pipe=0, data=34, model=0): 68, ProcessCoord(pipe=0, data=34, model=1): 69, Proce + 0: ssCoord(pipe=0, data=35, model=0): 70, ProcessCoord(pipe=0, data=35, model=1): 71, ProcessCoord(pipe=0, data=36, model=0): 72, ProcessCoord(pipe=0, data=36, model=1): 73, ProcessCoord(pipe=0, data=37, model=0): 74, ProcessCoord(pipe=0, data=37, model=1): 75, ProcessCoord(pipe=0, data=38, model=0): 76, ProcessCoord(pipe=0, data=38, model=1): 77, ProcessCoord(pipe=0, data=39, model=0): 78, ProcessCoord(pipe=0, data=39, model=1): 79, ProcessCoord(pipe=0, data=40, model=0): 80, ProcessCoord(pipe=0, data=40, model=1): 81, ProcessCoord(pipe=0, data=41, model=0): 82, ProcessCoord(pipe=0, data=41, model=1): 83, ProcessCoord(pipe=0, data=42, model=0): 84, ProcessCoord(pipe=0, data=42, model=1): 85, ProcessCoord(pipe=0, data=43, model=0): 86, ProcessCoord(pipe=0, data=43, model=1): 87, ProcessCoord(pipe=0, data=44, model=0): 88, ProcessCoord(pipe=0, data=44, model=1): 89, ProcessCoord(pipe=0, data=45, model=0): 90, ProcessCoord(pipe=0, data=45, model=1): 91, ProcessCoord(pipe=0, data=46, model=0): 92, ProcessCoord(pipe + 0: =0, data=46, model=1): 93, ProcessCoord(pipe=0, data=47, model=0): 94, ProcessCoord(pipe=0, data=47, model=1): 95, ProcessCoord(pipe=0, data=48, model=0): 96, ProcessCoord(pipe=0, data=48, model=1): 97, ProcessCoord(pipe=0, data=49, model=0): 98, ProcessCoord(pipe=0, data=49, model=1): 99, ProcessCoord(pipe=0, data=50, model=0): 100, ProcessCoord(pipe=0, data=50, model=1): 101, ProcessCoord(pipe=0, data=51, model=0): 102, ProcessCoord(pipe=0, data=51, model=1): 103, ProcessCoord(pipe=0, data=52, model=0): 104, ProcessCoord(pipe=0, data=52, model=1): 105, ProcessCoord(pipe=0, data=53, model=0): 106, ProcessCoord(pipe=0, data=53, model=1): 107, ProcessCoord(pipe=0, data=54, model=0): 108, ProcessCoord(pipe=0, data=54, model=1): 109, ProcessCoord(pipe=0, data=55, model=0): 110, ProcessCoord(pipe=0, data=55, model=1): 111, ProcessCoord(pipe=0, data=56, model=0): 112, ProcessCoord(pipe=0, data=56, model=1): 113, ProcessCoord(pipe=0, data=57, model=0): 114, ProcessCoord(pipe=0, data=57, model=1): 115, ProcessCoord( + 0: pipe=0, data=58, model=0): 116, ProcessCoord(pipe=0, data=58, model=1): 117, ProcessCoord(pipe=0, data=59, model=0): 118, ProcessCoord(pipe=0, data=59, model=1): 119, ProcessCoord(pipe=0, data=60, model=0): 120, ProcessCoord(pipe=0, data=60, model=1): 121, ProcessCoord(pipe=0, data=61, model=0): 122, ProcessCoord(pipe=0, data=61, model=1): 123, ProcessCoord(pipe=0, data=62, model=0): 124, ProcessCoord(pipe=0, data=62, model=1): 125, ProcessCoord(pipe=0, data=63, model=0): 126, ProcessCoord(pipe=0, data=63, model=1): 127, ProcessCoord(pipe=0, data=64, model=0): 128, ProcessCoord(pipe=0, data=64, model=1): 129, ProcessCoord(pipe=0, data=65, model=0): 130, ProcessCoord(pipe=0, data=65, model=1): 131, ProcessCoord(pipe=0, data=66, model=0): 132, ProcessCoord(pipe=0, data=66, model=1): 133, ProcessCoord(pipe=0, data=67, model=0): 134, ProcessCoord(pipe=0, data=67, model=1): 135, ProcessCoord(pipe=0, data=68, model=0): 136, ProcessCoord(pipe=0, data=68, model=1): 137, ProcessCoord(pipe=0, data=69, model=0): 138, Pr + 0: ocessCoord(pipe=0, data=69, model=1): 139, ProcessCoord(pipe=0, data=70, model=0): 140, ProcessCoord(pipe=0, data=70, model=1): 141, ProcessCoord(pipe=0, data=71, model=0): 142, ProcessCoord(pipe=0, data=71, model=1): 143, ProcessCoord(pipe=0, data=72, model=0): 144, ProcessCoord(pipe=0, data=72, model=1): 145, ProcessCoord(pipe=0, data=73, model=0): 146, ProcessCoord(pipe=0, data=73, model=1): 147, ProcessCoord(pipe=0, data=74, model=0): 148, ProcessCoord(pipe=0, data=74, model=1): 149, ProcessCoord(pipe=0, data=75, model=0): 150, ProcessCoord(pipe=0, data=75, model=1): 151, ProcessCoord(pipe=0, data=76, model=0): 152, ProcessCoord(pipe=0, data=76, model=1): 153, ProcessCoord(pipe=0, data=77, model=0): 154, ProcessCoord(pipe=0, data=77, model=1): 155, ProcessCoord(pipe=0, data=78, model=0): 156, ProcessCoord(pipe=0, data=78, model=1): 157, ProcessCoord(pipe=0, data=79, model=0): 158, ProcessCoord(pipe=0, data=79, model=1): 159, ProcessCoord(pipe=0, data=80, model=0): 160, ProcessCoord(pipe=0, data=80, model= + 0: 1): 161, ProcessCoord(pipe=0, data=81, model=0): 162, ProcessCoord(pipe=0, data=81, model=1): 163, ProcessCoord(pipe=0, data=82, model=0): 164, ProcessCoord(pipe=0, data=82, model=1): 165, ProcessCoord(pipe=0, data=83, model=0): 166, ProcessCoord(pipe=0, data=83, model=1): 167, ProcessCoord(pipe=0, data=84, model=0): 168, ProcessCoord(pipe=0, data=84, model=1): 169, ProcessCoord(pipe=0, data=85, model=0): 170, ProcessCoord(pipe=0, data=85, model=1): 171, ProcessCoord(pipe=0, data=86, model=0): 172, ProcessCoord(pipe=0, data=86, model=1): 173, ProcessCoord(pipe=0, data=87, model=0): 174, ProcessCoord(pipe=0, data=87, model=1): 175, ProcessCoord(pipe=0, data=88, model=0): 176, ProcessCoord(pipe=0, data=88, model=1): 177, ProcessCoord(pipe=0, data=89, model=0): 178, ProcessCoord(pipe=0, data=89, model=1): 179, ProcessCoord(pipe=0, data=90, model=0): 180, ProcessCoord(pipe=0, data=90, model=1): 181, ProcessCoord(pipe=0, data=91, model=0): 182, ProcessCoord(pipe=0, data=91, model=1): 183, ProcessCoord(pipe=0, data + 0: =92, model=0): 184, ProcessCoord(pipe=0, data=92, model=1): 185, ProcessCoord(pipe=0, data=93, model=0): 186, ProcessCoord(pipe=0, data=93, model=1): 187, ProcessCoord(pipe=0, data=94, model=0): 188, ProcessCoord(pipe=0, data=94, model=1): 189, ProcessCoord(pipe=0, data=95, model=0): 190, ProcessCoord(pipe=0, data=95, model=1): 191, ProcessCoord(pipe=0, data=96, model=0): 192, ProcessCoord(pipe=0, data=96, model=1): 193, ProcessCoord(pipe=0, data=97, model=0): 194, ProcessCoord(pipe=0, data=97, model=1): 195, ProcessCoord(pipe=0, data=98, model=0): 196, ProcessCoord(pipe=0, data=98, model=1): 197, ProcessCoord(pipe=0, data=99, model=0): 198, ProcessCoord(pipe=0, data=99, model=1): 199, ProcessCoord(pipe=0, data=100, model=0): 200, ProcessCoord(pipe=0, data=100, model=1): 201, ProcessCoord(pipe=0, data=101, model=0): 202, ProcessCoord(pipe=0, data=101, model=1): 203, ProcessCoord(pipe=0, data=102, model=0): 204, ProcessCoord(pipe=0, data=102, model=1): 205, ProcessCoord(pipe=0, data=103, model=0): 206, Process + 0: Coord(pipe=0, data=103, model=1): 207, ProcessCoord(pipe=0, data=104, model=0): 208, ProcessCoord(pipe=0, data=104, model=1): 209, ProcessCoord(pipe=0, data=105, model=0): 210, ProcessCoord(pipe=0, data=105, model=1): 211, ProcessCoord(pipe=0, data=106, model=0): 212, ProcessCoord(pipe=0, data=106, model=1): 213, ProcessCoord(pipe=0, data=107, model=0): 214, ProcessCoord(pipe=0, data=107, model=1): 215, ProcessCoord(pipe=0, data=108, model=0): 216, ProcessCoord(pipe=0, data=108, model=1): 217, ProcessCoord(pipe=0, data=109, model=0): 218, ProcessCoord(pipe=0, data=109, model=1): 219, ProcessCoord(pipe=0, data=110, model=0): 220, ProcessCoord(pipe=0, data=110, model=1): 221, ProcessCoord(pipe=0, data=111, model=0): 222, ProcessCoord(pipe=0, data=111, model=1): 223, ProcessCoord(pipe=0, data=112, model=0): 224, ProcessCoord(pipe=0, data=112, model=1): 225, ProcessCoord(pipe=0, data=113, model=0): 226, ProcessCoord(pipe=0, data=113, model=1): 227, ProcessCoord(pipe=0, data=114, model=0): 228, ProcessCoord(pipe=0 + 0: , data=114, model=1): 229, ProcessCoord(pipe=0, data=115, model=0): 230, ProcessCoord(pipe=0, data=115, model=1): 231, ProcessCoord(pipe=0, data=116, model=0): 232, ProcessCoord(pipe=0, data=116, model=1): 233, ProcessCoord(pipe=0, data=117, model=0): 234, ProcessCoord(pipe=0, data=117, model=1): 235, ProcessCoord(pipe=0, data=118, model=0): 236, ProcessCoord(pipe=0, data=118, model=1): 237, ProcessCoord(pipe=0, data=119, model=0): 238, ProcessCoord(pipe=0, data=119, model=1): 239, ProcessCoord(pipe=0, data=120, model=0): 240, ProcessCoord(pipe=0, data=120, model=1): 241, ProcessCoord(pipe=0, data=121, model=0): 242, ProcessCoord(pipe=0, data=121, model=1): 243, ProcessCoord(pipe=0, data=122, model=0): 244, ProcessCoord(pipe=0, data=122, model=1): 245, ProcessCoord(pipe=0, data=123, model=0): 246, ProcessCoord(pipe=0, data=123, model=1): 247, ProcessCoord(pipe=0, data=124, model=0): 248, ProcessCoord(pipe=0, data=124, model=1): 249, ProcessCoord(pipe=0, data=125, model=0): 250, ProcessCoord(pipe=0, data=125, + 0: model=1): 251, ProcessCoord(pipe=0, data=126, model=0): 252, ProcessCoord(pipe=0, data=126, model=1): 253, ProcessCoord(pipe=0, data=127, model=0): 254, ProcessCoord(pipe=0, data=127, model=1): 255} + 0: [2023-05-09 16:44:01,105] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer + 0: stage=0 layers=41 + 0: 0: _to_float16 + 0: 1: EmbeddingPipe + 0: 2: + 0: 3: ParallelTransformerLayerPipe + 0: 4: ParallelTransformerLayerPipe + 0: 5: ParallelTransformerLayerPipe + 0: 6: ParallelTransformerLayerPipe + 0: 7: ParallelTransformerLayerPipe + 0: 8: ParallelTransformerLayerPipe + 0: 9: ParallelTransformerLayerPipe + 0: 10: ParallelTransformerLayerPipe + 0: 11: ParallelTransformerLayerPipe + 0: 12: ParallelTransformerLayerPipe + 0: 13: ParallelTransformerLayerPipe + 0: 14: ParallelTransformerLayerPipe + 0: 15: ParallelTransformerLayerPipe + 0: 16: ParallelTransformerLayerPipe + 0: 17: ParallelTransformerLayerPipe + 0: 18: ParallelTransformerLayerPipe + 0: 19: ParallelTransformerLayerPipe + 0: 20: ParallelTransformerLayerPipe + 0: 21: ParallelTransformerLayerPipe + 0: 22: ParallelTransformerLayerPipe + 0: 23: ParallelTransformerLayerPipe + 0: 24: ParallelTransformerLayerPipe + 0: 25: ParallelTransformerLayerPipe + 0: 26: ParallelTransformerLayerPipe + 0: 27: ParallelTransformerLayerPipe + 0: 28: ParallelTransformerLayerPipe + 0: 29: ParallelTransformerLayerPipe + 0: 30: ParallelTransformerLayerPipe + 0: 31: ParallelTransformerLayerPipe + 0: 32: ParallelTransformerLayerPipe + 0: 33: ParallelTransformerLayerPipe + 0: 34: ParallelTransformerLayerPipe + 0: 35: ParallelTransformerLayerPipe + 0: 36: ParallelTransformerLayerPipe + 0: 37: undo + 0: 38: MixedFusedLayerNorm + 0: 39: EmbeddingPipe + 0: 40: float16_to_fp32 + 0: loss: CrossEntropy + 0: [2023-05-09 16:44:01,749] [INFO] [utils.py:827:see_memory_usage] After Building Model + 0: [2023-05-09 16:44:01,749] [INFO] [utils.py:828:see_memory_usage] MA 2.7 GB Max_MA 2.7 GB CA 2.77 GB Max_CA 3 GB + 0: [2023-05-09 16:44:01,750] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 38.23 GB, percent = 7.6% + 0: setting training iterations to 0 + 0: > learning rate decay style: cosine + 0: DeepSpeed is enabled. + 0: [2023-05-09 16:44:01,752] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.5, git-hash=unknown, git-branch=unknown + 0: [2023-05-09 16:44:07,302] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False + 0: [2023-05-09 16:44:07,302] [INFO] [logging.py:68:log_dist] [Rank 0] Removing param_group that has no 'params' in the client Optimizer + 0: [2023-05-09 16:44:07,302] [INFO] [logging.py:68:log_dist] [Rank 0] Using client Optimizer as basic optimizer + 0: [2023-05-09 16:44:07,322] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam + 0: [2023-05-09 16:44:07,322] [INFO] [logging.py:68:log_dist] [Rank 0] Creating BF16 optimizer + 0: [2023-05-09 16:44:07,446] [INFO] [utils.py:827:see_memory_usage] begin bf16_optimizer + 0: [2023-05-09 16:44:07,446] [INFO] [utils.py:828:see_memory_usage] MA 2.69 GB Max_MA 2.71 GB CA 2.77 GB Max_CA 3 GB + 0: [2023-05-09 16:44:07,446] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.0 GB, percent = 7.7% + 0: ninja: no work to do. + 4: Time to load utils op: 0.5226893424987793 seconds + 4: Time to load utils op: 0.5227012634277344 seconds + 4: Time to load utils op: 0.5227177143096924 secondsTime to load utils op: 0.5227208137512207 seconds + 4: +23: Time to load utils op: 0.5157878398895264 seconds + 0: Time to load utils op: 0.42182016372680664 seconds + 0: Time to load utils op: 0.5292057991027832 seconds + 0: Time to load utils op: 0.5292494297027588 seconds +23: Time to load utils op: 0.5166921615600586 secondsTime to load utils op: 0.5169551372528076 seconds +23: +10: Time to load utils op: 0.5186936855316162 seconds +10: Time to load utils op: 0.5187022686004639 seconds +10: Time to load utils op: 0.5187103748321533 seconds +23: Time to load utils op: 0.5166049003601074 seconds +10: Time to load utils op: 0.5187392234802246 seconds + 0: Time to load utils op: 0.5293352603912354 seconds +25: Time to load utils op: 0.5148637294769287 seconds +25: Time to load utils op: 0.51505446434021 seconds +25: Time to load utils op: 0.5154373645782471 secondsTime to load utils op: 0.5163166522979736 seconds +25: +16: Time to load utils op: 0.5198643207550049 seconds +16: Time to load utils op: 0.5210177898406982 secondsTime to load utils op: 0.5196571350097656 seconds +16: +16: Time to load utils op: 0.5200648307800293 seconds +24: Time to load utils op: 0.5120277404785156 secondsTime to load utils op: 0.5120244026184082 secondsTime to load utils op: 0.5120279788970947 seconds +24: +24: +24: Time to load utils op: 0.5120596885681152 seconds +30: Time to load utils op: 0.5097787380218506 secondsTime to load utils op: 0.5097699165344238 seconds +30: +30: Time to load utils op: 0.5097916126251221 seconds +30: Time to load utils op: 0.5097994804382324 seconds + 2: Time to load utils op: 0.523895263671875 secondsTime to load utils op: 0.5239126682281494 seconds + 2: + 2: Time to load utils op: 0.5239205360412598 seconds + 9: Time to load utils op: 0.521263599395752 secondsTime to load utils op: 0.5212609767913818 seconds + 9: + 9: Time to load utils op: 0.5212819576263428 seconds +12: Time to load utils op: 0.5188472270965576 secondsTime to load utils op: 0.518869161605835 seconds +12: Time to load utils op: 0.5188634395599365 seconds +12: + 1: Time to load utils op: 0.5285942554473877 secondsTime to load utils op: 0.5289018154144287 seconds + 1: + 1: Time to load utils op: 0.5295724868774414 seconds + 2: Time to load utils op: 0.5239667892456055 seconds + 9: Time to load utils op: 0.5212981700897217 seconds +12: Time to load utils op: 0.5189180374145508 seconds +13: Time to load utils op: 0.5182490348815918 seconds + 1: Time to load utils op: 0.5291874408721924 seconds +13: Time to load utils op: 0.518294095993042 seconds +13: Time to load utils op: 0.5183053016662598 secondsTime to load utils op: 0.5183134078979492 seconds +13: + 3: Time to load utils op: 0.5294861793518066 seconds + 3: Time to load utils op: 0.5294907093048096 seconds + 3: Time to load utils op: 0.5295088291168213 seconds +27: Time to load utils op: 0.5201349258422852 secondsTime to load utils op: 0.5208027362823486 seconds +27: +27: Time to load utils op: 0.521155834197998 secondsTime to load utils op: 0.5205729007720947 seconds +27: + 3: Time to load utils op: 0.5295207500457764 seconds +20: Time to load utils op: 0.5206985473632812 seconds +20: Time to load utils op: 0.520707368850708 seconds +20: Time to load utils op: 0.520716667175293 seconds + 5: Time to load utils op: 0.5286071300506592 secondsTime to load utils op: 0.5286128520965576 seconds + 5: + 5: Time to load utils op: 0.5286166667938232 secondsTime to load utils op: 0.5286233425140381 seconds + 5: +29: Time to load utils op: 0.5196397304534912 secondsTime to load utils op: 0.519359827041626 seconds +29: +20: Time to load utils op: 0.5207808017730713 seconds +29: Time to load utils op: 0.5189416408538818 seconds +15: Time to load utils op: 0.5269618034362793 secondsTime to load utils op: 0.5279891490936279 seconds +15: +15: Time to load utils op: 0.5270943641662598 seconds +26: Time to load utils op: 0.5210666656494141 seconds +26: Time to load utils op: 0.5214886665344238 seconds +29: Time to load utils op: 0.5193235874176025 seconds +22: Time to load utils op: 0.518977165222168 secondsTime to load utils op: 0.5189778804779053 seconds +22: +22: Time to load utils op: 0.5189878940582275 seconds +26: Time to load utils op: 0.5213377475738525 seconds +26: Time to load utils op: 0.5212368965148926 seconds + 6: Time to load utils op: 0.5277976989746094 seconds +31: Time to load utils op: 0.5189881324768066 seconds +31: Time to load utils op: 0.5194358825683594 seconds +15: Time to load utils op: 0.5268688201904297 seconds +21: Time to load utils op: 0.5199048519134521 seconds +21: Time to load utils op: 0.5199220180511475 seconds +11: Time to load utils op: 0.5254175662994385 seconds +11: Time to load utils op: 0.5254278182983398 seconds +31: Time to load utils op: 0.5182898044586182 seconds +17: Time to load utils op: 0.5225474834442139 seconds +17: Time to load utils op: 0.5225520133972168 seconds +11: Time to load utils op: 0.5254373550415039 seconds +11: Time to load utils op: 0.5254437923431396 seconds +22: Time to load utils op: 0.5190448760986328 seconds +21: Time to load utils op: 0.5199387073516846 seconds +31: Time to load utils op: 0.5179672241210938 seconds +14: Time to load utils op: 0.5237593650817871 seconds +21: Time to load utils op: 0.5200021266937256 seconds + 6: Time to load utils op: 0.5278747081756592 secondsTime to load utils op: 0.5278644561767578 seconds + 6: + 6: Time to load utils op: 0.5279016494750977 seconds +17: Time to load utils op: 0.5225341320037842 seconds +17: Time to load utils op: 0.5226178169250488 seconds +14: Time to load utils op: 0.5238096714019775 seconds +28: Time to load utils op: 0.5189690589904785 seconds +28: Time to load utils op: 0.5189664363861084 seconds +14: Time to load utils op: 0.5238213539123535 seconds +14: Time to load utils op: 0.5238323211669922 seconds +28: Time to load utils op: 0.5191910266876221 secondsTime to load utils op: 0.5190916061401367 seconds +28: + 8: Time to load utils op: 0.531109094619751 seconds + 8: Time to load utils op: 0.5317845344543457 seconds + 7: Time to load utils op: 0.528078556060791 secondsTime to load utils op: 0.528087854385376 seconds + 7: + 7: Time to load utils op: 0.5281009674072266 secondsTime to load utils op: 0.5281116962432861 seconds + 7: + 8: Time to load utils op: 0.5318245887756348 secondsTime to load utils op: 0.5304434299468994 seconds + 8: +18: Time to load utils op: 0.5227751731872559 secondsTime to load utils op: 0.5227799415588379 seconds +18: +18: Time to load utils op: 0.522789716720581 seconds +18: Time to load utils op: 0.5228433609008789 seconds +19: Time to load utils op: 0.5256407260894775 seconds +19: Time to load utils op: 0.5244617462158203 secondsTime to load utils op: 0.5261831283569336 seconds +19: +19: Time to load utils op: 0.5262515544891357 seconds + 0: [2023-05-09 16:44:07,981] [INFO] [utils.py:827:see_memory_usage] before initializing group 0 + 0: [2023-05-09 16:44:07,982] [INFO] [utils.py:828:see_memory_usage] MA 2.69 GB Max_MA 2.69 GB CA 2.77 GB Max_CA 3 GB + 0: [2023-05-09 16:44:07,982] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.0 GB, percent = 7.8% +29: Time to load utils op: 0.0007817745208740234 seconds +29: Time to load utils op: 0.0008721351623535156 secondsTime to load utils op: 0.0006601810455322266 seconds +29: +29: Time to load utils op: 0.0007255077362060547 seconds + 9: Time to load utils op: 0.0008084774017333984 secondsTime to load utils op: 0.0008418560028076172 seconds + 9: +15: Time to load utils op: 0.0007557868957519531 seconds + 9: Time to load utils op: 0.0008995532989501953 seconds +15: Time to load utils op: 0.0008940696716308594 seconds +25: Time to load utils op: 0.0008771419525146484 secondsTime to load utils op: 0.0008802413940429688 seconds +25: Time to load utils op: 0.0009338855743408203 seconds +25: +28: Time to load utils op: 0.0009582042694091797 secondsTime to load utils op: 0.0009493827819824219 seconds +28: +28: Time to load utils op: 0.0007963180541992188 seconds +15: Time to load utils op: 0.0005223751068115234 seconds + 9: Time to load utils op: 0.0007655620574951172 seconds +25: Time to load utils op: 0.0007333755493164062 seconds +15: Time to load utils op: 0.0009753704071044922 seconds +28: Time to load utils op: 0.0008065700531005859 seconds + 7: Time to load utils op: 0.0008783340454101562 seconds + 7: Time to load utils op: 0.0009124279022216797 secondsTime to load utils op: 0.0009257793426513672 seconds + 7: + 7: Time to load utils op: 0.0009684562683105469 seconds +19: Time to load utils op: 0.0009191036224365234 seconds +19: Time to load utils op: 0.0009341239929199219 seconds +19: Time to load utils op: 0.0008792877197265625 seconds +19: Time to load utils op: 0.00093841552734375 seconds +13: Time to load utils op: 0.0010933876037597656 seconds +13: Time to load utils op: 0.0010745525360107422 seconds +13: Time to load utils op: 0.0011119842529296875 seconds +13: Time to load utils op: 0.0011456012725830078 seconds + 3: Time to load utils op: 0.0007512569427490234 seconds + 3: Time to load utils op: 0.000823974609375 secondsTime to load utils op: 0.0007472038269042969 seconds + 3: + 3: Time to load utils op: 0.0008122920989990234 seconds + 0: Time to load utils op: 0.0005202293395996094 seconds + 0: Time to load utils op: 0.0004699230194091797 seconds + 0: Time to load utils op: 0.0004975795745849609 seconds + 2: Time to load utils op: 0.0007669925689697266 seconds + 2: Time to load utils op: 0.0003638267517089844 seconds +17: Time to load utils op: 0.0007755756378173828 seconds +17: Time to load utils op: 0.0004067420959472656 secondsTime to load utils op: 0.00040793418884277344 seconds +17: + 2: Time to load utils op: 0.0006988048553466797 seconds +16: Time to load utils op: 0.0008666515350341797 seconds +26: Time to load utils op: 0.0008859634399414062 seconds +22: Time to load utils op: 0.0008950233459472656 seconds + 8: Time to load utils op: 0.001088857650756836 seconds +30: Time to load utils op: 0.0009715557098388672 seconds +17: Time to load utils op: 0.0007460117340087891 seconds +31: Time to load utils op: 0.0010983943939208984 seconds +10: Time to load utils op: 0.0007390975952148438 seconds +30: Time to load utils op: 0.001062154769897461 seconds +26: Time to load utils op: 0.0013048648834228516 seconds + 8: Time to load utils op: 0.0012731552124023438 seconds +22: Time to load utils op: 0.0010333061218261719 seconds +31: Time to load utils op: 0.0012788772583007812 seconds +27: Time to load utils op: 0.0013375282287597656 seconds +18: Time to load utils op: 0.0009639263153076172 seconds + 8: Time to load utils op: 0.001401662826538086 seconds + 1: Time to load utils op: 0.0014529228210449219 seconds +10: Time to load utils op: 0.0008499622344970703 seconds +22: Time to load utils op: 0.001394033432006836 seconds + 8: Time to load utils op: 0.0015239715576171875 seconds +18: Time to load utils op: 0.0014843940734863281 seconds + 6: Time to load utils op: 0.0013501644134521484 seconds + 1: Time to load utils op: 0.0014057159423828125 seconds + 2: Time to load utils op: 0.0013904571533203125 seconds + 5: Time to load utils op: 0.0013737678527832031 seconds +11: Time to load utils op: 0.0010864734649658203 seconds +26: Time to load utils op: 0.001239776611328125 seconds +14: Time to load utils op: 0.0010089874267578125 seconds +21: Time to load utils op: 0.0007765293121337891 seconds +31: Time to load utils op: 0.001657247543334961 seconds +23: Time to load utils op: 0.0013513565063476562 seconds +26: Time to load utils op: 0.0013778209686279297 seconds + 4: Time to load utils op: 0.00138092041015625 seconds +14: Time to load utils op: 0.0011186599731445312 seconds +18: Time to load utils op: 0.001188039779663086 seconds +31: Time to load utils op: 0.0016753673553466797 seconds +30: Time to load utils op: 0.0016818046569824219 seconds + 5: Time to load utils op: 0.001603841781616211 seconds +16: Time to load utils op: 0.0016057491302490234 seconds +11: Time to load utils op: 0.0012786388397216797 seconds +22: Time to load utils op: 0.0014107227325439453 seconds +20: Time to load utils op: 0.0015535354614257812 seconds +24: Time to load utils op: 0.001531839370727539 seconds +27: Time to load utils op: 0.0012841224670410156 seconds + 1: Time to load utils op: 0.0017747879028320312 seconds +11: Time to load utils op: 0.001529693603515625 seconds +18: Time to load utils op: 0.001413106918334961 seconds + 5: Time to load utils op: 0.001642465591430664 seconds +16: Time to load utils op: 0.001584768295288086 seconds +30: Time to load utils op: 0.0017197132110595703 seconds + 4: Time to load utils op: 0.001661062240600586 secondsTime to load utils op: 0.0016460418701171875 seconds + 4: + 5: Time to load utils op: 0.0016536712646484375 seconds +10: Time to load utils op: 0.0014030933380126953 seconds +27: Time to load utils op: 0.0012798309326171875 seconds +16: Time to load utils op: 0.0016636848449707031 seconds +23: Time to load utils op: 0.0016756057739257812 secondsTime to load utils op: 0.0014386177062988281 seconds +23: + 1: Time to load utils op: 0.0016393661499023438 seconds +11: Time to load utils op: 0.0013399124145507812 seconds +21: Time to load utils op: 0.0012860298156738281 seconds +14: Time to load utils op: 0.0016319751739501953 seconds + 4: Time to load utils op: 0.0016889572143554688 seconds +12: Time to load utils op: 0.0018041133880615234 secondsTime to load utils op: 0.0017991065979003906 seconds +12: +12: Time to load utils op: 0.0017788410186767578 seconds +27: Time to load utils op: 0.0015552043914794922 seconds +23: Time to load utils op: 0.0014710426330566406 seconds +21: Time to load utils op: 0.0013289451599121094 seconds +10: Time to load utils op: 0.001420736312866211 seconds +20: Time to load utils op: 0.0017857551574707031 seconds +20: Time to load utils op: 0.0016181468963623047 seconds +24: Time to load utils op: 0.001802206039428711 secondsTime to load utils op: 0.0017714500427246094 seconds +24: +12: Time to load utils op: 0.0018968582153320312 seconds +14: Time to load utils op: 0.0016138553619384766 seconds +21: Time to load utils op: 0.0015561580657958984 seconds +24: Time to load utils op: 0.001852273941040039 seconds +20: Time to load utils op: 0.001802206039428711 seconds + 6: Time to load utils op: 0.002179384231567383 seconds + 6: Time to load utils op: 0.0022225379943847656 seconds + 6: Time to load utils op: 0.002276897430419922 seconds + 0: [2023-05-09 16:44:08,200] [INFO] [utils.py:827:see_memory_usage] after initializing group 0 + 0: [2023-05-09 16:44:08,200] [INFO] [utils.py:828:see_memory_usage] MA 5.38 GB Max_MA 5.38 GB CA 6.82 GB Max_CA 7 GB + 0: [2023-05-09 16:44:08,201] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.05 GB, percent = 7.8% + 0: [2023-05-09 16:44:08,308] [INFO] [utils.py:827:see_memory_usage] before initializing group 1 + 0: [2023-05-09 16:44:08,308] [INFO] [utils.py:828:see_memory_usage] MA 5.38 GB Max_MA 5.38 GB CA 6.82 GB Max_CA 7 GB + 0: [2023-05-09 16:44:08,309] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.1 GB, percent = 7.8% + 0: [2023-05-09 16:44:08,417] [INFO] [utils.py:827:see_memory_usage] after initializing group 1 + 0: [2023-05-09 16:44:08,418] [INFO] [utils.py:828:see_memory_usage] MA 7.9 GB Max_MA 7.9 GB CA 10.63 GB Max_CA 11 GB + 0: [2023-05-09 16:44:08,418] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.12 GB, percent = 7.8% + 0: [2023-05-09 16:44:08,525] [INFO] [utils.py:827:see_memory_usage] before initializing group 2 + 0: [2023-05-09 16:44:08,526] [INFO] [utils.py:828:see_memory_usage] MA 7.9 GB Max_MA 7.9 GB CA 10.63 GB Max_CA 11 GB + 0: [2023-05-09 16:44:08,526] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.13 GB, percent = 7.8% + 0: [2023-05-09 16:44:08,638] [INFO] [utils.py:827:see_memory_usage] after initializing group 2 + 0: [2023-05-09 16:44:08,638] [INFO] [utils.py:828:see_memory_usage] MA 7.91 GB Max_MA 7.91 GB CA 10.63 GB Max_CA 11 GB + 0: [2023-05-09 16:44:08,638] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.14 GB, percent = 7.8% + 0: [2023-05-09 16:44:08,746] [INFO] [utils.py:827:see_memory_usage] before initialize_optimizer + 0: [2023-05-09 16:44:08,746] [INFO] [utils.py:828:see_memory_usage] MA 7.91 GB Max_MA 7.91 GB CA 10.63 GB Max_CA 11 GB + 0: [2023-05-09 16:44:08,746] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.14 GB, percent = 7.8% + 0: [2023-05-09 16:44:08,910] [INFO] [utils.py:827:see_memory_usage] end initialize_optimizer + 0: [2023-05-09 16:44:08,911] [INFO] [utils.py:828:see_memory_usage] MA 7.99 GB Max_MA 7.99 GB CA 10.63 GB Max_CA 11 GB + 0: [2023-05-09 16:44:08,911] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.19 GB, percent = 7.8% + 0: [2023-05-09 16:44:09,022] [INFO] [utils.py:827:see_memory_usage] end bf16_optimizer + 0: [2023-05-09 16:44:09,023] [INFO] [utils.py:828:see_memory_usage] MA 7.99 GB Max_MA 7.99 GB CA 10.63 GB Max_CA 11 GB + 0: [2023-05-09 16:44:09,023] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.23 GB, percent = 7.8% + 0: [2023-05-09 16:44:09,023] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam + 0: [2023-05-09 16:44:09,023] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using client LR scheduler + 0: [2023-05-09 16:44:09,023] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = + 0: [2023-05-09 16:44:09,023] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0002, 0.0002, 0.0002], mom=[(0.9, 0.999), (0.9, 0.999), (0.9, 0.999)] + 0: [2023-05-09 16:44:09,024] [INFO] [config.py:1007:print] DeepSpeedEngine configuration: + 0: [2023-05-09 16:44:09,024] [INFO] [config.py:1011:print] activation_checkpointing_config { + 0: "partition_activations": false, + 0: "contiguous_memory_optimization": false, + 0: "cpu_checkpointing": false, + 0: "number_checkpoints": null, + 0: "synchronize_checkpoint_boundary": false, + 0: "profile": false + 0: } + 0: [2023-05-09 16:44:09,025] [INFO] [config.py:1011:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} + 0: [2023-05-09 16:44:09,025] [INFO] [config.py:1011:print] amp_enabled .................. False + 0: [2023-05-09 16:44:09,025] [INFO] [config.py:1011:print] amp_params ................... False + 0: [2023-05-09 16:44:09,025] [INFO] [config.py:1011:print] autotuning_config ............ { + 0: "enabled": false, + 0: "start_step": null, + 0: "end_step": null, + 0: "metric_path": null, + 0: "arg_mappings": null, + 0: "metric": "throughput", + 0: "model_info": null, + 0: "results_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_results", + 0: "exps_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_exps", + 0: "overwrite": true, + 0: "fast": true, + 0: "start_profile_step": 3, + 0: "end_profile_step": 5, + 0: "tuner_type": "gridsearch", + 0: "tuner_early_stopping": 5, + 0: "tuner_num_trials": 50, + 0: "model_info_path": null, + 0: "mp_size": 1, + 0: "max_train_batch_size": null, + 0: "min_train_batch_size": 1, + 0: "max_train_micro_batch_size_per_gpu": 1.024000e+03, + 0: "min_train_micro_batch_size_per_gpu": 1, + 0: "num_tuning_micro_batch_sizes": 3 + 0: } + 0: [2023-05-09 16:44:09,025] [INFO] [config.py:1011:print] bfloat16_enabled ............. True + 0: [2023-05-09 16:44:09,025] [INFO] [config.py:1011:print] checkpoint_parallel_write_pipeline False + 0: [2023-05-09 16:44:09,025] [INFO] [config.py:1011:print] checkpoint_tag_validation_enabled True + 0: [2023-05-09 16:44:09,025] [INFO] [config.py:1011:print] checkpoint_tag_validation_fail False + 0: [2023-05-09 16:44:09,025] [INFO] [config.py:1011:print] comms_config ................. + 0: [2023-05-09 16:44:09,025] [INFO] [config.py:1011:print] communication_data_type ...... None + 0: [2023-05-09 16:44:09,025] [INFO] [config.py:1011:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_pa + 0: rameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} + 0: [2023-05-09 16:44:09,025] [INFO] [config.py:1011:print] curriculum_enabled ........... False + 0: [2023-05-09 16:44:09,025] [INFO] [config.py:1011:print] curriculum_params ............ False + 0: [2023-05-09 16:44:09,025] [INFO] [config.py:1011:print] dataloader_drop_last ......... False + 0: [2023-05-09 16:44:09,025] [INFO] [config.py:1011:print] disable_allgather ............ False + 0: [2023-05-09 16:44:09,025] [INFO] [config.py:1011:print] dump_state ................... False + 0: [2023-05-09 16:44:09,025] [INFO] [config.py:1011:print] dynamic_loss_scale_args ...... None + 0: [2023-05-09 16:44:09,025] [INFO] [config.py:1011:print] eigenvalue_enabled ........... False + 0: [2023-05-09 16:44:09,025] [INFO] [config.py:1011:print] eigenvalue_gas_boundary_resolution 1 + 0: [2023-05-09 16:44:09,025] [INFO] [config.py:1011:print] eigenvalue_layer_name ........ bert.encoder.layer + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] eigenvalue_layer_num ......... 0 + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] eigenvalue_max_iter .......... 100 + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] eigenvalue_stability ......... 1e-06 + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] eigenvalue_tol ............... 0.01 + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] eigenvalue_verbose ........... False + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] elasticity_enabled ........... False + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] flops_profiler_config ........ { + 0: "enabled": false, + 0: "profile_step": 1, + 0: "module_depth": -1, + 0: "top_modules": 1, + 0: "detailed": true, + 0: "output_file": null + 0: } + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] fp16_auto_cast ............... None + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] fp16_enabled ................. False + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] fp16_master_weights_and_gradients False + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] global_rank .................. 0 + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] gradient_accumulation_steps .. 4 + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] gradient_clipping ............ 1.0 + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] gradient_predivide_factor .... 1.0 + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] initial_dynamic_scale ........ 1 + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] load_universal_checkpoint .... False + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] loss_scale ................... 1.0 + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] memory_breakdown ............. False + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] monitor_config ............... + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] nebula_config ................ { + 0: "enabled": false, + 0: "persistent_storage_path": null, + 0: "persistent_time_interval": 100, + 0: "num_of_version_in_retention": 2, + 0: "enable_nebula_load": true, + 0: "load_path": null + 0: } + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] optimizer_legacy_fusion ...... False + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] optimizer_name ............... None + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] optimizer_params ............. None + 0: [2023-05-09 16:44:09,026] [INFO] [config.py:1011:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} + 0: [2023-05-09 16:44:09,027] [INFO] [config.py:1011:print] pld_enabled .................. False + 0: [2023-05-09 16:44:09,027] [INFO] [config.py:1011:print] pld_params ................... False + 0: [2023-05-09 16:44:09,027] [INFO] [config.py:1011:print] prescale_gradients ........... False + 0: [2023-05-09 16:44:09,027] [INFO] [config.py:1011:print] scheduler_name ............... None + 0: [2023-05-09 16:44:09,027] [INFO] [config.py:1011:print] scheduler_params ............. None + 0: [2023-05-09 16:44:09,027] [INFO] [config.py:1011:print] sparse_attention ............. None + 0: [2023-05-09 16:44:09,027] [INFO] [config.py:1011:print] sparse_gradients_enabled ..... False + 0: [2023-05-09 16:44:09,027] [INFO] [config.py:1011:print] steps_per_print .............. 2000 + 0: [2023-05-09 16:44:09,027] [INFO] [config.py:1011:print] train_batch_size ............. 1024 + 0: [2023-05-09 16:44:09,027] [INFO] [config.py:1011:print] train_micro_batch_size_per_gpu 2 + 0: [2023-05-09 16:44:09,027] [INFO] [config.py:1011:print] use_node_local_storage ....... False + 0: [2023-05-09 16:44:09,027] [INFO] [config.py:1011:print] wall_clock_breakdown ......... False + 0: [2023-05-09 16:44:09,027] [INFO] [config.py:1011:print] world_size ................... 128 + 0: [2023-05-09 16:44:09,027] [INFO] [config.py:1011:print] zero_allow_untested_optimizer False + 0: [2023-05-09 16:44:09,027] [INFO] [config.py:1011:print] zero_config .................. stage=0 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False + 0: [2023-05-09 16:44:09,027] [INFO] [config.py:1011:print] zero_enabled ................. False + 0: [2023-05-09 16:44:09,027] [INFO] [config.py:1011:print] zero_optimization_stage ...... 0 + 0: [2023-05-09 16:44:09,027] [INFO] [config.py:996:print_user_config] json = { + 0: "train_micro_batch_size_per_gpu": 2, + 0: "train_batch_size": 1.024000e+03, + 0: "gradient_clipping": 1.0, + 0: "zero_optimization": { + 0: "stage": 0 + 0: }, + 0: "bf16": { + 0: "enabled": true + 0: }, + 0: "steps_per_print": 2.000000e+03, + 0: "wall_clock_breakdown": false + 0: } + 0: Time to load utils op: 0.0005881786346435547 seconds + 0: [2023-05-09 16:44:09,028] [INFO] [engine.py:87:__init__] CONFIG: micro_batches=4 micro_batch_size=2 + 0: ninja: no work to do. + 0: Time to load utils op: 0.30824828147888184 secondsTime to load utils op: 0.3077373504638672 seconds + 0: + 0: Time to load utils op: 0.3254852294921875 seconds + 0: Time to load utils op: 0.30867552757263184 seconds +12: Time to load utils op: 0.2707023620605469 seconds + 3: Time to load utils op: 0.27298831939697266 seconds +21: Time to load utils op: 0.25034308433532715 seconds + 5: Time to load utils op: 0.2559390068054199 seconds +12: Time to load utils op: 0.27969837188720703 seconds + 3: Time to load utils op: 0.27891111373901367 seconds +11: Time to load utils op: 0.2645530700683594 seconds +17: Time to load utils op: 0.25603747367858887 seconds +30: Time to load utils op: 0.25595903396606445 seconds +21: Time to load utils op: 0.2659728527069092 seconds + 5: Time to load utils op: 0.26202917098999023 seconds +12: Time to load utils op: 0.27997326850891113 seconds +20: Time to load utils op: 0.2555832862854004 seconds +13: Time to load utils op: 0.2449030876159668 seconds +21: Time to load utils op: 0.2594716548919678 seconds + 3: Time to load utils op: 0.28539276123046875 seconds +11: Time to load utils op: 0.2706785202026367 seconds +17: Time to load utils op: 0.28093814849853516 seconds + 4: Time to load utils op: 0.24691414833068848 seconds +14: Time to load utils op: 0.24415254592895508 seconds +30: Time to load utils op: 0.2772064208984375 seconds + 5: Time to load utils op: 0.2832660675048828 seconds +13: Time to load utils op: 0.25073719024658203 seconds +20: Time to load utils op: 0.26167750358581543 seconds +12: Time to load utils op: 0.28608274459838867 seconds + 3: Time to load utils op: 0.29129934310913086 seconds +21: Time to load utils op: 0.26598501205444336 seconds +11: Time to load utils op: 0.27675724029541016 seconds +17: Time to load utils op: 0.2682375907897949 seconds +14: Time to load utils op: 0.2501192092895508 seconds +30: Time to load utils op: 0.2681732177734375 seconds + 4: Time to load utils op: 0.252699613571167 seconds + 5: Time to load utils op: 0.2741835117340088 seconds +20: Time to load utils op: 0.2675149440765381 seconds +13: Time to load utils op: 0.25707173347473145 seconds +11: Time to load utils op: 0.28264760971069336 seconds + 4: Time to load utils op: 0.2588176727294922 seconds +13: Time to load utils op: 0.2604508399963379 seconds +14: Time to load utils op: 0.2561478614807129 seconds +17: Time to load utils op: 0.27451181411743164 seconds +30: Time to load utils op: 0.2744140625 seconds + 4: Time to load utils op: 0.26238536834716797 seconds +20: Time to load utils op: 0.2736067771911621 seconds + 2: Time to load utils op: 0.2679328918457031 seconds +14: Time to load utils op: 0.26244449615478516 seconds + 2: Time to load utils op: 0.2740781307220459 seconds + 9: Time to load utils op: 0.32839274406433105 seconds + 7: Time to load utils op: 0.3281881809234619 seconds + 9: Time to load utils op: 0.3221578598022461 seconds + 2: Time to load utils op: 0.2801671028137207 seconds + 1: Time to load utils op: 0.36095714569091797 seconds + 8: Time to load utils op: 0.3569676876068115 seconds + 7: Time to load utils op: 0.3281233310699463 seconds + 9: Time to load utils op: 0.328230619430542 seconds + 1: Time to load utils op: 0.36355161666870117 seconds + 2: Time to load utils op: 0.2862827777862549 seconds + 8: Time to load utils op: 0.36272096633911133 seconds +19: Time to load utils op: 0.3593461513519287 seconds + 7: Time to load utils op: 0.33432841300964355 seconds + 9: Time to load utils op: 0.33418822288513184 seconds + 1: Time to load utils op: 0.37035584449768066 seconds + 8: Time to load utils op: 0.3688027858734131 seconds +19: Time to load utils op: 0.36495161056518555 seconds + 7: Time to load utils op: 0.340287446975708 seconds + 1: Time to load utils op: 0.37453317642211914 seconds +16: Time to load utils op: 0.3673982620239258 seconds +23: Time to load utils op: 0.36582493782043457 seconds + 8: Time to load utils op: 0.37518763542175293 seconds +19: Time to load utils op: 0.3706779479980469 seconds +16: Time to load utils op: 0.374176025390625 seconds +10: Time to load utils op: 0.32486581802368164 seconds +23: Time to load utils op: 0.37192392349243164 seconds +19: Time to load utils op: 0.37639427185058594 seconds +10: Time to load utils op: 0.34618687629699707 seconds +23: Time to load utils op: 0.37575435638427734 seconds +16: Time to load utils op: 0.38074803352355957 seconds +10: Time to load utils op: 0.3338773250579834 seconds +29: Time to load utils op: 0.37762022018432617 seconds +28: Time to load utils op: 0.3783893585205078 seconds +23: Time to load utils op: 0.38083600997924805 seconds +15: Time to load utils op: 0.38706278800964355 seconds +25: Time to load utils op: 0.38170814514160156 seconds +16: Time to load utils op: 0.3857157230377197 seconds +26: Time to load utils op: 0.381927490234375 seconds +31: Time to load utils op: 0.3775022029876709 seconds +29: Time to load utils op: 0.3834514617919922 seconds +28: Time to load utils op: 0.38509249687194824 seconds +10: Time to load utils op: 0.3402538299560547 seconds +15: Time to load utils op: 0.39232349395751953 seconds +25: Time to load utils op: 0.38776326179504395 seconds + 6: Time to load utils op: 0.362027645111084 seconds +26: Time to load utils op: 0.38727617263793945 seconds +31: Time to load utils op: 0.38317012786865234 seconds +22: Time to load utils op: 0.32239794731140137 seconds +18: Time to load utils op: 0.31578946113586426 seconds +28: Time to load utils op: 0.3908658027648926 seconds +29: Time to load utils op: 0.38968706130981445 seconds +15: Time to load utils op: 0.3992915153503418 seconds +24: Time to load utils op: 0.3765740394592285 seconds +25: Time to load utils op: 0.393810510635376 seconds +26: Time to load utils op: 0.39377307891845703 seconds + 6: Time to load utils op: 0.34969377517700195 seconds +27: Time to load utils op: 0.395479679107666 seconds +31: Time to load utils op: 0.38930749893188477 seconds +25: Time to load utils op: 0.3966519832611084 seconds +22: Time to load utils op: 0.328336238861084 seconds +18: Time to load utils op: 0.32184863090515137 seconds +29: Time to load utils op: 0.39518046379089355 seconds +28: Time to load utils op: 0.39685606956481934 seconds +15: Time to load utils op: 0.40438175201416016 seconds +24: Time to load utils op: 0.38883519172668457 seconds +26: Time to load utils op: 0.39881229400634766 seconds + 6: Time to load utils op: 0.3649322986602783 seconds +31: Time to load utils op: 0.3952760696411133 seconds +27: Time to load utils op: 0.40082454681396484 seconds +22: Time to load utils op: 0.3347606658935547 seconds +18: Time to load utils op: 0.3276832103729248 seconds +24: Time to load utils op: 0.3948938846588135 seconds + 6: Time to load utils op: 0.3615841865539551 seconds +27: Time to load utils op: 0.4071803092956543 seconds +22: Time to load utils op: 0.3559858798980713 seconds +18: Time to load utils op: 0.3522822856903076 seconds +24: Time to load utils op: 0.4009997844696045 seconds +27: Time to load utils op: 0.41347312927246094 seconds +12: Time to load utils op: 0.002178668975830078 secondsTime to load utils op: 0.002133607864379883 seconds +12: +12: Time to load utils op: 0.0010704994201660156 seconds +12: Time to load utils op: 0.0004544258117675781 seconds + 0: Time to load utils op: 0.005436420440673828 secondsTime to load utils op: 0.005382061004638672 seconds + 0: + 0: Time to load utils op: 0.004421710968017578 seconds + 0: Time to load utils op: 0.004764556884765625 seconds + 0: [2023-05-09 16:44:09,429] [INFO] [engine.py:145:__init__] RANK=0 STAGE=0 LAYERS=41 [0, 41) STAGE_PARAMS=1407562240 (1407.562M) TOTAL_PARAMS=2815124480 (2815.124M) UNIQUE_PARAMS=2815124480 (2815.124M) + 0: [2023-05-09 16:44:09,429] [INFO] [engine.py:145:__init__] RANK=1 STAGE=0 LAYERS=41 [0, 41) STAGE_PARAMS=1407562240 (1407.562M) TOTAL_PARAMS=2815124480 (2815.124M) UNIQUE_PARAMS=2815124480 (2815.124M) +24: Time to load utils op: 0.004223823547363281 seconds +24: Time to load utils op: 0.004686117172241211 seconds +24: Time to load utils op: 0.0004394054412841797 seconds +24: Time to load utils op: 0.0004699230194091797 seconds +20: Time to load utils op: 0.0036187171936035156 secondsTime to load utils op: 0.003604412078857422 seconds +20: +20: Time to load utils op: 0.006472349166870117 secondsTime to load utils op: 0.006388664245605469 seconds +20: + 2: Time to load utils op: 0.006640434265136719 seconds +13: Time to load utils op: 0.006566524505615234 secondsTime to load utils op: 0.006561279296875 seconds +13: + 2: Time to load utils op: 0.006546497344970703 seconds + 2: Time to load utils op: 0.0065765380859375 seconds + 2: Time to load utils op: 0.0004775524139404297 seconds +13: Time to load utils op: 0.0005092620849609375 seconds +13: Time to load utils op: 0.0004723072052001953 seconds +21: Time to load utils op: 0.004004240036010742 seconds +21: Time to load utils op: 0.003870248794555664 seconds + 3: Time to load utils op: 0.001425027847290039 seconds +11: Time to load utils op: 0.0037353038787841797 seconds + 3: Time to load utils op: 0.0003440380096435547 seconds + 3: Time to load utils op: 0.0005047321319580078 seconds +17: Time to load utils op: 0.003823518753051758 seconds +17: Time to load utils op: 0.0004024505615234375 seconds + 5: Time to load utils op: 0.0038728713989257812 seconds +30: Time to load utils op: 0.0037076473236083984 seconds +30: Time to load utils op: 0.006643533706665039 seconds +30: Time to load utils op: 0.006512880325317383 seconds +21: Time to load utils op: 0.0003662109375 seconds + 3: Time to load utils op: 0.00040721893310546875 seconds + 4: Time to load utils op: 0.006443500518798828 seconds + 5: Time to load utils op: 0.0005035400390625 seconds +19: Time to load utils op: 0.004030704498291016 seconds +21: Time to load utils op: 0.00039005279541015625 seconds + 4: Time to load utils op: 0.00043582916259765625 seconds + 4: Time to load utils op: 0.0004246234893798828 seconds +17: Time to load utils op: 0.0004737377166748047 seconds +11: Time to load utils op: 0.0004661083221435547 seconds +19: Time to load utils op: 0.004099607467651367 seconds + 5: Time to load utils op: 0.0004150867462158203 seconds +11: Time to load utils op: 0.0004794597625732422 seconds +19: Time to load utils op: 0.0004811286926269531 seconds + 9: Time to load utils op: 0.0065310001373291016 seconds +17: Time to load utils op: 0.00033926963806152344 seconds + 5: Time to load utils op: 0.0004787445068359375 seconds + 9: Time to load utils op: 0.0003638267517089844 seconds +11: Time to load utils op: 0.0004062652587890625 seconds +30: Time to load utils op: 0.0004699230194091797 seconds + 9: Time to load utils op: 0.0004401206970214844 seconds + 9: Time to load utils op: 0.00046896934509277344 seconds +19: Time to load utils op: 0.00044083595275878906 seconds + 1: Time to load utils op: 0.003635406494140625 secondsTime to load utils op: 0.003590106964111328 seconds + 1: + 1: Time to load utils op: 0.0004432201385498047 seconds +14: Time to load utils op: 0.004092693328857422 seconds +14: Time to load utils op: 0.004019260406494141 seconds + 1: Time to load utils op: 0.006502628326416016 seconds + 4: Time to load utils op: 0.00043201446533203125 seconds +14: Time to load utils op: 0.004117727279663086 seconds +14: Time to load utils op: 0.0005125999450683594 seconds + 8: Time to load utils op: 0.003924846649169922 seconds + 8: Time to load utils op: 0.003732442855834961 seconds + 7: Time to load utils op: 0.0041484832763671875 seconds + 8: Time to load utils op: 0.006512165069580078 seconds + 7: Time to load utils op: 0.0040051937103271484 seconds + 7: Time to load utils op: 0.0039751529693603516 seconds + 7: Time to load utils op: 0.004045963287353516 seconds +16: Time to load utils op: 0.0041730403900146484 seconds +16: Time to load utils op: 0.006570339202880859 seconds +28: Time to load utils op: 0.004095554351806641 seconds +16: Time to load utils op: 0.0003323554992675781 seconds + 8: Time to load utils op: 0.00038814544677734375 seconds +28: Time to load utils op: 0.00400853157043457 seconds +28: Time to load utils op: 0.00412750244140625 seconds +16: Time to load utils op: 0.00046825408935546875 seconds +29: Time to load utils op: 0.006385087966918945 seconds +28: Time to load utils op: 0.0004601478576660156 seconds +29: Time to load utils op: 0.00040721893310546875 seconds +25: Time to load utils op: 0.004117488861083984 secondsTime to load utils op: 0.004038572311401367 secondsTime to load utils op: 0.004094123840332031 seconds +25: +25: +23: Time to load utils op: 0.00626063346862793 seconds +26: Time to load utils op: 0.0039196014404296875 seconds +27: Time to load utils op: 0.006387948989868164 seconds +23: Time to load utils op: 0.0004324913024902344 seconds +23: Time to load utils op: 0.0038361549377441406 seconds +10: Time to load utils op: 0.004061460494995117 seconds +10: Time to load utils op: 0.004077434539794922 seconds +15: Time to load utils op: 0.003708362579345703 seconds +27: Time to load utils op: 0.0003285408020019531 seconds +27: Time to load utils op: 0.006460666656494141 seconds +26: Time to load utils op: 0.00047779083251953125 seconds +15: Time to load utils op: 0.0003612041473388672 seconds +15: Time to load utils op: 0.00035071372985839844 seconds +25: Time to load utils op: 0.00048232078552246094 seconds +10: Time to load utils op: 0.003996133804321289 seconds +10: Time to load utils op: 0.00047898292541503906 seconds +18: Time to load utils op: 0.003813505172729492 seconds +15: Time to load utils op: 0.00040268898010253906 seconds +29: Time to load utils op: 0.00048470497131347656 seconds +22: Time to load utils op: 0.004044294357299805 secondsTime to load utils op: 0.0040700435638427734 seconds +22: +22: Time to load utils op: 0.0004799365997314453 seconds +18: Time to load utils op: 0.0064928531646728516 seconds +22: Time to load utils op: 0.000415802001953125 seconds + 6: Time to load utils op: 0.006618976593017578 seconds + 6: Time to load utils op: 0.006442546844482422 seconds +31: Time to load utils op: 0.003992319107055664 seconds +31: Time to load utils op: 0.004048347473144531 secondsTime to load utils op: 0.0041730403900146484 seconds +31: +31: Time to load utils op: 0.0035903453826904297 seconds +29: Time to load utils op: 0.0005393028259277344 seconds + 6: Time to load utils op: 0.006376028060913086 seconds +26: Time to load utils op: 0.00040912628173828125 seconds +27: Time to load utils op: 0.00047135353088378906 seconds +23: Time to load utils op: 0.0004749298095703125 seconds +26: Time to load utils op: 0.0004038810729980469 seconds + 6: Time to load utils op: 0.00045990943908691406 seconds +18: Time to load utils op: 0.0004229545593261719 seconds +18: Time to load utils op: 0.0003998279571533203 seconds +12: [2023-05-09 16:44:12,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +12: [2023-05-09 16:44:12,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +12: [2023-05-09 16:44:12,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +12: [2023-05-09 16:44:12,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +12: [2023-05-09 16:44:12,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +12: [2023-05-09 16:44:12,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +12: [2023-05-09 16:44:12,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +12: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +16: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 7: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 7: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 8: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 8: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 8: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 8: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 8: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 7: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 7: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 7: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 8: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +16: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +16: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +16: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +16: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +16: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 7: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 8: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +14: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +14: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +14: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 7: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 8: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +16: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +14: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +14: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +14: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +31: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +31: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +14: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +31: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +31: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 7: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +16: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +31: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +31: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +31: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +31: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +15: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +15: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +15: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +15: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +15: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +14: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +15: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +15: [2023-05-09 16:44:12,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +24: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +28: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +24: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +24: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +24: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +24: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +24: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +20: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +20: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +20: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +24: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +22: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +22: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +20: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +22: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +22: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +22: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +22: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +22: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +20: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +20: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +20: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +28: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +28: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +23: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +28: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +23: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +10: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +22: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +23: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +23: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +10: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +10: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +10: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +10: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +10: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +10: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +23: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +23: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +28: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +28: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +28: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +15: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +23: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +10: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +17: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +24: [2023-05-09 16:44:12,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +29: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +29: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +29: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +29: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +29: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 6: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 6: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +29: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +29: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 6: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 6: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 6: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 6: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 4: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 6: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +17: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +20: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +13: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +13: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +13: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +13: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +13: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +13: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +13: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +28: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +17: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +17: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +17: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 3: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +17: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +17: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +23: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 6: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 4: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 9: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 9: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 3: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 9: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +19: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +19: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +19: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 9: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 9: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 9: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +18: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +18: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +18: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +18: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +18: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +19: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 3: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 2: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 2: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 2: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 2: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 4: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 4: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 4: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +18: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +19: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 2: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 2: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 9: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +29: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +17: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +26: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +18: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +19: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 3: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 3: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 2: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 4: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 4: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +30: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +30: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +30: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +30: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +13: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +26: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 9: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +30: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +26: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +26: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +26: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +19: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 3: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 3: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +30: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +30: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +26: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +30: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 5: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 5: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 5: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 5: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 5: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 5: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +26: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 5: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +19: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 0: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 0: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 2: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 0: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 0: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 3: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 4: [2023-05-09 16:44:12,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 5: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +26: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 0: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 0: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 0: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +18: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +27: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +27: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +27: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +21: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +21: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +27: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 0: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +21: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +27: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +27: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +27: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +21: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +21: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +21: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +21: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +11: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +11: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +11: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +11: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +11: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +21: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +11: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +11: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +27: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +11: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 1: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 1: [2023-05-09 16:44:12,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 1: [2023-05-09 16:44:12,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 1: [2023-05-09 16:44:12,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 1: [2023-05-09 16:44:12,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 1: [2023-05-09 16:44:12,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 1: [2023-05-09 16:44:12,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 1: [2023-05-09 16:44:12,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +25: [2023-05-09 16:44:12,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +25: [2023-05-09 16:44:12,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +25: [2023-05-09 16:44:12,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +25: [2023-05-09 16:44:12,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +25: [2023-05-09 16:44:12,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +25: [2023-05-09 16:44:12,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +25: [2023-05-09 16:44:12,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +25: [2023-05-09 16:44:12,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +25: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +25: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +25: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +25: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +25: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +15: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +15: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +15: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +29: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +29: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +29: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +29: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +29: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +25: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +25: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +16: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +16: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +16: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +15: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +15: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +15: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +15: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +29: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +29: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +25: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +29: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +29: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +15: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +15: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +15: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +15: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +25: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +25: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +25: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +25: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +29: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +16: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +16: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +16: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +16: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +15: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +16: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +29: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +29: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +16: [2023-05-09 16:44:12,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +16: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +16: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +16: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +15: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +15: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +31: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +31: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +31: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +31: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +31: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +25: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +25: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +15: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +15: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +29: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +29: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +29: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +31: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +31: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +25: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +29: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +25: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 8: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 8: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 8: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 8: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 8: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +16: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +16: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +31: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +31: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +31: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +31: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 8: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +16: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 8: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +16: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 8: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 8: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +31: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 8: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 8: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 8: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 1: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 1: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 1: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 1: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 1: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 1: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +27: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +27: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +27: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +27: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +27: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +31: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +31: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +31: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +31: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 1: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 1: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 1: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +27: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +27: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +26: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +26: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 0: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 0: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 0: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +27: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +27: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 1: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 0: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 8: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 8: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 8: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +27: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +27: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 1: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +26: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +26: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +26: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +26: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 0: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 0: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 0: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +19: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +19: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +19: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +19: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +19: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 1: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +26: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 0: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 0: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 0: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +27: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +19: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +19: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +26: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +26: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +19: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +19: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +26: [2023-05-09 16:44:12,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 8: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +26: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +19: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +19: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +19: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +29: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +27: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 1: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 1: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 1: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 0: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +27: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +27: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +27: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 1: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +29: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +26: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 7: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 7: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 7: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +14: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +14: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +29: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 7: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 7: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 7: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 7: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +14: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +14: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +14: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +14: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 0: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 0: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 0: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 7: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +14: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +15: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 0: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 0: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +19: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +19: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 7: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +29: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +14: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +26: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +26: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +26: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 7: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 7: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +14: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +14: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +19: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +14: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +15: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +19: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +24: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +24: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +24: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +24: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +24: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +26: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 7: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +14: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +24: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +10: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +10: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +10: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +10: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +18: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +18: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +18: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +24: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +24: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +24: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 2: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 2: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 2: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 2: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 2: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +10: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +10: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +10: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +12: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +12: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +12: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +17: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +17: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +17: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +17: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +24: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 8: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +18: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +18: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +18: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +18: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 2: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +10: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +12: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +12: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +12: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +12: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +17: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +17: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +17: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 2: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +10: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 6: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 6: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 6: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 6: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 6: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +15: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +18: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +24: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +12: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +17: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +24: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 2: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +10: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +10: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 6: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 6: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +14: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +14: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 2: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +10: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +12: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +12: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +12: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +12: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +17: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +17: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +13: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +13: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +13: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +13: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +13: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +17: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +14: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +15: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +18: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +18: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 2: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 2: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 2: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 7: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 7: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 6: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 6: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 6: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +14: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +30: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +30: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +30: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +30: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +30: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +13: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +13: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +18: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 6: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +16: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +17: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +11: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +11: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +11: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +11: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +11: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 7: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 7: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 8: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +30: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +13: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +13: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +13: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +13: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +29: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +30: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +11: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +24: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +24: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +24: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +24: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +31: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +30: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +30: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +13: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +11: [2023-05-09 16:44:12,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 8: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 6: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +30: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +10: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +10: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +29: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +11: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +11: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +11: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +11: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +18: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +30: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 5: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 5: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 5: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 5: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 5: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +10: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +12: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +12: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +12: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +12: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +17: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +17: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +17: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +17: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +22: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +22: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +22: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +22: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +22: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +10: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +30: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 5: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 5: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +22: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +22: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +29: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 5: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 5: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +28: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 8: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +22: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +22: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +18: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +18: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +18: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 5: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +31: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +22: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 9: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 9: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 9: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 9: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 9: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 6: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +27: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +16: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +11: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +18: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +21: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +21: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +21: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +21: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +21: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 2: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 2: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 2: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 2: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 4: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 4: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 5: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +28: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +28: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +15: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +13: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +13: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +13: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 9: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 9: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +28: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +28: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 6: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +29: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +31: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +30: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +21: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +21: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 4: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 4: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 4: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 4: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 9: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 5: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +28: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +28: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +22: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +23: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +23: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +23: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +23: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +23: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +20: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +20: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +20: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 4: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +31: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +30: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +30: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +13: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +21: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +21: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +21: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 3: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 3: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 3: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 9: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 9: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +28: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +30: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +22: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +23: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +23: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +21: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +20: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 4: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +28: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 6: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +15: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +20: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +20: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +20: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 3: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 3: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 3: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 3: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 4: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 9: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +28: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +11: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +11: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +23: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +23: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +21: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 8: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +15: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +20: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +20: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +20: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 4: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 4: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 6: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +15: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +11: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +11: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +23: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 3: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +28: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +16: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +16: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +20: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 4: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +28: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 0: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 9: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +27: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +25: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +23: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 3: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 3: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 3: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 3: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 5: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 7: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +22: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +22: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +22: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +22: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +20: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... + 5: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 5: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +31: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 5: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +27: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +14: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 0: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 8: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 8: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 0: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +23: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt... +31: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +25: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 4: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 4: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 4: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 4: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 7: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +25: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +16: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +31: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 0: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 8: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +12: [2023-05-09 16:44:12,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +10: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +31: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +14: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +20: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +20: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 7: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +17: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +27: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 1: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +17: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +20: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +20: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +24: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +25: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 9: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +27: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +26: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 3: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 3: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 3: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 3: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 6: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 9: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 9: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 9: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 7: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +27: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +17: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +14: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +19: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 1: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +13: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +21: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +21: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +21: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +21: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +30: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +14: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +19: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +10: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +28: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +28: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +28: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 1: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 7: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +12: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +27: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +23: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +26: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +13: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +23: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +23: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... + 7: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +28: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +23: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt... +16: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +16: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +30: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +24: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 6: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 0: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +12: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +17: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +12: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +30: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 7: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +16: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +19: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +13: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 0: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +21: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 0: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +24: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +24: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +10: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +10: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 6: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +27: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +17: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +17: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 0: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 7: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +14: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +14: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +13: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +21: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 6: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +12: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +10: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +24: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 6: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +12: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +30: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +17: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 1: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +14: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +14: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +13: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +13: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +25: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +21: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +26: [2023-05-09 16:44:12,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +26: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 6: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +25: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +25: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +24: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +12: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +18: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +21: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +17: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 2: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +10: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +12: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +22: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +19: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +13: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +22: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +25: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 6: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 1: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +30: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +19: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +21: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +24: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 9: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +10: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 1: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +13: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +24: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 6: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +30: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +11: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +21: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +22: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +30: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +30: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 1: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +26: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +21: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +19: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 4: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +19: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +22: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +21: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 4: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +11: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +10: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +11: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 5: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +26: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 9: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 5: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +18: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 9: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +11: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +23: [2023-05-09 16:44:12,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 2: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 9: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +18: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +28: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +22: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 1: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +18: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +28: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +26: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +26: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +22: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +22: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +18: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 9: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 2: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +28: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +22: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +19: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 2: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +20: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 5: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +23: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 3: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +23: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +28: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 4: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 5: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +11: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 3: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 2: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +23: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 4: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +11: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 4: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 9: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +20: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 3: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 9: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +11: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 4: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +18: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 3: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +11: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +20: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. + 9: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +20: [2023-05-09 16:44:12,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt. +18: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 5: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +23: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 2: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 5: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +18: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +28: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +28: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 5: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +23: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 4: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 2: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +23: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +28: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 5: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +28: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 4: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +20: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 2: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 3: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +23: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 3: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +20: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 3: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 3: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +20: [2023-05-09 16:44:12,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +20: [2023-05-09 16:44:12,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +15: [2023-05-09 16:44:12,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +15: [2023-05-09 16:44:12,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +15: [2023-05-09 16:44:12,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +15: [2023-05-09 16:44:12,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +15: [2023-05-09 16:44:12,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +15: [2023-05-09 16:44:12,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +15: [2023-05-09 16:44:12,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 8: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 8: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 8: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 8: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +15: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 7: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +29: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +29: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +29: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +17: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +17: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +14: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +17: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 0: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 0: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +24: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +27: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +27: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +31: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +14: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +14: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 0: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 7: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +29: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +31: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +31: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +13: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +12: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +17: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +30: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +30: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +31: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 7: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +27: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +12: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +14: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 0: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +10: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 8: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 8: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 8: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 6: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 6: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +12: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +13: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +30: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 6: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +13: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +22: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 8: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 7: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +10: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +10: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +27: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +17: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 7: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 6: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +29: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +29: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +13: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +17: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +14: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +21: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +21: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +21: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +29: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +30: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +17: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +12: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 0: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 0: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +10: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +24: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 7: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +14: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +14: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 0: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +21: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +29: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +27: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +22: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +24: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +31: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +12: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +17: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +13: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +27: [2023-05-09 16:44:12,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +31: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +31: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +30: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 7: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +12: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +16: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 6: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +30: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +14: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +31: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +13: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 6: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 6: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +27: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 0: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +10: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +30: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +24: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +12: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +13: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +22: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 7: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +16: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +22: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 6: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +16: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +10: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +10: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 1: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +22: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +27: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +30: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 9: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +13: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +21: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +21: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +21: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +12: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +18: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +16: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +24: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +10: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +21: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +19: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +19: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +18: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +25: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +22: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 1: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 1: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 9: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 9: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +18: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +26: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 1: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +11: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +25: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +25: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 9: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +16: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +26: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +18: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +11: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +25: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +23: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +22: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +11: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +19: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 4: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 5: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +24: [2023-05-09 16:44:12,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +22: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +16: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 9: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +20: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 5: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +26: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +19: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +16: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +24: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 4: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +20: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +23: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 4: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 2: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 3: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +18: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 5: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 1: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 2: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 2: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 3: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 3: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +26: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +20: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +18: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +16: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +28: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +28: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +28: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +23: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +24: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 2: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 9: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +20: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 3: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 5: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +19: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +19: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 9: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +11: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +18: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 9: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +28: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +11: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +25: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 1: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 1: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +18: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +11: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +23: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +11: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +23: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. + 4: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt. +26: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +25: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 5: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 1: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +25: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 4: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +26: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +25: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +19: [2023-05-09 16:44:12,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 4: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +23: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 4: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 2: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +20: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 5: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +19: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +26: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +20: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 5: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 3: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 2: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 3: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 3: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 2: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +26: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +20: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +28: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +28: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +28: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 3: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +23: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 2: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 5: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +20: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +11: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +28: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +23: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 4: [2023-05-09 16:44:12,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +29: [2023-05-09 16:44:13,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +29: [2023-05-09 16:44:13,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +29: [2023-05-09 16:44:13,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +29: [2023-05-09 16:44:13,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +29: [2023-05-09 16:44:13,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +29: [2023-05-09 16:44:13,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +29: [2023-05-09 16:44:13,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +29: [2023-05-09 16:44:13,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +14: [2023-05-09 16:44:13,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +14: [2023-05-09 16:44:13,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +14: [2023-05-09 16:44:13,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +14: [2023-05-09 16:44:13,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +14: [2023-05-09 16:44:13,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +14: [2023-05-09 16:44:13,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +14: [2023-05-09 16:44:13,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +14: [2023-05-09 16:44:13,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +14: [2023-05-09 16:44:13,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +14: [2023-05-09 16:44:13,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +14: [2023-05-09 16:44:13,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +14: [2023-05-09 16:44:13,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +29: [2023-05-09 16:44:13,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +29: [2023-05-09 16:44:13,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +29: [2023-05-09 16:44:13,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +29: [2023-05-09 16:44:13,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +14: [2023-05-09 16:44:13,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +14: [2023-05-09 16:44:13,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +10: [2023-05-09 16:44:13,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +10: [2023-05-09 16:44:13,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +14: [2023-05-09 16:44:13,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +14: [2023-05-09 16:44:13,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +29: [2023-05-09 16:44:13,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +29: [2023-05-09 16:44:13,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +29: [2023-05-09 16:44:13,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +29: [2023-05-09 16:44:13,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +10: [2023-05-09 16:44:13,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +10: [2023-05-09 16:44:13,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +10: [2023-05-09 16:44:13,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +10: [2023-05-09 16:44:13,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +10: [2023-05-09 16:44:13,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +10: [2023-05-09 16:44:13,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +10: [2023-05-09 16:44:13,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +10: [2023-05-09 16:44:13,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +10: [2023-05-09 16:44:13,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +10: [2023-05-09 16:44:13,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +10: [2023-05-09 16:44:13,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +10: [2023-05-09 16:44:13,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +10: [2023-05-09 16:44:13,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +10: [2023-05-09 16:44:13,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +14: [2023-05-09 16:44:13,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +14: [2023-05-09 16:44:13,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +14: [2023-05-09 16:44:13,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +14: [2023-05-09 16:44:13,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +10: [2023-05-09 16:44:13,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +16: [2023-05-09 16:44:13,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +16: [2023-05-09 16:44:13,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +22: [2023-05-09 16:44:13,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +22: [2023-05-09 16:44:13,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +22: [2023-05-09 16:44:13,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +22: [2023-05-09 16:44:13,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +22: [2023-05-09 16:44:13,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +22: [2023-05-09 16:44:13,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +22: [2023-05-09 16:44:13,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +13: [2023-05-09 16:44:13,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +13: [2023-05-09 16:44:13,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +13: [2023-05-09 16:44:13,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +13: [2023-05-09 16:44:13,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +22: [2023-05-09 16:44:13,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +13: [2023-05-09 16:44:13,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +13: [2023-05-09 16:44:13,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +13: [2023-05-09 16:44:13,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +13: [2023-05-09 16:44:13,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +16: [2023-05-09 16:44:13,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +16: [2023-05-09 16:44:13,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +16: [2023-05-09 16:44:13,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +16: [2023-05-09 16:44:13,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +16: [2023-05-09 16:44:13,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +16: [2023-05-09 16:44:13,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +29: [2023-05-09 16:44:13,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +29: [2023-05-09 16:44:13,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +22: [2023-05-09 16:44:13,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +22: [2023-05-09 16:44:13,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +13: [2023-05-09 16:44:13,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +13: [2023-05-09 16:44:13,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +13: [2023-05-09 16:44:13,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +13: [2023-05-09 16:44:13,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +13: [2023-05-09 16:44:13,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +13: [2023-05-09 16:44:13,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +13: [2023-05-09 16:44:13,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +29: [2023-05-09 16:44:13,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +29: [2023-05-09 16:44:13,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +22: [2023-05-09 16:44:13,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +22: [2023-05-09 16:44:13,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +18: [2023-05-09 16:44:13,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +18: [2023-05-09 16:44:13,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +18: [2023-05-09 16:44:13,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +18: [2023-05-09 16:44:13,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +18: [2023-05-09 16:44:13,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +22: [2023-05-09 16:44:13,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +22: [2023-05-09 16:44:13,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +18: [2023-05-09 16:44:13,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +18: [2023-05-09 16:44:13,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +13: [2023-05-09 16:44:13,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +22: [2023-05-09 16:44:13,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +18: [2023-05-09 16:44:13,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +16: [2023-05-09 16:44:13,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +22: [2023-05-09 16:44:13,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +16: [2023-05-09 16:44:13,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +16: [2023-05-09 16:44:13,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +16: [2023-05-09 16:44:13,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +16: [2023-05-09 16:44:13,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +16: [2023-05-09 16:44:13,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +18: [2023-05-09 16:44:13,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +16: [2023-05-09 16:44:13,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +18: [2023-05-09 16:44:13,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +18: [2023-05-09 16:44:13,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +18: [2023-05-09 16:44:13,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +18: [2023-05-09 16:44:13,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +16: [2023-05-09 16:44:13,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +18: [2023-05-09 16:44:13,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +18: [2023-05-09 16:44:13,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +18: [2023-05-09 16:44:13,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +10: [2023-05-09 16:44:13,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +10: [2023-05-09 16:44:13,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +10: [2023-05-09 16:44:13,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +21: [2023-05-09 16:44:13,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +21: [2023-05-09 16:44:13,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +21: [2023-05-09 16:44:13,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +21: [2023-05-09 16:44:13,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +21: [2023-05-09 16:44:13,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +21: [2023-05-09 16:44:13,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +21: [2023-05-09 16:44:13,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +21: [2023-05-09 16:44:13,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +21: [2023-05-09 16:44:13,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +21: [2023-05-09 16:44:13,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +21: [2023-05-09 16:44:13,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +14: [2023-05-09 16:44:13,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +14: [2023-05-09 16:44:13,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +14: [2023-05-09 16:44:13,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +21: [2023-05-09 16:44:13,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +21: [2023-05-09 16:44:13,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +21: [2023-05-09 16:44:13,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +21: [2023-05-09 16:44:13,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +21: [2023-05-09 16:44:13,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +10: [2023-05-09 16:44:13,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +29: [2023-05-09 16:44:13,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +29: [2023-05-09 16:44:13,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +14: [2023-05-09 16:44:13,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +29: [2023-05-09 16:44:13,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +10: [2023-05-09 16:44:13,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +29: [2023-05-09 16:44:13,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +10: [2023-05-09 16:44:13,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +10: [2023-05-09 16:44:13,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 5: [2023-05-09 16:44:13,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 5: [2023-05-09 16:44:13,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 5: [2023-05-09 16:44:13,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 5: [2023-05-09 16:44:13,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 5: [2023-05-09 16:44:13,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 5: [2023-05-09 16:44:13,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 5: [2023-05-09 16:44:13,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 5: [2023-05-09 16:44:13,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 5: [2023-05-09 16:44:13,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 5: [2023-05-09 16:44:13,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 5: [2023-05-09 16:44:13,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 5: [2023-05-09 16:44:13,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 5: [2023-05-09 16:44:13,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 5: [2023-05-09 16:44:13,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 5: [2023-05-09 16:44:13,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 5: [2023-05-09 16:44:13,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +13: [2023-05-09 16:44:13,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +22: [2023-05-09 16:44:13,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +22: [2023-05-09 16:44:13,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +16: [2023-05-09 16:44:13,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +18: [2023-05-09 16:44:13,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +13: [2023-05-09 16:44:13,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +13: [2023-05-09 16:44:13,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +13: [2023-05-09 16:44:13,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +22: [2023-05-09 16:44:13,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +18: [2023-05-09 16:44:13,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +18: [2023-05-09 16:44:13,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +18: [2023-05-09 16:44:13,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +16: [2023-05-09 16:44:13,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +16: [2023-05-09 16:44:13,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +30: [2023-05-09 16:44:13,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +30: [2023-05-09 16:44:13,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +30: [2023-05-09 16:44:13,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +30: [2023-05-09 16:44:13,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +30: [2023-05-09 16:44:13,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +30: [2023-05-09 16:44:13,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +30: [2023-05-09 16:44:13,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +30: [2023-05-09 16:44:13,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +22: [2023-05-09 16:44:13,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +16: [2023-05-09 16:44:13,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +30: [2023-05-09 16:44:13,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +30: [2023-05-09 16:44:13,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +30: [2023-05-09 16:44:13,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +30: [2023-05-09 16:44:13,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +30: [2023-05-09 16:44:13,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +30: [2023-05-09 16:44:13,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +30: [2023-05-09 16:44:13,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +30: [2023-05-09 16:44:13,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +13: [2023-05-09 16:44:13,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +18: [2023-05-09 16:44:13,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +13: [2023-05-09 16:44:13,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +22: [2023-05-09 16:44:13,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +22: [2023-05-09 16:44:13,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +13: [2023-05-09 16:44:13,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +13: [2023-05-09 16:44:13,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +21: [2023-05-09 16:44:13,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +18: [2023-05-09 16:44:13,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +18: [2023-05-09 16:44:13,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +18: [2023-05-09 16:44:13,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +16: [2023-05-09 16:44:13,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +22: [2023-05-09 16:44:13,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +21: [2023-05-09 16:44:13,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +21: [2023-05-09 16:44:13,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +21: [2023-05-09 16:44:13,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +17: [2023-05-09 16:44:13,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +17: [2023-05-09 16:44:13,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +16: [2023-05-09 16:44:13,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +16: [2023-05-09 16:44:13,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +17: [2023-05-09 16:44:13,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +17: [2023-05-09 16:44:13,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +17: [2023-05-09 16:44:13,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +17: [2023-05-09 16:44:13,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +17: [2023-05-09 16:44:13,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +17: [2023-05-09 16:44:13,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +31: [2023-05-09 16:44:13,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +31: [2023-05-09 16:44:13,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +31: [2023-05-09 16:44:13,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +31: [2023-05-09 16:44:13,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +26: [2023-05-09 16:44:13,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +26: [2023-05-09 16:44:13,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +26: [2023-05-09 16:44:13,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +26: [2023-05-09 16:44:13,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +26: [2023-05-09 16:44:13,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +31: [2023-05-09 16:44:13,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +31: [2023-05-09 16:44:13,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +26: [2023-05-09 16:44:13,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +31: [2023-05-09 16:44:13,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +26: [2023-05-09 16:44:13,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 2: [2023-05-09 16:44:13,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 2: [2023-05-09 16:44:13,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 2: [2023-05-09 16:44:13,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 2: [2023-05-09 16:44:13,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 2: [2023-05-09 16:44:13,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +12: [2023-05-09 16:44:13,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +12: [2023-05-09 16:44:13,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +26: [2023-05-09 16:44:13,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 2: [2023-05-09 16:44:13,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 2: [2023-05-09 16:44:13,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +31: [2023-05-09 16:44:13,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 2: [2023-05-09 16:44:13,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +12: [2023-05-09 16:44:13,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +12: [2023-05-09 16:44:13,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +12: [2023-05-09 16:44:13,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +12: [2023-05-09 16:44:13,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +12: [2023-05-09 16:44:13,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +12: [2023-05-09 16:44:13,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +17: [2023-05-09 16:44:13,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +17: [2023-05-09 16:44:13,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +17: [2023-05-09 16:44:13,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +16: [2023-05-09 16:44:13,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +26: [2023-05-09 16:44:13,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +26: [2023-05-09 16:44:13,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +12: [2023-05-09 16:44:13,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +26: [2023-05-09 16:44:13,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +26: [2023-05-09 16:44:13,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +26: [2023-05-09 16:44:13,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +26: [2023-05-09 16:44:13,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +12: [2023-05-09 16:44:13,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +12: [2023-05-09 16:44:13,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +12: [2023-05-09 16:44:13,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 2: [2023-05-09 16:44:13,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 2: [2023-05-09 16:44:13,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 2: [2023-05-09 16:44:13,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 2: [2023-05-09 16:44:13,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 2: [2023-05-09 16:44:13,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +22: [2023-05-09 16:44:13,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +12: [2023-05-09 16:44:13,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +12: [2023-05-09 16:44:13,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 2: [2023-05-09 16:44:13,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +17: [2023-05-09 16:44:13,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +12: [2023-05-09 16:44:13,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +17: [2023-05-09 16:44:13,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +17: [2023-05-09 16:44:13,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 2: [2023-05-09 16:44:13,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +17: [2023-05-09 16:44:13,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +12: [2023-05-09 16:44:13,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 2: [2023-05-09 16:44:13,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +17: [2023-05-09 16:44:13,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +31: [2023-05-09 16:44:13,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +31: [2023-05-09 16:44:13,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +31: [2023-05-09 16:44:13,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +26: [2023-05-09 16:44:13,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +31: [2023-05-09 16:44:13,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +31: [2023-05-09 16:44:13,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +31: [2023-05-09 16:44:13,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +26: [2023-05-09 16:44:13,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +31: [2023-05-09 16:44:13,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +31: [2023-05-09 16:44:13,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 5: [2023-05-09 16:44:13,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 5: [2023-05-09 16:44:13,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 5: [2023-05-09 16:44:13,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 5: [2023-05-09 16:44:13,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +21: [2023-05-09 16:44:13,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +21: [2023-05-09 16:44:13,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +21: [2023-05-09 16:44:13,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +21: [2023-05-09 16:44:13,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +24: [2023-05-09 16:44:13,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +24: [2023-05-09 16:44:13,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +24: [2023-05-09 16:44:13,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +24: [2023-05-09 16:44:13,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +24: [2023-05-09 16:44:13,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +24: [2023-05-09 16:44:13,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +24: [2023-05-09 16:44:13,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +24: [2023-05-09 16:44:13,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +24: [2023-05-09 16:44:13,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +30: [2023-05-09 16:44:13,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +19: [2023-05-09 16:44:13,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 5: [2023-05-09 16:44:13,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 5: [2023-05-09 16:44:13,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +24: [2023-05-09 16:44:13,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +24: [2023-05-09 16:44:13,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +19: [2023-05-09 16:44:13,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +24: [2023-05-09 16:44:13,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +24: [2023-05-09 16:44:13,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +24: [2023-05-09 16:44:13,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +24: [2023-05-09 16:44:13,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +19: [2023-05-09 16:44:13,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +19: [2023-05-09 16:44:13,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +19: [2023-05-09 16:44:13,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +19: [2023-05-09 16:44:13,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +19: [2023-05-09 16:44:13,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +19: [2023-05-09 16:44:13,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +24: [2023-05-09 16:44:13,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +30: [2023-05-09 16:44:13,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +30: [2023-05-09 16:44:13,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +30: [2023-05-09 16:44:13,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +19: [2023-05-09 16:44:13,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 5: [2023-05-09 16:44:13,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +19: [2023-05-09 16:44:13,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 5: [2023-05-09 16:44:13,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +19: [2023-05-09 16:44:13,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +19: [2023-05-09 16:44:13,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +19: [2023-05-09 16:44:13,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +19: [2023-05-09 16:44:13,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +19: [2023-05-09 16:44:13,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +19: [2023-05-09 16:44:13,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +30: [2023-05-09 16:44:13,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 3: [2023-05-09 16:44:13,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 3: [2023-05-09 16:44:13,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 3: [2023-05-09 16:44:13,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 3: [2023-05-09 16:44:13,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 3: [2023-05-09 16:44:13,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 3: [2023-05-09 16:44:13,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 3: [2023-05-09 16:44:13,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 3: [2023-05-09 16:44:13,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +12: [2023-05-09 16:44:13,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +30: [2023-05-09 16:44:13,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +17: [2023-05-09 16:44:13,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +30: [2023-05-09 16:44:13,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +30: [2023-05-09 16:44:13,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +17: [2023-05-09 16:44:13,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +26: [2023-05-09 16:44:13,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +26: [2023-05-09 16:44:13,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +15: [2023-05-09 16:44:13,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +15: [2023-05-09 16:44:13,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +15: [2023-05-09 16:44:13,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 3: [2023-05-09 16:44:13,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +15: [2023-05-09 16:44:13,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +15: [2023-05-09 16:44:13,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +15: [2023-05-09 16:44:13,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +15: [2023-05-09 16:44:13,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +15: [2023-05-09 16:44:13,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 2: [2023-05-09 16:44:13,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 2: [2023-05-09 16:44:13,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 3: [2023-05-09 16:44:13,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 3: [2023-05-09 16:44:13,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 3: [2023-05-09 16:44:13,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +26: [2023-05-09 16:44:13,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 2: [2023-05-09 16:44:13,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 2: [2023-05-09 16:44:13,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +12: [2023-05-09 16:44:13,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +12: [2023-05-09 16:44:13,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +12: [2023-05-09 16:44:13,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +15: [2023-05-09 16:44:13,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +26: [2023-05-09 16:44:13,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +15: [2023-05-09 16:44:13,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +15: [2023-05-09 16:44:13,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +15: [2023-05-09 16:44:13,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 3: [2023-05-09 16:44:13,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 3: [2023-05-09 16:44:13,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 3: [2023-05-09 16:44:13,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 3: [2023-05-09 16:44:13,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +15: [2023-05-09 16:44:13,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +15: [2023-05-09 16:44:13,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +31: [2023-05-09 16:44:13,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +31: [2023-05-09 16:44:13,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +15: [2023-05-09 16:44:13,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +15: [2023-05-09 16:44:13,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +17: [2023-05-09 16:44:13,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +17: [2023-05-09 16:44:13,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +20: [2023-05-09 16:44:13,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +20: [2023-05-09 16:44:13,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +20: [2023-05-09 16:44:13,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +20: [2023-05-09 16:44:13,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +17: [2023-05-09 16:44:13,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +20: [2023-05-09 16:44:13,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +20: [2023-05-09 16:44:13,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +20: [2023-05-09 16:44:13,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +20: [2023-05-09 16:44:13,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +17: [2023-05-09 16:44:13,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +12: [2023-05-09 16:44:13,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +31: [2023-05-09 16:44:13,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +26: [2023-05-09 16:44:13,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +26: [2023-05-09 16:44:13,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +31: [2023-05-09 16:44:13,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +20: [2023-05-09 16:44:13,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 2: [2023-05-09 16:44:13,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 2: [2023-05-09 16:44:13,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 2: [2023-05-09 16:44:13,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +20: [2023-05-09 16:44:13,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +20: [2023-05-09 16:44:13,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +20: [2023-05-09 16:44:13,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +20: [2023-05-09 16:44:13,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +20: [2023-05-09 16:44:13,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 2: [2023-05-09 16:44:13,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +20: [2023-05-09 16:44:13,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +26: [2023-05-09 16:44:13,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +26: [2023-05-09 16:44:13,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +24: [2023-05-09 16:44:13,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +24: [2023-05-09 16:44:13,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +20: [2023-05-09 16:44:13,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +24: [2023-05-09 16:44:13,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +12: [2023-05-09 16:44:13,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +12: [2023-05-09 16:44:13,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +12: [2023-05-09 16:44:13,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +17: [2023-05-09 16:44:13,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 9: [2023-05-09 16:44:13,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +31: [2023-05-09 16:44:13,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 9: [2023-05-09 16:44:13,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +17: [2023-05-09 16:44:13,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +31: [2023-05-09 16:44:13,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +19: [2023-05-09 16:44:13,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +24: [2023-05-09 16:44:13,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 9: [2023-05-09 16:44:13,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 9: [2023-05-09 16:44:13,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 9: [2023-05-09 16:44:13,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +31: [2023-05-09 16:44:13,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 9: [2023-05-09 16:44:13,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 9: [2023-05-09 16:44:13,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 9: [2023-05-09 16:44:13,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +11: [2023-05-09 16:44:13,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +11: [2023-05-09 16:44:13,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +11: [2023-05-09 16:44:13,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +11: [2023-05-09 16:44:13,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +11: [2023-05-09 16:44:13,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +11: [2023-05-09 16:44:13,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +11: [2023-05-09 16:44:13,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +19: [2023-05-09 16:44:13,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +19: [2023-05-09 16:44:13,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +19: [2023-05-09 16:44:13,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +11: [2023-05-09 16:44:13,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +31: [2023-05-09 16:44:13,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 3: [2023-05-09 16:44:13,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +11: [2023-05-09 16:44:13,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +11: [2023-05-09 16:44:13,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +24: [2023-05-09 16:44:13,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +24: [2023-05-09 16:44:13,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +15: [2023-05-09 16:44:13,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 3: [2023-05-09 16:44:13,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +24: [2023-05-09 16:44:13,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +11: [2023-05-09 16:44:13,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +11: [2023-05-09 16:44:13,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +11: [2023-05-09 16:44:13,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +11: [2023-05-09 16:44:13,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +15: [2023-05-09 16:44:13,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +11: [2023-05-09 16:44:13,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +11: [2023-05-09 16:44:13,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +24: [2023-05-09 16:44:13,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 1: [2023-05-09 16:44:13,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +19: [2023-05-09 16:44:13,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +28: [2023-05-09 16:44:13,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +28: [2023-05-09 16:44:13,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +28: [2023-05-09 16:44:13,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +28: [2023-05-09 16:44:13,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +28: [2023-05-09 16:44:13,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +28: [2023-05-09 16:44:13,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +28: [2023-05-09 16:44:13,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +25: [2023-05-09 16:44:13,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +28: [2023-05-09 16:44:13,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 3: [2023-05-09 16:44:13,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +15: [2023-05-09 16:44:13,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 7: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 7: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +25: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 6: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 6: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 8: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +27: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +27: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +27: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +27: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 7: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 7: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 7: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 7: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 7: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 0: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 4: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 6: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 6: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 6: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 6: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 6: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +27: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +27: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 7: [2023-05-09 16:44:13,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +27: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 0: [2023-05-09 16:44:13,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 0: [2023-05-09 16:44:13,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 0: [2023-05-09 16:44:13,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +25: [2023-05-09 16:44:13,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 6: [2023-05-09 16:44:13,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +25: [2023-05-09 16:44:13,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +25: [2023-05-09 16:44:13,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +25: [2023-05-09 16:44:13,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 4: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 8: [2023-05-09 16:44:13,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 8: [2023-05-09 16:44:13,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 8: [2023-05-09 16:44:13,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 8: [2023-05-09 16:44:13,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 8: [2023-05-09 16:44:13,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +27: [2023-05-09 16:44:13,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 0: [2023-05-09 16:44:13,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +19: [2023-05-09 16:44:13,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 8: [2023-05-09 16:44:13,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +19: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 1: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 1: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +19: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +25: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +23: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +23: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 4: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +25: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +15: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 0: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +23: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +23: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +23: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +23: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +23: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 8: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 0: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +23: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 0: [2023-05-09 16:44:13,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 4: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 4: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 4: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 4: [2023-05-09 16:44:13,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +20: [2023-05-09 16:44:13,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 4: [2023-05-09 16:44:13,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +25: [2023-05-09 16:44:13,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +15: [2023-05-09 16:44:13,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 4: [2023-05-09 16:44:13,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 0: [2023-05-09 16:44:13,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 4: [2023-05-09 16:44:13,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +20: [2023-05-09 16:44:13,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +27: [2023-05-09 16:44:13,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +27: [2023-05-09 16:44:13,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 6: [2023-05-09 16:44:13,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 6: [2023-05-09 16:44:13,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 6: [2023-05-09 16:44:13,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +27: [2023-05-09 16:44:13,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +27: [2023-05-09 16:44:13,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 0: [2023-05-09 16:44:13,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +27: [2023-05-09 16:44:13,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 6: [2023-05-09 16:44:13,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 6: [2023-05-09 16:44:13,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 7: [2023-05-09 16:44:13,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 7: [2023-05-09 16:44:13,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 7: [2023-05-09 16:44:13,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +25: [2023-05-09 16:44:13,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 7: [2023-05-09 16:44:13,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 7: [2023-05-09 16:44:13,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 7: [2023-05-09 16:44:13,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 7: [2023-05-09 16:44:13,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 7: [2023-05-09 16:44:13,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +28: [2023-05-09 16:44:13,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +28: [2023-05-09 16:44:13,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +28: [2023-05-09 16:44:13,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +28: [2023-05-09 16:44:13,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 6: [2023-05-09 16:44:13,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 0: [2023-05-09 16:44:13,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +27: [2023-05-09 16:44:13,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +28: [2023-05-09 16:44:13,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +15: [2023-05-09 16:44:13,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +27: [2023-05-09 16:44:13,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 0: [2023-05-09 16:44:13,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 0: [2023-05-09 16:44:13,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +28: [2023-05-09 16:44:13,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +28: [2023-05-09 16:44:13,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +27: [2023-05-09 16:44:13,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 0: [2023-05-09 16:44:13,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 6: [2023-05-09 16:44:13,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 3: [2023-05-09 16:44:13,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +28: [2023-05-09 16:44:13,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 6: [2023-05-09 16:44:13,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 3: [2023-05-09 16:44:13,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 0: [2023-05-09 16:44:13,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 0: [2023-05-09 16:44:13,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +25: [2023-05-09 16:44:13,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 8: [2023-05-09 16:44:13,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 3: [2023-05-09 16:44:13,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 8: [2023-05-09 16:44:13,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 8: [2023-05-09 16:44:13,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +23: [2023-05-09 16:44:13,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 8: [2023-05-09 16:44:13,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +23: [2023-05-09 16:44:13,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +20: [2023-05-09 16:44:13,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 8: [2023-05-09 16:44:13,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +23: [2023-05-09 16:44:13,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +25: [2023-05-09 16:44:13,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +23: [2023-05-09 16:44:13,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 4: [2023-05-09 16:44:13,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 4: [2023-05-09 16:44:13,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 4: [2023-05-09 16:44:13,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 4: [2023-05-09 16:44:13,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 4: [2023-05-09 16:44:13,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... + 8: [2023-05-09 16:44:13,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +25: [2023-05-09 16:44:13,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 8: [2023-05-09 16:44:13,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 4: [2023-05-09 16:44:13,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +25: [2023-05-09 16:44:13,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +23: [2023-05-09 16:44:13,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +23: [2023-05-09 16:44:13,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... + 8: [2023-05-09 16:44:13,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +25: [2023-05-09 16:44:13,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +25: [2023-05-09 16:44:13,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +23: [2023-05-09 16:44:13,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt... +23: [2023-05-09 16:44:13,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt... +15: [2023-05-09 16:44:13,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 3: [2023-05-09 16:44:13,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 9: [2023-05-09 16:44:13,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +20: [2023-05-09 16:44:13,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +15: [2023-05-09 16:44:13,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +20: [2023-05-09 16:44:13,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 9: [2023-05-09 16:44:13,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +20: [2023-05-09 16:44:13,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 3: [2023-05-09 16:44:13,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +20: [2023-05-09 16:44:13,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +20: [2023-05-09 16:44:13,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 9: [2023-05-09 16:44:13,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 9: [2023-05-09 16:44:13,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 9: [2023-05-09 16:44:13,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 9: [2023-05-09 16:44:13,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 1: [2023-05-09 16:44:13,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +11: [2023-05-09 16:44:13,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +11: [2023-05-09 16:44:13,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +11: [2023-05-09 16:44:13,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +11: [2023-05-09 16:44:13,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +25: [2023-05-09 16:44:13,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +27: [2023-05-09 16:44:13,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 4: [2023-05-09 16:44:13,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 4: [2023-05-09 16:44:13,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +27: [2023-05-09 16:44:13,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 0: [2023-05-09 16:44:13,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +28: [2023-05-09 16:44:13,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 7: [2023-05-09 16:44:13,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 7: [2023-05-09 16:44:13,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 6: [2023-05-09 16:44:13,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 6: [2023-05-09 16:44:13,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 6: [2023-05-09 16:44:13,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 6: [2023-05-09 16:44:13,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 7: [2023-05-09 16:44:13,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 8: [2023-05-09 16:44:13,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 8: [2023-05-09 16:44:13,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 7: [2023-05-09 16:44:13,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 0: [2023-05-09 16:44:13,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +27: [2023-05-09 16:44:13,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 0: [2023-05-09 16:44:13,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 4: [2023-05-09 16:44:13,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 4: [2023-05-09 16:44:13,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +27: [2023-05-09 16:44:13,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 0: [2023-05-09 16:44:13,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +28: [2023-05-09 16:44:13,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 8: [2023-05-09 16:44:13,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +25: [2023-05-09 16:44:13,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +28: [2023-05-09 16:44:13,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 8: [2023-05-09 16:44:13,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +28: [2023-05-09 16:44:13,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +25: [2023-05-09 16:44:13,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +25: [2023-05-09 16:44:13,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +23: [2023-05-09 16:44:13,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +23: [2023-05-09 16:44:13,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. +27: [2023-05-09 16:44:13,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +27: [2023-05-09 16:44:13,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 4: [2023-05-09 16:44:13,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +25: [2023-05-09 16:44:13,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 4: [2023-05-09 16:44:13,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +23: [2023-05-09 16:44:13,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 7: [2023-05-09 16:44:13,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 7: [2023-05-09 16:44:13,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 0: [2023-05-09 16:44:13,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 7: [2023-05-09 16:44:13,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 7: [2023-05-09 16:44:13,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +28: [2023-05-09 16:44:13,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +23: [2023-05-09 16:44:13,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt. + 6: [2023-05-09 16:44:13,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 6: [2023-05-09 16:44:13,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +27: [2023-05-09 16:44:13,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 6: [2023-05-09 16:44:13,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 1: [2023-05-09 16:44:13,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 6: [2023-05-09 16:44:13,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 1: [2023-05-09 16:44:13,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 0: [2023-05-09 16:44:13,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 4: [2023-05-09 16:44:13,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 4: [2023-05-09 16:44:13,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +11: [2023-05-09 16:44:13,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +27: [2023-05-09 16:44:13,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +11: [2023-05-09 16:44:13,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +11: [2023-05-09 16:44:13,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 8: [2023-05-09 16:44:13,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 8: [2023-05-09 16:44:13,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 0: [2023-05-09 16:44:13,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +11: [2023-05-09 16:44:13,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 0: [2023-05-09 16:44:13,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 1: [2023-05-09 16:44:13,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 8: [2023-05-09 16:44:13,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +28: [2023-05-09 16:44:13,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +25: [2023-05-09 16:44:13,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +25: [2023-05-09 16:44:13,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +25: [2023-05-09 16:44:13,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 8: [2023-05-09 16:44:13,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +28: [2023-05-09 16:44:13,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +28: [2023-05-09 16:44:13,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +23: [2023-05-09 16:44:13,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +23: [2023-05-09 16:44:13,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +23: [2023-05-09 16:44:13,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +23: [2023-05-09 16:44:13,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +13: [2023-05-09 16:44:13,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +13: [2023-05-09 16:44:13,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +13: [2023-05-09 16:44:13,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +13: [2023-05-09 16:44:13,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +22: [2023-05-09 16:44:13,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +22: [2023-05-09 16:44:13,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +16: [2023-05-09 16:44:13,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +16: [2023-05-09 16:44:13,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +16: [2023-05-09 16:44:13,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +22: [2023-05-09 16:44:13,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +22: [2023-05-09 16:44:13,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +16: [2023-05-09 16:44:13,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +22: [2023-05-09 16:44:13,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +16: [2023-05-09 16:44:13,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +22: [2023-05-09 16:44:13,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +16: [2023-05-09 16:44:13,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +14: [2023-05-09 16:44:13,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +14: [2023-05-09 16:44:13,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +22: [2023-05-09 16:44:13,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +12: [2023-05-09 16:44:13,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +16: [2023-05-09 16:44:13,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +12: [2023-05-09 16:44:13,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +12: [2023-05-09 16:44:13,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +22: [2023-05-09 16:44:13,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +14: [2023-05-09 16:44:13,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +12: [2023-05-09 16:44:13,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +14: [2023-05-09 16:44:13,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +16: [2023-05-09 16:44:13,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +14: [2023-05-09 16:44:13,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +14: [2023-05-09 16:44:13,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +14: [2023-05-09 16:44:13,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +14: [2023-05-09 16:44:13,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +12: [2023-05-09 16:44:13,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +12: [2023-05-09 16:44:13,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +12: [2023-05-09 16:44:13,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +12: [2023-05-09 16:44:13,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +13: [2023-05-09 16:44:13,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +13: [2023-05-09 16:44:13,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +13: [2023-05-09 16:44:13,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +13: [2023-05-09 16:44:13,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +21: [2023-05-09 16:44:13,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +21: [2023-05-09 16:44:13,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +21: [2023-05-09 16:44:13,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +21: [2023-05-09 16:44:13,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +26: [2023-05-09 16:44:13,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +26: [2023-05-09 16:44:13,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +26: [2023-05-09 16:44:13,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +26: [2023-05-09 16:44:13,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +22: [2023-05-09 16:44:13,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +16: [2023-05-09 16:44:13,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +26: [2023-05-09 16:44:13,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +26: [2023-05-09 16:44:13,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +26: [2023-05-09 16:44:13,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +26: [2023-05-09 16:44:13,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +14: [2023-05-09 16:44:13,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +22: [2023-05-09 16:44:13,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +22: [2023-05-09 16:44:13,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +16: [2023-05-09 16:44:13,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +12: [2023-05-09 16:44:13,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +22: [2023-05-09 16:44:13,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +14: [2023-05-09 16:44:13,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +14: [2023-05-09 16:44:13,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +16: [2023-05-09 16:44:13,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +16: [2023-05-09 16:44:13,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +21: [2023-05-09 16:44:13,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +14: [2023-05-09 16:44:13,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +21: [2023-05-09 16:44:13,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +21: [2023-05-09 16:44:13,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +21: [2023-05-09 16:44:13,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +12: [2023-05-09 16:44:13,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +12: [2023-05-09 16:44:13,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +12: [2023-05-09 16:44:13,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +21: [2023-05-09 16:44:13,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +21: [2023-05-09 16:44:13,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +21: [2023-05-09 16:44:13,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +16: [2023-05-09 16:44:13,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +22: [2023-05-09 16:44:13,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +14: [2023-05-09 16:44:13,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +21: [2023-05-09 16:44:13,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +10: [2023-05-09 16:44:13,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +10: [2023-05-09 16:44:13,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +10: [2023-05-09 16:44:13,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +10: [2023-05-09 16:44:13,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +21: [2023-05-09 16:44:13,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +21: [2023-05-09 16:44:13,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +12: [2023-05-09 16:44:13,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +14: [2023-05-09 16:44:13,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +14: [2023-05-09 16:44:13,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +22: [2023-05-09 16:44:13,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +22: [2023-05-09 16:44:13,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +22: [2023-05-09 16:44:13,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +21: [2023-05-09 16:44:13,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +21: [2023-05-09 16:44:13,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +14: [2023-05-09 16:44:13,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +16: [2023-05-09 16:44:13,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +16: [2023-05-09 16:44:13,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +26: [2023-05-09 16:44:13,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +16: [2023-05-09 16:44:13,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +26: [2023-05-09 16:44:13,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +26: [2023-05-09 16:44:13,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +26: [2023-05-09 16:44:13,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +12: [2023-05-09 16:44:13,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +12: [2023-05-09 16:44:13,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +12: [2023-05-09 16:44:13,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +24: [2023-05-09 16:44:13,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +24: [2023-05-09 16:44:13,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +24: [2023-05-09 16:44:13,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +24: [2023-05-09 16:44:13,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +17: [2023-05-09 16:44:13,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +17: [2023-05-09 16:44:13,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +17: [2023-05-09 16:44:13,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +17: [2023-05-09 16:44:13,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +17: [2023-05-09 16:44:13,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +17: [2023-05-09 16:44:13,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +17: [2023-05-09 16:44:13,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +16: [2023-05-09 16:44:13,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +16: [2023-05-09 16:44:13,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +24: [2023-05-09 16:44:13,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +16: [2023-05-09 16:44:13,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +17: [2023-05-09 16:44:13,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 9: [2023-05-09 16:44:13,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +16: [2023-05-09 16:44:13,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +24: [2023-05-09 16:44:13,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +29: [2023-05-09 16:44:13,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +24: [2023-05-09 16:44:13,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +24: [2023-05-09 16:44:13,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +29: [2023-05-09 16:44:13,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +29: [2023-05-09 16:44:13,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +29: [2023-05-09 16:44:13,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 9: [2023-05-09 16:44:13,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +10: [2023-05-09 16:44:13,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 9: [2023-05-09 16:44:13,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +30: [2023-05-09 16:44:13,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +30: [2023-05-09 16:44:13,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +30: [2023-05-09 16:44:13,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +30: [2023-05-09 16:44:13,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +22: [2023-05-09 16:44:13,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +22: [2023-05-09 16:44:13,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +22: [2023-05-09 16:44:13,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +10: [2023-05-09 16:44:13,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +30: [2023-05-09 16:44:13,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +30: [2023-05-09 16:44:13,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +30: [2023-05-09 16:44:13,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +22: [2023-05-09 16:44:13,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +21: [2023-05-09 16:44:13,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +30: [2023-05-09 16:44:13,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +10: [2023-05-09 16:44:13,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +29: [2023-05-09 16:44:13,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +29: [2023-05-09 16:44:13,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +29: [2023-05-09 16:44:13,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +10: [2023-05-09 16:44:13,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +29: [2023-05-09 16:44:13,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +15: [2023-05-09 16:44:13,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +15: [2023-05-09 16:44:13,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +15: [2023-05-09 16:44:13,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +15: [2023-05-09 16:44:13,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +19: [2023-05-09 16:44:13,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +19: [2023-05-09 16:44:13,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +19: [2023-05-09 16:44:13,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +15: [2023-05-09 16:44:13,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +19: [2023-05-09 16:44:13,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +26: [2023-05-09 16:44:13,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +15: [2023-05-09 16:44:13,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +26: [2023-05-09 16:44:13,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +26: [2023-05-09 16:44:13,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +15: [2023-05-09 16:44:13,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +15: [2023-05-09 16:44:13,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 2: [2023-05-09 16:44:13,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 2: [2023-05-09 16:44:13,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 2: [2023-05-09 16:44:13,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 2: [2023-05-09 16:44:13,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 3: [2023-05-09 16:44:13,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 3: [2023-05-09 16:44:13,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 3: [2023-05-09 16:44:13,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +18: [2023-05-09 16:44:13,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 3: [2023-05-09 16:44:13,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 5: [2023-05-09 16:44:13,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 5: [2023-05-09 16:44:13,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 5: [2023-05-09 16:44:13,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 5: [2023-05-09 16:44:13,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +18: [2023-05-09 16:44:13,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +18: [2023-05-09 16:44:13,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +10: [2023-05-09 16:44:13,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +18: [2023-05-09 16:44:13,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +18: [2023-05-09 16:44:13,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +19: [2023-05-09 16:44:13,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +19: [2023-05-09 16:44:13,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +18: [2023-05-09 16:44:13,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +19: [2023-05-09 16:44:13,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +10: [2023-05-09 16:44:13,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +19: [2023-05-09 16:44:13,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +10: [2023-05-09 16:44:13,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +10: [2023-05-09 16:44:13,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +26: [2023-05-09 16:44:13,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +18: [2023-05-09 16:44:13,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +20: [2023-05-09 16:44:13,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +20: [2023-05-09 16:44:13,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +18: [2023-05-09 16:44:13,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +20: [2023-05-09 16:44:13,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +18: [2023-05-09 16:44:13,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +18: [2023-05-09 16:44:13,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +24: [2023-05-09 16:44:13,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +24: [2023-05-09 16:44:13,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +24: [2023-05-09 16:44:13,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +20: [2023-05-09 16:44:13,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +18: [2023-05-09 16:44:13,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +24: [2023-05-09 16:44:13,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +18: [2023-05-09 16:44:13,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 3: [2023-05-09 16:44:13,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +10: [2023-05-09 16:44:13,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +20: [2023-05-09 16:44:13,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +20: [2023-05-09 16:44:13,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 9: [2023-05-09 16:44:13,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 9: [2023-05-09 16:44:13,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 9: [2023-05-09 16:44:13,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +20: [2023-05-09 16:44:13,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 9: [2023-05-09 16:44:13,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 3: [2023-05-09 16:44:13,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 3: [2023-05-09 16:44:13,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +20: [2023-05-09 16:44:13,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +20: [2023-05-09 16:44:13,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 3: [2023-05-09 16:44:13,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +20: [2023-05-09 16:44:13,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +29: [2023-05-09 16:44:13,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +29: [2023-05-09 16:44:13,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +20: [2023-05-09 16:44:13,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +10: [2023-05-09 16:44:13,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +29: [2023-05-09 16:44:13,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +20: [2023-05-09 16:44:13,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +29: [2023-05-09 16:44:13,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +10: [2023-05-09 16:44:13,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +10: [2023-05-09 16:44:13,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 2: [2023-05-09 16:44:13,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 2: [2023-05-09 16:44:13,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 2: [2023-05-09 16:44:13,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 2: [2023-05-09 16:44:13,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +21: [2023-05-09 16:44:13,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +21: [2023-05-09 16:44:13,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +21: [2023-05-09 16:44:13,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 2: [2023-05-09 16:44:13,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +14: [2023-05-09 16:44:13,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 2: [2023-05-09 16:44:13,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 2: [2023-05-09 16:44:13,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 2: [2023-05-09 16:44:13,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +14: [2023-05-09 16:44:13,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +14: [2023-05-09 16:44:13,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +14: [2023-05-09 16:44:13,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +16: [2023-05-09 16:44:13,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +15: [2023-05-09 16:44:13,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +15: [2023-05-09 16:44:13,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +15: [2023-05-09 16:44:13,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +22: [2023-05-09 16:44:13,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +22: [2023-05-09 16:44:13,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +21: [2023-05-09 16:44:13,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +21: [2023-05-09 16:44:13,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +16: [2023-05-09 16:44:13,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +22: [2023-05-09 16:44:13,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +16: [2023-05-09 16:44:13,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +22: [2023-05-09 16:44:13,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +15: [2023-05-09 16:44:13,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +16: [2023-05-09 16:44:13,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +21: [2023-05-09 16:44:13,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +21: [2023-05-09 16:44:13,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +17: [2023-05-09 16:44:13,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +17: [2023-05-09 16:44:13,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +21: [2023-05-09 16:44:13,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +19: [2023-05-09 16:44:13,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +19: [2023-05-09 16:44:13,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 2: [2023-05-09 16:44:13,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +18: [2023-05-09 16:44:13,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +18: [2023-05-09 16:44:13,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 2: [2023-05-09 16:44:13,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 2: [2023-05-09 16:44:13,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +17: [2023-05-09 16:44:13,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +17: [2023-05-09 16:44:13,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +19: [2023-05-09 16:44:13,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +19: [2023-05-09 16:44:13,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 2: [2023-05-09 16:44:13,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +24: [2023-05-09 16:44:13,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +18: [2023-05-09 16:44:13,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +18: [2023-05-09 16:44:13,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 5: [2023-05-09 16:44:13,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +30: [2023-05-09 16:44:13,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +30: [2023-05-09 16:44:13,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +24: [2023-05-09 16:44:13,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +24: [2023-05-09 16:44:13,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +30: [2023-05-09 16:44:13,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +30: [2023-05-09 16:44:13,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 9: [2023-05-09 16:44:13,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 9: [2023-05-09 16:44:13,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 5: [2023-05-09 16:44:13,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +11: [2023-05-09 16:44:13,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +11: [2023-05-09 16:44:13,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +11: [2023-05-09 16:44:13,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +10: [2023-05-09 16:44:13,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 5: [2023-05-09 16:44:13,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 5: [2023-05-09 16:44:13,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +14: [2023-05-09 16:44:13,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +20: [2023-05-09 16:44:13,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +11: [2023-05-09 16:44:13,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +15: [2023-05-09 16:44:13,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +20: [2023-05-09 16:44:13,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 9: [2023-05-09 16:44:13,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +20: [2023-05-09 16:44:13,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +20: [2023-05-09 16:44:13,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +10: [2023-05-09 16:44:13,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 9: [2023-05-09 16:44:13,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +21: [2023-05-09 16:44:13,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +21: [2023-05-09 16:44:13,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +29: [2023-05-09 16:44:13,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +29: [2023-05-09 16:44:13,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 3: [2023-05-09 16:44:13,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +14: [2023-05-09 16:44:13,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +10: [2023-05-09 16:44:13,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +12: [2023-05-09 16:44:13,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +12: [2023-05-09 16:44:13,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +12: [2023-05-09 16:44:13,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +12: [2023-05-09 16:44:13,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 8: [2023-05-09 16:44:13,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 8: [2023-05-09 16:44:13,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 8: [2023-05-09 16:44:13,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 8: [2023-05-09 16:44:13,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +24: [2023-05-09 16:44:13,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +14: [2023-05-09 16:44:13,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +15: [2023-05-09 16:44:13,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +15: [2023-05-09 16:44:13,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +17: [2023-05-09 16:44:13,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +14: [2023-05-09 16:44:13,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +19: [2023-05-09 16:44:13,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +21: [2023-05-09 16:44:13,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 8: [2023-05-09 16:44:13,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +21: [2023-05-09 16:44:13,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +24: [2023-05-09 16:44:13,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 8: [2023-05-09 16:44:13,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 8: [2023-05-09 16:44:13,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 8: [2023-05-09 16:44:13,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +15: [2023-05-09 16:44:13,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +15: [2023-05-09 16:44:13,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +21: [2023-05-09 16:44:13,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +15: [2023-05-09 16:44:13,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +29: [2023-05-09 16:44:13,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +21: [2023-05-09 16:44:13,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +29: [2023-05-09 16:44:13,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +29: [2023-05-09 16:44:13,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +17: [2023-05-09 16:44:13,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +21: [2023-05-09 16:44:13,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +24: [2023-05-09 16:44:13,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +24: [2023-05-09 16:44:13,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +24: [2023-05-09 16:44:13,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +17: [2023-05-09 16:44:13,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +29: [2023-05-09 16:44:13,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +18: [2023-05-09 16:44:13,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +17: [2023-05-09 16:44:13,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +20: [2023-05-09 16:44:13,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 3: [2023-05-09 16:44:13,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +15: [2023-05-09 16:44:13,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 3: [2023-05-09 16:44:13,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +30: [2023-05-09 16:44:13,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +30: [2023-05-09 16:44:13,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 3: [2023-05-09 16:44:13,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +15: [2023-05-09 16:44:13,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +30: [2023-05-09 16:44:13,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +29: [2023-05-09 16:44:13,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +29: [2023-05-09 16:44:13,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +30: [2023-05-09 16:44:13,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +20: [2023-05-09 16:44:13,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 3: [2023-05-09 16:44:13,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 3: [2023-05-09 16:44:13,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 3: [2023-05-09 16:44:13,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +18: [2023-05-09 16:44:13,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +19: [2023-05-09 16:44:13,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +19: [2023-05-09 16:44:13,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +10: [2023-05-09 16:44:13,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 2: [2023-05-09 16:44:13,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +16: [2023-05-09 16:44:13,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +20: [2023-05-09 16:44:13,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 1: [2023-05-09 16:44:13,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 3: [2023-05-09 16:44:13,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 2: [2023-05-09 16:44:13,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +15: [2023-05-09 16:44:13,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 1: [2023-05-09 16:44:13,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +24: [2023-05-09 16:44:13,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +10: [2023-05-09 16:44:13,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 9: [2023-05-09 16:44:13,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 2: [2023-05-09 16:44:13,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 2: [2023-05-09 16:44:13,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +10: [2023-05-09 16:44:13,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +10: [2023-05-09 16:44:13,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +10: [2023-05-09 16:44:13,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +10: [2023-05-09 16:44:13,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +16: [2023-05-09 16:44:13,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +22: [2023-05-09 16:44:13,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +22: [2023-05-09 16:44:13,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 1: [2023-05-09 16:44:13,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +16: [2023-05-09 16:44:13,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +18: [2023-05-09 16:44:13,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +19: [2023-05-09 16:44:13,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +19: [2023-05-09 16:44:13,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +22: [2023-05-09 16:44:13,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +11: [2023-05-09 16:44:13,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +19: [2023-05-09 16:44:13,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +19: [2023-05-09 16:44:13,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +19: [2023-05-09 16:44:13,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +22: [2023-05-09 16:44:13,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +20: [2023-05-09 16:44:13,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +22: [2023-05-09 16:44:13,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +22: [2023-05-09 16:44:13,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +12: [2023-05-09 16:44:13,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +12: [2023-05-09 16:44:13,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +16: [2023-05-09 16:44:13,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +12: [2023-05-09 16:44:13,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +20: [2023-05-09 16:44:13,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +22: [2023-05-09 16:44:13,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +10: [2023-05-09 16:44:13,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +10: [2023-05-09 16:44:13,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +15: [2023-05-09 16:44:13,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +10: [2023-05-09 16:44:13,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +22: [2023-05-09 16:44:13,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +16: [2023-05-09 16:44:13,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +16: [2023-05-09 16:44:13,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +18: [2023-05-09 16:44:13,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +18: [2023-05-09 16:44:13,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 1: [2023-05-09 16:44:13,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +12: [2023-05-09 16:44:13,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +19: [2023-05-09 16:44:13,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 1: [2023-05-09 16:44:13,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +15: [2023-05-09 16:44:13,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 1: [2023-05-09 16:44:13,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +16: [2023-05-09 16:44:13,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +16: [2023-05-09 16:44:13,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +10: [2023-05-09 16:44:13,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +11: [2023-05-09 16:44:13,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +11: [2023-05-09 16:44:13,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 3: [2023-05-09 16:44:13,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 1: [2023-05-09 16:44:13,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +29: [2023-05-09 16:44:13,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 1: [2023-05-09 16:44:13,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 1: [2023-05-09 16:44:13,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 1: [2023-05-09 16:44:13,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +18: [2023-05-09 16:44:13,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +18: [2023-05-09 16:44:13,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 2: [2023-05-09 16:44:13,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +14: [2023-05-09 16:44:13,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +11: [2023-05-09 16:44:13,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 2: [2023-05-09 16:44:13,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +15: [2023-05-09 16:44:13,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +18: [2023-05-09 16:44:13,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 3: [2023-05-09 16:44:13,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 2: [2023-05-09 16:44:13,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 9: [2023-05-09 16:44:13,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +18: [2023-05-09 16:44:13,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +18: [2023-05-09 16:44:13,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +24: [2023-05-09 16:44:13,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +24: [2023-05-09 16:44:13,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 2: [2023-05-09 16:44:13,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 2: [2023-05-09 16:44:13,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +24: [2023-05-09 16:44:13,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +14: [2023-05-09 16:44:13,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +20: [2023-05-09 16:44:13,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +29: [2023-05-09 16:44:13,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 8: [2023-05-09 16:44:13,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 8: [2023-05-09 16:44:13,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 2: [2023-05-09 16:44:13,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +20: [2023-05-09 16:44:13,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +10: [2023-05-09 16:44:13,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 2: [2023-05-09 16:44:13,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 2: [2023-05-09 16:44:13,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 2: [2023-05-09 16:44:13,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 2: [2023-05-09 16:44:13,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 2: [2023-05-09 16:44:13,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 2: [2023-05-09 16:44:13,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 8: [2023-05-09 16:44:13,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 8: [2023-05-09 16:44:13,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +18: [2023-05-09 16:44:13,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 1: [2023-05-09 16:44:13,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 9: [2023-05-09 16:44:13,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 3: [2023-05-09 16:44:13,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 3: [2023-05-09 16:44:13,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +10: [2023-05-09 16:44:13,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 9: [2023-05-09 16:44:13,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 9: [2023-05-09 16:44:13,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 1: [2023-05-09 16:44:13,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 1: [2023-05-09 16:44:13,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +24: [2023-05-09 16:44:13,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +14: [2023-05-09 16:44:13,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +14: [2023-05-09 16:44:13,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +14: [2023-05-09 16:44:13,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +24: [2023-05-09 16:44:13,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 3: [2023-05-09 16:44:13,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 3: [2023-05-09 16:44:13,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 3: [2023-05-09 16:44:13,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +18: [2023-05-09 16:44:13,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 3: [2023-05-09 16:44:13,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +29: [2023-05-09 16:44:13,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +24: [2023-05-09 16:44:13,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +14: [2023-05-09 16:44:13,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +14: [2023-05-09 16:44:13,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +14: [2023-05-09 16:44:13,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +19: [2023-05-09 16:44:13,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +29: [2023-05-09 16:44:13,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +29: [2023-05-09 16:44:13,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +20: [2023-05-09 16:44:13,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 9: [2023-05-09 16:44:13,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +24: [2023-05-09 16:44:13,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +19: [2023-05-09 16:44:13,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +19: [2023-05-09 16:44:13,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +18: [2023-05-09 16:44:13,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +18: [2023-05-09 16:44:13,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 9: [2023-05-09 16:44:13,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 1: [2023-05-09 16:44:13,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +20: [2023-05-09 16:44:13,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +20: [2023-05-09 16:44:13,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +29: [2023-05-09 16:44:13,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +18: [2023-05-09 16:44:13,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +15: [2023-05-09 16:44:13,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +15: [2023-05-09 16:44:13,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +24: [2023-05-09 16:44:13,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +18: [2023-05-09 16:44:13,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +10: [2023-05-09 16:44:13,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +20: [2023-05-09 16:44:13,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +20: [2023-05-09 16:44:13,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +20: [2023-05-09 16:44:13,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +24: [2023-05-09 16:44:13,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +20: [2023-05-09 16:44:13,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +15: [2023-05-09 16:44:13,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +15: [2023-05-09 16:44:13,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +15: [2023-05-09 16:44:13,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +15: [2023-05-09 16:44:13,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +24: [2023-05-09 16:44:13,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +20: [2023-05-09 16:44:13,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +20: [2023-05-09 16:44:13,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +15: [2023-05-09 16:44:13,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +15: [2023-05-09 16:44:13,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 8: [2023-05-09 16:44:13,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +24: [2023-05-09 16:44:13,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +29: [2023-05-09 16:44:13,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 8: [2023-05-09 16:44:13,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 8: [2023-05-09 16:44:13,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 6: [2023-05-09 16:44:13,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 6: [2023-05-09 16:44:13,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 6: [2023-05-09 16:44:13,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 6: [2023-05-09 16:44:13,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +29: [2023-05-09 16:44:13,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 8: [2023-05-09 16:44:13,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +29: [2023-05-09 16:44:13,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +19: [2023-05-09 16:44:13,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +19: [2023-05-09 16:44:13,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 6: [2023-05-09 16:44:13,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 6: [2023-05-09 16:44:13,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 6: [2023-05-09 16:44:13,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 6: [2023-05-09 16:44:13,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +12: [2023-05-09 16:44:13,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +12: [2023-05-09 16:44:13,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +12: [2023-05-09 16:44:13,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +29: [2023-05-09 16:44:13,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +29: [2023-05-09 16:44:13,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +19: [2023-05-09 16:44:13,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +19: [2023-05-09 16:44:13,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +12: [2023-05-09 16:44:13,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +12: [2023-05-09 16:44:13,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +12: [2023-05-09 16:44:13,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +12: [2023-05-09 16:44:13,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +12: [2023-05-09 16:44:13,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +29: [2023-05-09 16:44:13,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 1: [2023-05-09 16:44:13,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +25: [2023-05-09 16:44:13,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +25: [2023-05-09 16:44:13,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +25: [2023-05-09 16:44:13,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +27: [2023-05-09 16:44:13,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +27: [2023-05-09 16:44:13,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +27: [2023-05-09 16:44:13,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +25: [2023-05-09 16:44:13,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +19: [2023-05-09 16:44:13,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +19: [2023-05-09 16:44:13,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 4: [2023-05-09 16:44:13,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 4: [2023-05-09 16:44:13,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +26: [2023-05-09 16:44:13,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +27: [2023-05-09 16:44:13,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 4: [2023-05-09 16:44:13,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +26: [2023-05-09 16:44:13,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +26: [2023-05-09 16:44:13,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +19: [2023-05-09 16:44:13,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +19: [2023-05-09 16:44:13,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 4: [2023-05-09 16:44:13,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +26: [2023-05-09 16:44:13,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 0: [2023-05-09 16:44:13,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 0: [2023-05-09 16:44:13,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 0: [2023-05-09 16:44:13,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 0: [2023-05-09 16:44:13,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 7: [2023-05-09 16:44:13,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 7: [2023-05-09 16:44:13,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 7: [2023-05-09 16:44:13,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 7: [2023-05-09 16:44:13,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 6: [2023-05-09 16:44:13,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 6: [2023-05-09 16:44:13,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 6: [2023-05-09 16:44:13,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 3: [2023-05-09 16:44:13,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 3: [2023-05-09 16:44:13,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 6: [2023-05-09 16:44:13,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +23: [2023-05-09 16:44:13,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +23: [2023-05-09 16:44:13,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 5: [2023-05-09 16:44:13,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 5: [2023-05-09 16:44:13,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 5: [2023-05-09 16:44:13,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 5: [2023-05-09 16:44:13,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 5: [2023-05-09 16:44:13,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +23: [2023-05-09 16:44:13,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +23: [2023-05-09 16:44:13,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 5: [2023-05-09 16:44:13,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 5: [2023-05-09 16:44:13,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +27: [2023-05-09 16:44:13,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +27: [2023-05-09 16:44:13,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +27: [2023-05-09 16:44:13,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 5: [2023-05-09 16:44:13,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +30: [2023-05-09 16:44:13,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +30: [2023-05-09 16:44:13,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +30: [2023-05-09 16:44:13,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 3: [2023-05-09 16:44:13,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 3: [2023-05-09 16:44:13,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 7: [2023-05-09 16:44:13,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 7: [2023-05-09 16:44:13,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 7: [2023-05-09 16:44:13,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +23: [2023-05-09 16:44:13,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +23: [2023-05-09 16:44:13,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +30: [2023-05-09 16:44:13,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +27: [2023-05-09 16:44:13,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 3: [2023-05-09 16:44:13,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 3: [2023-05-09 16:44:13,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 5: [2023-05-09 16:44:13,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +27: [2023-05-09 16:44:13,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +27: [2023-05-09 16:44:13,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 5: [2023-05-09 16:44:13,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 5: [2023-05-09 16:44:13,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +23: [2023-05-09 16:44:13,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +27: [2023-05-09 16:44:13,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 5: [2023-05-09 16:44:13,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 7: [2023-05-09 16:44:13,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 5: [2023-05-09 16:44:13,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 7: [2023-05-09 16:44:13,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 7: [2023-05-09 16:44:13,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 5: [2023-05-09 16:44:13,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 3: [2023-05-09 16:44:13,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 5: [2023-05-09 16:44:13,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 3: [2023-05-09 16:44:13,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 5: [2023-05-09 16:44:13,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 7: [2023-05-09 16:44:13,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +23: [2023-05-09 16:44:13,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 8: [2023-05-09 16:44:13,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 8: [2023-05-09 16:44:13,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 8: [2023-05-09 16:44:13,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 1: [2023-05-09 16:44:13,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +27: [2023-05-09 16:44:13,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +28: [2023-05-09 16:44:13,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +28: [2023-05-09 16:44:13,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +28: [2023-05-09 16:44:13,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +28: [2023-05-09 16:44:13,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +28: [2023-05-09 16:44:13,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 8: [2023-05-09 16:44:13,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 7: [2023-05-09 16:44:13,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 1: [2023-05-09 16:44:13,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +28: [2023-05-09 16:44:13,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +28: [2023-05-09 16:44:13,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +28: [2023-05-09 16:44:13,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 1: [2023-05-09 16:44:13,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +27: [2023-05-09 16:44:13,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +27: [2023-05-09 16:44:13,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +23: [2023-05-09 16:44:13,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +23: [2023-05-09 16:44:13,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +23: [2023-05-09 16:44:13,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 1: [2023-05-09 16:44:13,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 1: [2023-05-09 16:44:13,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +13: [2023-05-09 16:44:13,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +13: [2023-05-09 16:44:13,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +13: [2023-05-09 16:44:13,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +13: [2023-05-09 16:44:13,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +13: [2023-05-09 16:44:13,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +23: [2023-05-09 16:44:13,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +13: [2023-05-09 16:44:13,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +13: [2023-05-09 16:44:13,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +25: [2023-05-09 16:44:13,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 0: [2023-05-09 16:44:13,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +25: [2023-05-09 16:44:13,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +13: [2023-05-09 16:44:13,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +28: [2023-05-09 16:44:13,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +28: [2023-05-09 16:44:13,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +28: [2023-05-09 16:44:13,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +25: [2023-05-09 16:44:13,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +25: [2023-05-09 16:44:13,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +28: [2023-05-09 16:44:13,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 1: [2023-05-09 16:44:13,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 1: [2023-05-09 16:44:13,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 1: [2023-05-09 16:44:13,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +27: [2023-05-09 16:44:13,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +13: [2023-05-09 16:44:13,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 0: [2023-05-09 16:44:13,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +13: [2023-05-09 16:44:13,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +13: [2023-05-09 16:44:13,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +13: [2023-05-09 16:44:13,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +13: [2023-05-09 16:44:13,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +13: [2023-05-09 16:44:13,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 0: [2023-05-09 16:44:13,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 0: [2023-05-09 16:44:13,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 0: [2023-05-09 16:44:13,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 1: [2023-05-09 16:44:13,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +13: [2023-05-09 16:44:13,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +13: [2023-05-09 16:44:13,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +27: [2023-05-09 16:44:13,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +11: [2023-05-09 16:44:13,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +26: [2023-05-09 16:44:13,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +26: [2023-05-09 16:44:13,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +11: [2023-05-09 16:44:13,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +11: [2023-05-09 16:44:13,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +11: [2023-05-09 16:44:13,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +11: [2023-05-09 16:44:13,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +11: [2023-05-09 16:44:13,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +11: [2023-05-09 16:44:13,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +26: [2023-05-09 16:44:13,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 1: [2023-05-09 16:44:13,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +11: [2023-05-09 16:44:13,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +26: [2023-05-09 16:44:13,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 4: [2023-05-09 16:44:13,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 4: [2023-05-09 16:44:13,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 0: [2023-05-09 16:44:13,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 0: [2023-05-09 16:44:13,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 0: [2023-05-09 16:44:13,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 0: [2023-05-09 16:44:13,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 6: [2023-05-09 16:44:13,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +11: [2023-05-09 16:44:13,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 6: [2023-05-09 16:44:13,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 4: [2023-05-09 16:44:13,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 4: [2023-05-09 16:44:13,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 4: [2023-05-09 16:44:13,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +11: [2023-05-09 16:44:13,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 6: [2023-05-09 16:44:13,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 6: [2023-05-09 16:44:13,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 6: [2023-05-09 16:44:13,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 0: [2023-05-09 16:44:13,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 4: [2023-05-09 16:44:13,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 7: [2023-05-09 16:44:13,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 7: [2023-05-09 16:44:13,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 7: [2023-05-09 16:44:13,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 7: [2023-05-09 16:44:13,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 6: [2023-05-09 16:44:13,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 0: [2023-05-09 16:44:13,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +30: [2023-05-09 16:44:13,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 0: [2023-05-09 16:44:13,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +30: [2023-05-09 16:44:13,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +30: [2023-05-09 16:44:13,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 6: [2023-05-09 16:44:13,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +11: [2023-05-09 16:44:13,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +11: [2023-05-09 16:44:13,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +11: [2023-05-09 16:44:13,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +11: [2023-05-09 16:44:13,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +11: [2023-05-09 16:44:13,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 6: [2023-05-09 16:44:13,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +11: [2023-05-09 16:44:13,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +30: [2023-05-09 16:44:13,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 4: [2023-05-09 16:44:13,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 4: [2023-05-09 16:44:13,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 4: [2023-05-09 16:44:13,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 4: [2023-05-09 16:44:13,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 4: [2023-05-09 16:44:13,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 4: [2023-05-09 16:44:13,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 8: [2023-05-09 16:44:13,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 8: [2023-05-09 16:44:13,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 8: [2023-05-09 16:44:13,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 8: [2023-05-09 16:44:13,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +28: [2023-05-09 16:44:13,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +28: [2023-05-09 16:44:13,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 5: [2023-05-09 16:44:13,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +23: [2023-05-09 16:44:14,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +23: [2023-05-09 16:44:14,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +23: [2023-05-09 16:44:14,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +28: [2023-05-09 16:44:14,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +28: [2023-05-09 16:44:14,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +27: [2023-05-09 16:44:14,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +27: [2023-05-09 16:44:14,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +27: [2023-05-09 16:44:14,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +23: [2023-05-09 16:44:14,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +27: [2023-05-09 16:44:14,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +13: [2023-05-09 16:44:14,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 0: [2023-05-09 16:44:14,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +23: [2023-05-09 16:44:14,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +23: [2023-05-09 16:44:14,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +23: [2023-05-09 16:44:14,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +23: [2023-05-09 16:44:14,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +13: [2023-05-09 16:44:14,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +13: [2023-05-09 16:44:14,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +13: [2023-05-09 16:44:14,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +26: [2023-05-09 16:44:14,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +28: [2023-05-09 16:44:14,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +27: [2023-05-09 16:44:14,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +28: [2023-05-09 16:44:14,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +27: [2023-05-09 16:44:14,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +26: [2023-05-09 16:44:14,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +26: [2023-05-09 16:44:14,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +26: [2023-05-09 16:44:14,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +26: [2023-05-09 16:44:14,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 0: [2023-05-09 16:44:14,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +27: [2023-05-09 16:44:14,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +27: [2023-05-09 16:44:14,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +27: [2023-05-09 16:44:14,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +27: [2023-05-09 16:44:14,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +30: [2023-05-09 16:44:14,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +27: [2023-05-09 16:44:14,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +28: [2023-05-09 16:44:14,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +28: [2023-05-09 16:44:14,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +26: [2023-05-09 16:44:14,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +26: [2023-05-09 16:44:14,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +26: [2023-05-09 16:44:14,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +31: [2023-05-09 16:44:14,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +31: [2023-05-09 16:44:14,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +31: [2023-05-09 16:44:14,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 0: [2023-05-09 16:44:14,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 0: [2023-05-09 16:44:14,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +31: [2023-05-09 16:44:14,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +13: [2023-05-09 16:44:14,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +30: [2023-05-09 16:44:14,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +30: [2023-05-09 16:44:14,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +30: [2023-05-09 16:44:14,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +30: [2023-05-09 16:44:14,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +27: [2023-05-09 16:44:14,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +27: [2023-05-09 16:44:14,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +27: [2023-05-09 16:44:14,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +27: [2023-05-09 16:44:14,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +31: [2023-05-09 16:44:14,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +30: [2023-05-09 16:44:14,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 6: [2023-05-09 16:44:14,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 6: [2023-05-09 16:44:14,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 5: [2023-05-09 16:44:14,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +27: [2023-05-09 16:44:14,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +30: [2023-05-09 16:44:14,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +30: [2023-05-09 16:44:14,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +17: [2023-05-09 16:44:14,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +17: [2023-05-09 16:44:14,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +31: [2023-05-09 16:44:14,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +31: [2023-05-09 16:44:14,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +17: [2023-05-09 16:44:14,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +31: [2023-05-09 16:44:14,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +11: [2023-05-09 16:44:14,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +11: [2023-05-09 16:44:14,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +11: [2023-05-09 16:44:14,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +11: [2023-05-09 16:44:14,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +17: [2023-05-09 16:44:14,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 8: [2023-05-09 16:44:14,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +23: [2023-05-09 16:44:14,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +13: [2023-05-09 16:44:14,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 0: [2023-05-09 16:44:14,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +13: [2023-05-09 16:44:14,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +13: [2023-05-09 16:44:14,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 0: [2023-05-09 16:44:14,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 8: [2023-05-09 16:44:14,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 8: [2023-05-09 16:44:14,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +28: [2023-05-09 16:44:14,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 0: [2023-05-09 16:44:14,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 4: [2023-05-09 16:44:14,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 4: [2023-05-09 16:44:14,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 7: [2023-05-09 16:44:14,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 7: [2023-05-09 16:44:14,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +31: [2023-05-09 16:44:14,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +31: [2023-05-09 16:44:14,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 7: [2023-05-09 16:44:14,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +23: [2023-05-09 16:44:14,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +31: [2023-05-09 16:44:14,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. +23: [2023-05-09 16:44:14,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. + 0: [2023-05-09 16:44:14,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +28: [2023-05-09 16:44:14,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +31: [2023-05-09 16:44:14,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt. + 0: [2023-05-09 16:44:14,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 0: [2023-05-09 16:44:14,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +28: [2023-05-09 16:44:14,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 4: [2023-05-09 16:44:14,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +28: [2023-05-09 16:44:14,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +28: [2023-05-09 16:44:14,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 0: [2023-05-09 16:44:14,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... + 4: [2023-05-09 16:44:14,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +23: [2023-05-09 16:44:14,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +23: [2023-05-09 16:44:14,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +23: [2023-05-09 16:44:14,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +28: [2023-05-09 16:44:14,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +23: [2023-05-09 16:44:14,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +23: [2023-05-09 16:44:14,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +28: [2023-05-09 16:44:14,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +28: [2023-05-09 16:44:14,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +28: [2023-05-09 16:44:14,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +23: [2023-05-09 16:44:14,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +28: [2023-05-09 16:44:14,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +28: [2023-05-09 16:44:14,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +28: [2023-05-09 16:44:14,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +17: [2023-05-09 16:44:14,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +23: [2023-05-09 16:44:14,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +11: [2023-05-09 16:44:14,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +23: [2023-05-09 16:44:14,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +11: [2023-05-09 16:44:14,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +17: [2023-05-09 16:44:14,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +17: [2023-05-09 16:44:14,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +11: [2023-05-09 16:44:14,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +11: [2023-05-09 16:44:14,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +23: [2023-05-09 16:44:14,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +31: [2023-05-09 16:44:14,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +17: [2023-05-09 16:44:14,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +31: [2023-05-09 16:44:14,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +31: [2023-05-09 16:44:14,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +31: [2023-05-09 16:44:14,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +31: [2023-05-09 16:44:14,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +31: [2023-05-09 16:44:14,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +31: [2023-05-09 16:44:14,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +31: [2023-05-09 16:44:14,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +25: [2023-05-09 16:44:14,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +25: [2023-05-09 16:44:14,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +25: [2023-05-09 16:44:14,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +25: [2023-05-09 16:44:14,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +25: [2023-05-09 16:44:14,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +25: [2023-05-09 16:44:14,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +25: [2023-05-09 16:44:14,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +25: [2023-05-09 16:44:14,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +25: [2023-05-09 16:44:14,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +25: [2023-05-09 16:44:14,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +25: [2023-05-09 16:44:14,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +25: [2023-05-09 16:44:14,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +25: [2023-05-09 16:44:14,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +25: [2023-05-09 16:44:14,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +25: [2023-05-09 16:44:14,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt... +25: [2023-05-09 16:44:14,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +31: [2023-05-09 16:44:14,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +31: [2023-05-09 16:44:14,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +31: [2023-05-09 16:44:14,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +31: [2023-05-09 16:44:14,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +17: [2023-05-09 16:44:14,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +17: [2023-05-09 16:44:14,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +17: [2023-05-09 16:44:14,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +25: [2023-05-09 16:44:14,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +17: [2023-05-09 16:44:14,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +17: [2023-05-09 16:44:14,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +17: [2023-05-09 16:44:14,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +17: [2023-05-09 16:44:14,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +31: [2023-05-09 16:44:14,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +31: [2023-05-09 16:44:14,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +31: [2023-05-09 16:44:14,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +17: [2023-05-09 16:44:14,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +31: [2023-05-09 16:44:14,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +31: [2023-05-09 16:44:14,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +31: [2023-05-09 16:44:14,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +31: [2023-05-09 16:44:14,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +25: [2023-05-09 16:44:14,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +25: [2023-05-09 16:44:14,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +25: [2023-05-09 16:44:14,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt. +31: [2023-05-09 16:44:14,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt... +25: [2023-05-09 16:44:14,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +25: [2023-05-09 16:44:14,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +25: [2023-05-09 16:44:14,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +25: [2023-05-09 16:44:14,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +22: [2023-05-09 16:44:14,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +22: [2023-05-09 16:44:14,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +22: [2023-05-09 16:44:14,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +22: [2023-05-09 16:44:14,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +22: [2023-05-09 16:44:14,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +22: [2023-05-09 16:44:14,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +22: [2023-05-09 16:44:14,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +22: [2023-05-09 16:44:14,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +21: [2023-05-09 16:44:14,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +21: [2023-05-09 16:44:14,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +21: [2023-05-09 16:44:14,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +21: [2023-05-09 16:44:14,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +16: [2023-05-09 16:44:14,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +16: [2023-05-09 16:44:14,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +16: [2023-05-09 16:44:14,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +16: [2023-05-09 16:44:14,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +12: [2023-05-09 16:44:14,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +12: [2023-05-09 16:44:14,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +12: [2023-05-09 16:44:14,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +12: [2023-05-09 16:44:14,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +16: [2023-05-09 16:44:14,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +16: [2023-05-09 16:44:14,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +16: [2023-05-09 16:44:14,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +21: [2023-05-09 16:44:14,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +16: [2023-05-09 16:44:14,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +21: [2023-05-09 16:44:14,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +21: [2023-05-09 16:44:14,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +21: [2023-05-09 16:44:14,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +12: [2023-05-09 16:44:14,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +12: [2023-05-09 16:44:14,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +12: [2023-05-09 16:44:14,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +12: [2023-05-09 16:44:14,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +14: [2023-05-09 16:44:14,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +14: [2023-05-09 16:44:14,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +14: [2023-05-09 16:44:14,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +14: [2023-05-09 16:44:14,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 9: [2023-05-09 16:44:14,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +14: [2023-05-09 16:44:14,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +14: [2023-05-09 16:44:14,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +14: [2023-05-09 16:44:14,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +14: [2023-05-09 16:44:14,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +14: [2023-05-09 16:44:14,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +14: [2023-05-09 16:44:14,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +14: [2023-05-09 16:44:14,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +14: [2023-05-09 16:44:14,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 9: [2023-05-09 16:44:14,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 9: [2023-05-09 16:44:14,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +22: [2023-05-09 16:44:14,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +22: [2023-05-09 16:44:14,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +22: [2023-05-09 16:44:14,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +22: [2023-05-09 16:44:14,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +16: [2023-05-09 16:44:14,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 9: [2023-05-09 16:44:14,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 9: [2023-05-09 16:44:14,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +16: [2023-05-09 16:44:14,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +16: [2023-05-09 16:44:14,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +16: [2023-05-09 16:44:14,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +22: [2023-05-09 16:44:14,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +14: [2023-05-09 16:44:14,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +14: [2023-05-09 16:44:14,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +14: [2023-05-09 16:44:14,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +14: [2023-05-09 16:44:14,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +21: [2023-05-09 16:44:14,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +22: [2023-05-09 16:44:14,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +16: [2023-05-09 16:44:14,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +22: [2023-05-09 16:44:14,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +22: [2023-05-09 16:44:14,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +12: [2023-05-09 16:44:14,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +21: [2023-05-09 16:44:14,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +21: [2023-05-09 16:44:14,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +21: [2023-05-09 16:44:14,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +16: [2023-05-09 16:44:14,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +16: [2023-05-09 16:44:14,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +16: [2023-05-09 16:44:14,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +12: [2023-05-09 16:44:14,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +12: [2023-05-09 16:44:14,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +12: [2023-05-09 16:44:14,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +14: [2023-05-09 16:44:14,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +14: [2023-05-09 16:44:14,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +14: [2023-05-09 16:44:14,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +21: [2023-05-09 16:44:14,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +12: [2023-05-09 16:44:14,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +21: [2023-05-09 16:44:14,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +21: [2023-05-09 16:44:14,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +21: [2023-05-09 16:44:14,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +14: [2023-05-09 16:44:14,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +10: [2023-05-09 16:44:14,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +14: [2023-05-09 16:44:14,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +14: [2023-05-09 16:44:14,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +10: [2023-05-09 16:44:14,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +10: [2023-05-09 16:44:14,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +10: [2023-05-09 16:44:14,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +12: [2023-05-09 16:44:14,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +14: [2023-05-09 16:44:14,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +12: [2023-05-09 16:44:14,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +12: [2023-05-09 16:44:14,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +14: [2023-05-09 16:44:14,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +29: [2023-05-09 16:44:14,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +29: [2023-05-09 16:44:14,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +14: [2023-05-09 16:44:14,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +29: [2023-05-09 16:44:14,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +14: [2023-05-09 16:44:14,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +29: [2023-05-09 16:44:14,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +14: [2023-05-09 16:44:14,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +14: [2023-05-09 16:44:14,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +14: [2023-05-09 16:44:14,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +10: [2023-05-09 16:44:14,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +14: [2023-05-09 16:44:14,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +14: [2023-05-09 16:44:14,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +14: [2023-05-09 16:44:14,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +10: [2023-05-09 16:44:14,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +10: [2023-05-09 16:44:14,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +10: [2023-05-09 16:44:14,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +29: [2023-05-09 16:44:14,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +24: [2023-05-09 16:44:14,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +24: [2023-05-09 16:44:14,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +24: [2023-05-09 16:44:14,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +24: [2023-05-09 16:44:14,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +29: [2023-05-09 16:44:14,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +29: [2023-05-09 16:44:14,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +29: [2023-05-09 16:44:14,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 1: [2023-05-09 16:44:14,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 1: [2023-05-09 16:44:14,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 1: [2023-05-09 16:44:14,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +24: [2023-05-09 16:44:14,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +24: [2023-05-09 16:44:14,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +24: [2023-05-09 16:44:14,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +24: [2023-05-09 16:44:14,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +13: [2023-05-09 16:44:14,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +13: [2023-05-09 16:44:14,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +13: [2023-05-09 16:44:14,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +13: [2023-05-09 16:44:14,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +22: [2023-05-09 16:44:14,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +22: [2023-05-09 16:44:14,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +22: [2023-05-09 16:44:14,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +22: [2023-05-09 16:44:14,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +26: [2023-05-09 16:44:14,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +26: [2023-05-09 16:44:14,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +26: [2023-05-09 16:44:14,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +11: [2023-05-09 16:44:14,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +11: [2023-05-09 16:44:14,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +26: [2023-05-09 16:44:14,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +11: [2023-05-09 16:44:14,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 2: [2023-05-09 16:44:14,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 2: [2023-05-09 16:44:14,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 3: [2023-05-09 16:44:14,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 3: [2023-05-09 16:44:14,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 3: [2023-05-09 16:44:14,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +21: [2023-05-09 16:44:14,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 3: [2023-05-09 16:44:14,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 2: [2023-05-09 16:44:14,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +11: [2023-05-09 16:44:14,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +15: [2023-05-09 16:44:14,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 2: [2023-05-09 16:44:14,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +21: [2023-05-09 16:44:14,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +15: [2023-05-09 16:44:14,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 0: [2023-05-09 16:44:14,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 0: [2023-05-09 16:44:14,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 0: [2023-05-09 16:44:14,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +15: [2023-05-09 16:44:14,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +21: [2023-05-09 16:44:14,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 5: [2023-05-09 16:44:14,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 5: [2023-05-09 16:44:14,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 5: [2023-05-09 16:44:14,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +15: [2023-05-09 16:44:14,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +21: [2023-05-09 16:44:14,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 5: [2023-05-09 16:44:14,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 0: [2023-05-09 16:44:14,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +16: [2023-05-09 16:44:14,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +16: [2023-05-09 16:44:14,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +16: [2023-05-09 16:44:14,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +16: [2023-05-09 16:44:14,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +18: [2023-05-09 16:44:14,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +18: [2023-05-09 16:44:14,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +18: [2023-05-09 16:44:14,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +18: [2023-05-09 16:44:14,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +28: [2023-05-09 16:44:14,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +28: [2023-05-09 16:44:14,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +28: [2023-05-09 16:44:14,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +28: [2023-05-09 16:44:14,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +20: [2023-05-09 16:44:14,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +20: [2023-05-09 16:44:14,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +20: [2023-05-09 16:44:14,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +19: [2023-05-09 16:44:14,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +19: [2023-05-09 16:44:14,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +19: [2023-05-09 16:44:14,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +20: [2023-05-09 16:44:14,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +19: [2023-05-09 16:44:14,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +12: [2023-05-09 16:44:14,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +12: [2023-05-09 16:44:14,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +12: [2023-05-09 16:44:14,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +12: [2023-05-09 16:44:14,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +23: [2023-05-09 16:44:14,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +23: [2023-05-09 16:44:14,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +23: [2023-05-09 16:44:14,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +23: [2023-05-09 16:44:14,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +13: [2023-05-09 16:44:14,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +13: [2023-05-09 16:44:14,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +13: [2023-05-09 16:44:14,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +13: [2023-05-09 16:44:14,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +26: [2023-05-09 16:44:14,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +22: [2023-05-09 16:44:14,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +22: [2023-05-09 16:44:14,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +26: [2023-05-09 16:44:14,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +22: [2023-05-09 16:44:14,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +22: [2023-05-09 16:44:14,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 2: [2023-05-09 16:44:14,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +26: [2023-05-09 16:44:14,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +26: [2023-05-09 16:44:14,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 2: [2023-05-09 16:44:14,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 2: [2023-05-09 16:44:14,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 2: [2023-05-09 16:44:14,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +16: [2023-05-09 16:44:14,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +28: [2023-05-09 16:44:14,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +17: [2023-05-09 16:44:14,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +17: [2023-05-09 16:44:14,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +17: [2023-05-09 16:44:14,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +16: [2023-05-09 16:44:14,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +30: [2023-05-09 16:44:14,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +30: [2023-05-09 16:44:14,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +30: [2023-05-09 16:44:14,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 3: [2023-05-09 16:44:14,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +30: [2023-05-09 16:44:14,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 0: [2023-05-09 16:44:14,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +16: [2023-05-09 16:44:14,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +28: [2023-05-09 16:44:14,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +11: [2023-05-09 16:44:14,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +18: [2023-05-09 16:44:14,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +19: [2023-05-09 16:44:14,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +19: [2023-05-09 16:44:14,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +18: [2023-05-09 16:44:14,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +18: [2023-05-09 16:44:14,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +18: [2023-05-09 16:44:14,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +28: [2023-05-09 16:44:14,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 2: [2023-05-09 16:44:14,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 2: [2023-05-09 16:44:14,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 2: [2023-05-09 16:44:14,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 2: [2023-05-09 16:44:14,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +28: [2023-05-09 16:44:14,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +16: [2023-05-09 16:44:14,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +26: [2023-05-09 16:44:14,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +26: [2023-05-09 16:44:14,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +26: [2023-05-09 16:44:14,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +26: [2023-05-09 16:44:14,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +19: [2023-05-09 16:44:14,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +19: [2023-05-09 16:44:14,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +17: [2023-05-09 16:44:14,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +11: [2023-05-09 16:44:14,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +20: [2023-05-09 16:44:14,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +20: [2023-05-09 16:44:14,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +17: [2023-05-09 16:44:14,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +20: [2023-05-09 16:44:14,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +17: [2023-05-09 16:44:14,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +17: [2023-05-09 16:44:14,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +20: [2023-05-09 16:44:14,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +15: [2023-05-09 16:44:14,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +15: [2023-05-09 16:44:14,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +11: [2023-05-09 16:44:14,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +21: [2023-05-09 16:44:14,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +21: [2023-05-09 16:44:14,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +23: [2023-05-09 16:44:14,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 2: [2023-05-09 16:44:14,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 2: [2023-05-09 16:44:14,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 2: [2023-05-09 16:44:14,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +26: [2023-05-09 16:44:14,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +23: [2023-05-09 16:44:14,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +26: [2023-05-09 16:44:14,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +26: [2023-05-09 16:44:14,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +18: [2023-05-09 16:44:14,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +21: [2023-05-09 16:44:14,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +26: [2023-05-09 16:44:14,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +18: [2023-05-09 16:44:14,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +18: [2023-05-09 16:44:14,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +18: [2023-05-09 16:44:14,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +21: [2023-05-09 16:44:14,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 3: [2023-05-09 16:44:14,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 3: [2023-05-09 16:44:14,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +11: [2023-05-09 16:44:14,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 7: [2023-05-09 16:44:14,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 7: [2023-05-09 16:44:14,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 7: [2023-05-09 16:44:14,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +20: [2023-05-09 16:44:14,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +18: [2023-05-09 16:44:14,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +18: [2023-05-09 16:44:14,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +30: [2023-05-09 16:44:14,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +30: [2023-05-09 16:44:14,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +18: [2023-05-09 16:44:14,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +23: [2023-05-09 16:44:14,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +17: [2023-05-09 16:44:14,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +20: [2023-05-09 16:44:14,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +18: [2023-05-09 16:44:14,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +20: [2023-05-09 16:44:14,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 2: [2023-05-09 16:44:14,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +15: [2023-05-09 16:44:14,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +20: [2023-05-09 16:44:14,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +30: [2023-05-09 16:44:14,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +15: [2023-05-09 16:44:14,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +30: [2023-05-09 16:44:14,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +30: [2023-05-09 16:44:14,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +12: [2023-05-09 16:44:14,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +30: [2023-05-09 16:44:14,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +30: [2023-05-09 16:44:14,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +12: [2023-05-09 16:44:14,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +12: [2023-05-09 16:44:14,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +23: [2023-05-09 16:44:14,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +12: [2023-05-09 16:44:14,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +30: [2023-05-09 16:44:14,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +20: [2023-05-09 16:44:14,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +20: [2023-05-09 16:44:14,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +20: [2023-05-09 16:44:14,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +20: [2023-05-09 16:44:14,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +30: [2023-05-09 16:44:14,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +30: [2023-05-09 16:44:14,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +30: [2023-05-09 16:44:14,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +30: [2023-05-09 16:44:14,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 0: [2023-05-09 16:44:14,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 0: [2023-05-09 16:44:14,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 0: [2023-05-09 16:44:14,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +22: [2023-05-09 16:44:14,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +22: [2023-05-09 16:44:14,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 0: [2023-05-09 16:44:14,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +26: [2023-05-09 16:44:14,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +26: [2023-05-09 16:44:14,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +26: [2023-05-09 16:44:14,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +16: [2023-05-09 16:44:14,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 2: [2023-05-09 16:44:14,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +22: [2023-05-09 16:44:14,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +22: [2023-05-09 16:44:14,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +22: [2023-05-09 16:44:14,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +26: [2023-05-09 16:44:14,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +26: [2023-05-09 16:44:14,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +26: [2023-05-09 16:44:14,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +22: [2023-05-09 16:44:14,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +16: [2023-05-09 16:44:14,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 2: [2023-05-09 16:44:14,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 2: [2023-05-09 16:44:14,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +22: [2023-05-09 16:44:14,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +22: [2023-05-09 16:44:14,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 2: [2023-05-09 16:44:14,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +26: [2023-05-09 16:44:14,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +26: [2023-05-09 16:44:14,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 2: [2023-05-09 16:44:14,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 2: [2023-05-09 16:44:14,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +18: [2023-05-09 16:44:14,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +18: [2023-05-09 16:44:14,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 2: [2023-05-09 16:44:14,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +27: [2023-05-09 16:44:14,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +27: [2023-05-09 16:44:14,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 6: [2023-05-09 16:44:14,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 6: [2023-05-09 16:44:14,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +16: [2023-05-09 16:44:14,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +16: [2023-05-09 16:44:14,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 2: [2023-05-09 16:44:14,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +18: [2023-05-09 16:44:14,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 2: [2023-05-09 16:44:14,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +16: [2023-05-09 16:44:14,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +18: [2023-05-09 16:44:14,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +25: [2023-05-09 16:44:14,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 2: [2023-05-09 16:44:14,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +25: [2023-05-09 16:44:14,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +25: [2023-05-09 16:44:14,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +25: [2023-05-09 16:44:14,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 3: [2023-05-09 16:44:14,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 3: [2023-05-09 16:44:14,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 3: [2023-05-09 16:44:14,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 3: [2023-05-09 16:44:14,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +16: [2023-05-09 16:44:14,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 4: [2023-05-09 16:44:14,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 4: [2023-05-09 16:44:14,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +26: [2023-05-09 16:44:14,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +26: [2023-05-09 16:44:14,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +27: [2023-05-09 16:44:14,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 4: [2023-05-09 16:44:14,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 3: [2023-05-09 16:44:14,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +27: [2023-05-09 16:44:14,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +16: [2023-05-09 16:44:14,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +26: [2023-05-09 16:44:14,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +31: [2023-05-09 16:44:14,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +26: [2023-05-09 16:44:14,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +17: [2023-05-09 16:44:14,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +21: [2023-05-09 16:44:14,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +21: [2023-05-09 16:44:14,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +31: [2023-05-09 16:44:14,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +31: [2023-05-09 16:44:14,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +31: [2023-05-09 16:44:14,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 2: [2023-05-09 16:44:14,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +16: [2023-05-09 16:44:14,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 3: [2023-05-09 16:44:14,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 3: [2023-05-09 16:44:14,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +21: [2023-05-09 16:44:14,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +21: [2023-05-09 16:44:14,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +18: [2023-05-09 16:44:14,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +18: [2023-05-09 16:44:14,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +18: [2023-05-09 16:44:14,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 3: [2023-05-09 16:44:14,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +18: [2023-05-09 16:44:14,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 2: [2023-05-09 16:44:14,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +18: [2023-05-09 16:44:14,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 3: [2023-05-09 16:44:14,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 8: [2023-05-09 16:44:14,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 8: [2023-05-09 16:44:14,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 0: [2023-05-09 16:44:14,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 0: [2023-05-09 16:44:14,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 0: [2023-05-09 16:44:14,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 0: [2023-05-09 16:44:14,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +20: [2023-05-09 16:44:14,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +17: [2023-05-09 16:44:14,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +21: [2023-05-09 16:44:14,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +21: [2023-05-09 16:44:14,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 8: [2023-05-09 16:44:14,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +18: [2023-05-09 16:44:14,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +18: [2023-05-09 16:44:14,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 8: [2023-05-09 16:44:14,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +21: [2023-05-09 16:44:14,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +17: [2023-05-09 16:44:14,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +21: [2023-05-09 16:44:14,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 0: [2023-05-09 16:44:14,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 0: [2023-05-09 16:44:14,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 0: [2023-05-09 16:44:14,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 0: [2023-05-09 16:44:14,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 3: [2023-05-09 16:44:14,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 3: [2023-05-09 16:44:14,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +12: [2023-05-09 16:44:14,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +12: [2023-05-09 16:44:14,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +30: [2023-05-09 16:44:14,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +30: [2023-05-09 16:44:14,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +30: [2023-05-09 16:44:14,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +30: [2023-05-09 16:44:14,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 3: [2023-05-09 16:44:14,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +20: [2023-05-09 16:44:14,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 2: [2023-05-09 16:44:14,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 2: [2023-05-09 16:44:14,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +26: [2023-05-09 16:44:14,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +26: [2023-05-09 16:44:14,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +20: [2023-05-09 16:44:14,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +20: [2023-05-09 16:44:14,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +20: [2023-05-09 16:44:14,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +27: [2023-05-09 16:44:14,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +17: [2023-05-09 16:44:14,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +27: [2023-05-09 16:44:14,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +12: [2023-05-09 16:44:14,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +18: [2023-05-09 16:44:14,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +30: [2023-05-09 16:44:14,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +12: [2023-05-09 16:44:14,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +30: [2023-05-09 16:44:14,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +25: [2023-05-09 16:44:14,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 2: [2023-05-09 16:44:14,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +12: [2023-05-09 16:44:14,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +12: [2023-05-09 16:44:14,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +26: [2023-05-09 16:44:14,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +18: [2023-05-09 16:44:14,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +18: [2023-05-09 16:44:14,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 2: [2023-05-09 16:44:14,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +26: [2023-05-09 16:44:14,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +25: [2023-05-09 16:44:14,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +30: [2023-05-09 16:44:14,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +30: [2023-05-09 16:44:14,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +18: [2023-05-09 16:44:14,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +25: [2023-05-09 16:44:14,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +25: [2023-05-09 16:44:14,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +30: [2023-05-09 16:44:14,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +27: [2023-05-09 16:44:14,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +27: [2023-05-09 16:44:14,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +30: [2023-05-09 16:44:14,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +20: [2023-05-09 16:44:14,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +20: [2023-05-09 16:44:14,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +12: [2023-05-09 16:44:14,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +12: [2023-05-09 16:44:14,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +17: [2023-05-09 16:44:14,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +20: [2023-05-09 16:44:14,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +30: [2023-05-09 16:44:14,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +30: [2023-05-09 16:44:14,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 8: [2023-05-09 16:44:14,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +20: [2023-05-09 16:44:14,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +20: [2023-05-09 16:44:14,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +20: [2023-05-09 16:44:14,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 8: [2023-05-09 16:44:14,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +17: [2023-05-09 16:44:14,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +17: [2023-05-09 16:44:14,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +31: [2023-05-09 16:44:14,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +17: [2023-05-09 16:44:14,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +17: [2023-05-09 16:44:14,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. +31: [2023-05-09 16:44:14,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +30: [2023-05-09 16:44:14,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +30: [2023-05-09 16:44:14,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +17: [2023-05-09 16:44:14,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt. + 0: [2023-05-09 16:44:14,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 8: [2023-05-09 16:44:14,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +30: [2023-05-09 16:44:14,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +31: [2023-05-09 16:44:14,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +31: [2023-05-09 16:44:14,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +30: [2023-05-09 16:44:14,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +17: [2023-05-09 16:44:14,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +18: [2023-05-09 16:44:14,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 8: [2023-05-09 16:44:14,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +20: [2023-05-09 16:44:14,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +20: [2023-05-09 16:44:14,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +20: [2023-05-09 16:44:14,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +17: [2023-05-09 16:44:14,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +19: [2023-05-09 16:44:14,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +19: [2023-05-09 16:44:14,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 0: [2023-05-09 16:44:14,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +19: [2023-05-09 16:44:14,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +19: [2023-05-09 16:44:14,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +19: [2023-05-09 16:44:14,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +19: [2023-05-09 16:44:14,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +19: [2023-05-09 16:44:14,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +19: [2023-05-09 16:44:14,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +19: [2023-05-09 16:44:14,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +19: [2023-05-09 16:44:14,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +20: [2023-05-09 16:44:14,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +19: [2023-05-09 16:44:14,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +19: [2023-05-09 16:44:14,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +19: [2023-05-09 16:44:14,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +19: [2023-05-09 16:44:14,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +19: [2023-05-09 16:44:14,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +19: [2023-05-09 16:44:14,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 0: [2023-05-09 16:44:14,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 0: [2023-05-09 16:44:14,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +20: [2023-05-09 16:44:14,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +29: [2023-05-09 16:44:14,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +29: [2023-05-09 16:44:14,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +29: [2023-05-09 16:44:14,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +29: [2023-05-09 16:44:14,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +29: [2023-05-09 16:44:14,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +29: [2023-05-09 16:44:14,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +29: [2023-05-09 16:44:14,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +29: [2023-05-09 16:44:14,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +29: [2023-05-09 16:44:14,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 0: [2023-05-09 16:44:14,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +29: [2023-05-09 16:44:14,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +29: [2023-05-09 16:44:14,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 5: [2023-05-09 16:44:14,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 5: [2023-05-09 16:44:14,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 5: [2023-05-09 16:44:14,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +17: [2023-05-09 16:44:14,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +29: [2023-05-09 16:44:14,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +29: [2023-05-09 16:44:14,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +29: [2023-05-09 16:44:14,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +29: [2023-05-09 16:44:14,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +29: [2023-05-09 16:44:14,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 0: [2023-05-09 16:44:14,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +17: [2023-05-09 16:44:14,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +17: [2023-05-09 16:44:14,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +17: [2023-05-09 16:44:14,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 9: [2023-05-09 16:44:14,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 9: [2023-05-09 16:44:14,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 9: [2023-05-09 16:44:14,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 9: [2023-05-09 16:44:14,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 9: [2023-05-09 16:44:14,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 9: [2023-05-09 16:44:14,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 9: [2023-05-09 16:44:14,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 9: [2023-05-09 16:44:14,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +19: [2023-05-09 16:44:14,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +27: [2023-05-09 16:44:14,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +27: [2023-05-09 16:44:14,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +27: [2023-05-09 16:44:14,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +27: [2023-05-09 16:44:14,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +25: [2023-05-09 16:44:14,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +25: [2023-05-09 16:44:14,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +27: [2023-05-09 16:44:14,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +27: [2023-05-09 16:44:14,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +27: [2023-05-09 16:44:14,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +27: [2023-05-09 16:44:14,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +27: [2023-05-09 16:44:14,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +27: [2023-05-09 16:44:14,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +25: [2023-05-09 16:44:14,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +25: [2023-05-09 16:44:14,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +25: [2023-05-09 16:44:14,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +25: [2023-05-09 16:44:14,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +25: [2023-05-09 16:44:14,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +25: [2023-05-09 16:44:14,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +10: [2023-05-09 16:44:14,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +27: [2023-05-09 16:44:14,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +10: [2023-05-09 16:44:14,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +25: [2023-05-09 16:44:14,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +10: [2023-05-09 16:44:14,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +25: [2023-05-09 16:44:14,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +10: [2023-05-09 16:44:14,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +10: [2023-05-09 16:44:14,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +10: [2023-05-09 16:44:14,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +10: [2023-05-09 16:44:14,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +27: [2023-05-09 16:44:14,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +10: [2023-05-09 16:44:14,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +27: [2023-05-09 16:44:14,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +13: [2023-05-09 16:44:14,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +13: [2023-05-09 16:44:14,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +13: [2023-05-09 16:44:14,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +13: [2023-05-09 16:44:14,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +13: [2023-05-09 16:44:14,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +13: [2023-05-09 16:44:14,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +13: [2023-05-09 16:44:14,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +19: [2023-05-09 16:44:14,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +19: [2023-05-09 16:44:14,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +19: [2023-05-09 16:44:14,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +27: [2023-05-09 16:44:14,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +13: [2023-05-09 16:44:14,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +10: [2023-05-09 16:44:14,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +27: [2023-05-09 16:44:14,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +27: [2023-05-09 16:44:14,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +10: [2023-05-09 16:44:14,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +13: [2023-05-09 16:44:14,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +25: [2023-05-09 16:44:14,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +25: [2023-05-09 16:44:14,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +25: [2023-05-09 16:44:14,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +29: [2023-05-09 16:44:14,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +25: [2023-05-09 16:44:14,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +13: [2023-05-09 16:44:14,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +13: [2023-05-09 16:44:14,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +13: [2023-05-09 16:44:14,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +25: [2023-05-09 16:44:14,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +13: [2023-05-09 16:44:14,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +13: [2023-05-09 16:44:14,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +25: [2023-05-09 16:44:14,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +13: [2023-05-09 16:44:14,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +13: [2023-05-09 16:44:14,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +10: [2023-05-09 16:44:14,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +17: [2023-05-09 16:44:14,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +19: [2023-05-09 16:44:14,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +10: [2023-05-09 16:44:14,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +10: [2023-05-09 16:44:14,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +10: [2023-05-09 16:44:14,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +10: [2023-05-09 16:44:14,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +10: [2023-05-09 16:44:14,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +29: [2023-05-09 16:44:14,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +29: [2023-05-09 16:44:14,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +29: [2023-05-09 16:44:14,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +17: [2023-05-09 16:44:14,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 5: [2023-05-09 16:44:14,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +17: [2023-05-09 16:44:14,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +17: [2023-05-09 16:44:14,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +17: [2023-05-09 16:44:14,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +19: [2023-05-09 16:44:14,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +19: [2023-05-09 16:44:14,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +19: [2023-05-09 16:44:14,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +17: [2023-05-09 16:44:14,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 6: [2023-05-09 16:44:14,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +29: [2023-05-09 16:44:14,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +17: [2023-05-09 16:44:14,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +31: [2023-05-09 16:44:14,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +31: [2023-05-09 16:44:14,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +31: [2023-05-09 16:44:14,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +31: [2023-05-09 16:44:14,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +31: [2023-05-09 16:44:14,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +31: [2023-05-09 16:44:14,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +31: [2023-05-09 16:44:14,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +31: [2023-05-09 16:44:14,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +17: [2023-05-09 16:44:14,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 6: [2023-05-09 16:44:14,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 6: [2023-05-09 16:44:14,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 6: [2023-05-09 16:44:14,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 6: [2023-05-09 16:44:14,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 5: [2023-05-09 16:44:14,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +29: [2023-05-09 16:44:14,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 9: [2023-05-09 16:44:14,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +29: [2023-05-09 16:44:14,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +29: [2023-05-09 16:44:14,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 8: [2023-05-09 16:44:14,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +31: [2023-05-09 16:44:14,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +31: [2023-05-09 16:44:14,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +31: [2023-05-09 16:44:14,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +31: [2023-05-09 16:44:14,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 8: [2023-05-09 16:44:14,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +31: [2023-05-09 16:44:14,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +31: [2023-05-09 16:44:14,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +31: [2023-05-09 16:44:14,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +31: [2023-05-09 16:44:14,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 8: [2023-05-09 16:44:14,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 8: [2023-05-09 16:44:14,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 8: [2023-05-09 16:44:14,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 8: [2023-05-09 16:44:14,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 9: [2023-05-09 16:44:14,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 8: [2023-05-09 16:44:14,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 8: [2023-05-09 16:44:14,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 8: [2023-05-09 16:44:14,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +27: [2023-05-09 16:44:14,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +25: [2023-05-09 16:44:14,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +27: [2023-05-09 16:44:14,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +10: [2023-05-09 16:44:14,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +27: [2023-05-09 16:44:14,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +13: [2023-05-09 16:44:14,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +27: [2023-05-09 16:44:14,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +13: [2023-05-09 16:44:14,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +13: [2023-05-09 16:44:14,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +13: [2023-05-09 16:44:14,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +25: [2023-05-09 16:44:14,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +27: [2023-05-09 16:44:14,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +10: [2023-05-09 16:44:14,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +25: [2023-05-09 16:44:14,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +25: [2023-05-09 16:44:14,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +25: [2023-05-09 16:44:14,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +10: [2023-05-09 16:44:14,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +10: [2023-05-09 16:44:14,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 7: [2023-05-09 16:44:14,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 7: [2023-05-09 16:44:14,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 7: [2023-05-09 16:44:14,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 7: [2023-05-09 16:44:14,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +10: [2023-05-09 16:44:14,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +28: [2023-05-09 16:44:14,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +28: [2023-05-09 16:44:14,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +27: [2023-05-09 16:44:14,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +28: [2023-05-09 16:44:14,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +28: [2023-05-09 16:44:14,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +28: [2023-05-09 16:44:14,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +28: [2023-05-09 16:44:14,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +28: [2023-05-09 16:44:14,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +28: [2023-05-09 16:44:14,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +11: [2023-05-09 16:44:14,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +11: [2023-05-09 16:44:14,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +13: [2023-05-09 16:44:14,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +11: [2023-05-09 16:44:14,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +11: [2023-05-09 16:44:14,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +11: [2023-05-09 16:44:14,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +11: [2023-05-09 16:44:14,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +11: [2023-05-09 16:44:14,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +11: [2023-05-09 16:44:14,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +28: [2023-05-09 16:44:14,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +27: [2023-05-09 16:44:14,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +28: [2023-05-09 16:44:14,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +28: [2023-05-09 16:44:14,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +28: [2023-05-09 16:44:14,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +28: [2023-05-09 16:44:14,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +27: [2023-05-09 16:44:14,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +28: [2023-05-09 16:44:14,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 9: [2023-05-09 16:44:14,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +28: [2023-05-09 16:44:14,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +28: [2023-05-09 16:44:14,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 9: [2023-05-09 16:44:14,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +15: [2023-05-09 16:44:14,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +15: [2023-05-09 16:44:14,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +24: [2023-05-09 16:44:14,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +15: [2023-05-09 16:44:14,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +15: [2023-05-09 16:44:14,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +15: [2023-05-09 16:44:14,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +15: [2023-05-09 16:44:14,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +15: [2023-05-09 16:44:14,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +24: [2023-05-09 16:44:14,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +24: [2023-05-09 16:44:14,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +24: [2023-05-09 16:44:14,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +15: [2023-05-09 16:44:14,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +24: [2023-05-09 16:44:14,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +24: [2023-05-09 16:44:14,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +24: [2023-05-09 16:44:14,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +11: [2023-05-09 16:44:14,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 1: [2023-05-09 16:44:14,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +24: [2023-05-09 16:44:14,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 1: [2023-05-09 16:44:14,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 4: [2023-05-09 16:44:14,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 1: [2023-05-09 16:44:14,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 8: [2023-05-09 16:44:14,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +14: [2023-05-09 16:44:14,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +14: [2023-05-09 16:44:14,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +14: [2023-05-09 16:44:14,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +14: [2023-05-09 16:44:14,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +14: [2023-05-09 16:44:14,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +14: [2023-05-09 16:44:14,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +14: [2023-05-09 16:44:14,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +14: [2023-05-09 16:44:14,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +13: [2023-05-09 16:44:14,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +11: [2023-05-09 16:44:14,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 6: [2023-05-09 16:44:14,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +13: [2023-05-09 16:44:14,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +13: [2023-05-09 16:44:14,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +25: [2023-05-09 16:44:14,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +11: [2023-05-09 16:44:14,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +11: [2023-05-09 16:44:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +10: [2023-05-09 16:44:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +15: [2023-05-09 16:44:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +15: [2023-05-09 16:44:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +11: [2023-05-09 16:44:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +11: [2023-05-09 16:44:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +11: [2023-05-09 16:44:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 4: [2023-05-09 16:44:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +25: [2023-05-09 16:44:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +11: [2023-05-09 16:44:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +15: [2023-05-09 16:44:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +25: [2023-05-09 16:44:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +15: [2023-05-09 16:44:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +24: [2023-05-09 16:44:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +24: [2023-05-09 16:44:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +24: [2023-05-09 16:44:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 4: [2023-05-09 16:44:14,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +24: [2023-05-09 16:44:14,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +24: [2023-05-09 16:44:14,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +31: [2023-05-09 16:44:14,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +24: [2023-05-09 16:44:14,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +10: [2023-05-09 16:44:14,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +10: [2023-05-09 16:44:14,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +23: [2023-05-09 16:44:14,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +23: [2023-05-09 16:44:14,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +23: [2023-05-09 16:44:14,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +23: [2023-05-09 16:44:14,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +15: [2023-05-09 16:44:14,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +24: [2023-05-09 16:44:14,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +24: [2023-05-09 16:44:14,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +23: [2023-05-09 16:44:14,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +23: [2023-05-09 16:44:14,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +23: [2023-05-09 16:44:14,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +15: [2023-05-09 16:44:14,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +15: [2023-05-09 16:44:14,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +23: [2023-05-09 16:44:14,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +15: [2023-05-09 16:44:14,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +23: [2023-05-09 16:44:14,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... + 8: [2023-05-09 16:44:14,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +31: [2023-05-09 16:44:14,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +31: [2023-05-09 16:44:14,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +23: [2023-05-09 16:44:14,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +23: [2023-05-09 16:44:14,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +23: [2023-05-09 16:44:14,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +31: [2023-05-09 16:44:14,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +23: [2023-05-09 16:44:14,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +23: [2023-05-09 16:44:14,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt... +23: [2023-05-09 16:44:14,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... +23: [2023-05-09 16:44:14,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt... + 8: [2023-05-09 16:44:14,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 8: [2023-05-09 16:44:14,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +14: [2023-05-09 16:44:14,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +14: [2023-05-09 16:44:14,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +31: [2023-05-09 16:44:14,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +14: [2023-05-09 16:44:14,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +14: [2023-05-09 16:44:14,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 8: [2023-05-09 16:44:14,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +31: [2023-05-09 16:44:14,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +31: [2023-05-09 16:44:14,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 8: [2023-05-09 16:44:14,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +28: [2023-05-09 16:44:14,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +31: [2023-05-09 16:44:14,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +14: [2023-05-09 16:44:14,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +14: [2023-05-09 16:44:14,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +14: [2023-05-09 16:44:14,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +28: [2023-05-09 16:44:14,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +14: [2023-05-09 16:44:14,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +28: [2023-05-09 16:44:14,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +28: [2023-05-09 16:44:14,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +15: [2023-05-09 16:44:14,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +24: [2023-05-09 16:44:14,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +24: [2023-05-09 16:44:14,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +28: [2023-05-09 16:44:14,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +15: [2023-05-09 16:44:14,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +15: [2023-05-09 16:44:14,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +15: [2023-05-09 16:44:14,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +11: [2023-05-09 16:44:14,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +11: [2023-05-09 16:44:14,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +11: [2023-05-09 16:44:14,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +11: [2023-05-09 16:44:14,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +24: [2023-05-09 16:44:14,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +24: [2023-05-09 16:44:14,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +28: [2023-05-09 16:44:14,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +23: [2023-05-09 16:44:14,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +23: [2023-05-09 16:44:14,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +23: [2023-05-09 16:44:14,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +23: [2023-05-09 16:44:14,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt. +28: [2023-05-09 16:44:14,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +28: [2023-05-09 16:44:14,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +24: [2023-05-09 16:44:14,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +15: [2023-05-09 16:44:14,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +24: [2023-05-09 16:44:14,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +15: [2023-05-09 16:44:14,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +11: [2023-05-09 16:44:14,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +23: [2023-05-09 16:44:14,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +24: [2023-05-09 16:44:14,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +15: [2023-05-09 16:44:14,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +15: [2023-05-09 16:44:14,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +11: [2023-05-09 16:44:14,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +11: [2023-05-09 16:44:14,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +11: [2023-05-09 16:44:14,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +23: [2023-05-09 16:44:14,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +24: [2023-05-09 16:44:14,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +23: [2023-05-09 16:44:14,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +23: [2023-05-09 16:44:14,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +12: [2023-05-09 16:44:14,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +12: [2023-05-09 16:44:14,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +12: [2023-05-09 16:44:14,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +12: [2023-05-09 16:44:14,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +12: [2023-05-09 16:44:14,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +12: [2023-05-09 16:44:14,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +12: [2023-05-09 16:44:14,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +12: [2023-05-09 16:44:14,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +12: [2023-05-09 16:44:14,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +12: [2023-05-09 16:44:14,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +12: [2023-05-09 16:44:14,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +12: [2023-05-09 16:44:14,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +22: [2023-05-09 16:44:14,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +22: [2023-05-09 16:44:14,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +22: [2023-05-09 16:44:14,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +22: [2023-05-09 16:44:14,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +22: [2023-05-09 16:44:14,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +22: [2023-05-09 16:44:14,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +22: [2023-05-09 16:44:14,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +22: [2023-05-09 16:44:14,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +12: [2023-05-09 16:44:14,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +12: [2023-05-09 16:44:14,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +12: [2023-05-09 16:44:14,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +22: [2023-05-09 16:44:14,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +22: [2023-05-09 16:44:14,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +12: [2023-05-09 16:44:14,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +16: [2023-05-09 16:44:14,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +16: [2023-05-09 16:44:14,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +16: [2023-05-09 16:44:14,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +16: [2023-05-09 16:44:14,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +16: [2023-05-09 16:44:14,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +16: [2023-05-09 16:44:14,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +16: [2023-05-09 16:44:14,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +16: [2023-05-09 16:44:14,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +14: [2023-05-09 16:44:14,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +14: [2023-05-09 16:44:14,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +14: [2023-05-09 16:44:14,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +14: [2023-05-09 16:44:14,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +22: [2023-05-09 16:44:14,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +22: [2023-05-09 16:44:14,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +22: [2023-05-09 16:44:14,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +14: [2023-05-09 16:44:14,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +14: [2023-05-09 16:44:14,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +14: [2023-05-09 16:44:14,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +22: [2023-05-09 16:44:14,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +14: [2023-05-09 16:44:14,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +16: [2023-05-09 16:44:14,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +16: [2023-05-09 16:44:14,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +22: [2023-05-09 16:44:14,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +22: [2023-05-09 16:44:14,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +16: [2023-05-09 16:44:14,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +16: [2023-05-09 16:44:14,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +21: [2023-05-09 16:44:14,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +21: [2023-05-09 16:44:14,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +21: [2023-05-09 16:44:14,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +21: [2023-05-09 16:44:14,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +21: [2023-05-09 16:44:14,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +21: [2023-05-09 16:44:14,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +21: [2023-05-09 16:44:14,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +21: [2023-05-09 16:44:14,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +14: [2023-05-09 16:44:14,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +14: [2023-05-09 16:44:14,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +16: [2023-05-09 16:44:14,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +16: [2023-05-09 16:44:14,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +14: [2023-05-09 16:44:14,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +14: [2023-05-09 16:44:14,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +16: [2023-05-09 16:44:14,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +16: [2023-05-09 16:44:14,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +21: [2023-05-09 16:44:14,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +14: [2023-05-09 16:44:14,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +14: [2023-05-09 16:44:14,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +14: [2023-05-09 16:44:14,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +14: [2023-05-09 16:44:14,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +21: [2023-05-09 16:44:14,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +22: [2023-05-09 16:44:14,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +22: [2023-05-09 16:44:14,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +22: [2023-05-09 16:44:14,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +26: [2023-05-09 16:44:14,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +22: [2023-05-09 16:44:14,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +26: [2023-05-09 16:44:14,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +26: [2023-05-09 16:44:14,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +26: [2023-05-09 16:44:14,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +21: [2023-05-09 16:44:14,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +21: [2023-05-09 16:44:14,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +22: [2023-05-09 16:44:14,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +22: [2023-05-09 16:44:14,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +26: [2023-05-09 16:44:14,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +22: [2023-05-09 16:44:14,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +22: [2023-05-09 16:44:14,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +26: [2023-05-09 16:44:14,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +21: [2023-05-09 16:44:14,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +26: [2023-05-09 16:44:14,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +26: [2023-05-09 16:44:14,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +21: [2023-05-09 16:44:14,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +21: [2023-05-09 16:44:14,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 0: [2023-05-09 16:44:14,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 0: [2023-05-09 16:44:14,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 0: [2023-05-09 16:44:14,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +16: [2023-05-09 16:44:14,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +16: [2023-05-09 16:44:14,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +16: [2023-05-09 16:44:14,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +16: [2023-05-09 16:44:14,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +18: [2023-05-09 16:44:14,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +18: [2023-05-09 16:44:14,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +18: [2023-05-09 16:44:14,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +18: [2023-05-09 16:44:14,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +21: [2023-05-09 16:44:14,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +22: [2023-05-09 16:44:14,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +16: [2023-05-09 16:44:14,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +22: [2023-05-09 16:44:14,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +16: [2023-05-09 16:44:14,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +16: [2023-05-09 16:44:14,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +22: [2023-05-09 16:44:14,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +22: [2023-05-09 16:44:14,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +16: [2023-05-09 16:44:14,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +22: [2023-05-09 16:44:14,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +22: [2023-05-09 16:44:14,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +22: [2023-05-09 16:44:14,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +22: [2023-05-09 16:44:14,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +18: [2023-05-09 16:44:14,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +18: [2023-05-09 16:44:14,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +18: [2023-05-09 16:44:14,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +18: [2023-05-09 16:44:14,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +16: [2023-05-09 16:44:14,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +16: [2023-05-09 16:44:14,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +16: [2023-05-09 16:44:14,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +16: [2023-05-09 16:44:14,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +16: [2023-05-09 16:44:14,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +16: [2023-05-09 16:44:14,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +16: [2023-05-09 16:44:14,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +16: [2023-05-09 16:44:14,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 8: [2023-05-09 16:44:14,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 8: [2023-05-09 16:44:14,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 8: [2023-05-09 16:44:14,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 8: [2023-05-09 16:44:14,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 8: [2023-05-09 16:44:14,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 8: [2023-05-09 16:44:14,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 8: [2023-05-09 16:44:14,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 8: [2023-05-09 16:44:14,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +17: [2023-05-09 16:44:14,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +17: [2023-05-09 16:44:14,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +17: [2023-05-09 16:44:14,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +17: [2023-05-09 16:44:14,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +28: [2023-05-09 16:44:14,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +28: [2023-05-09 16:44:14,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +28: [2023-05-09 16:44:14,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +28: [2023-05-09 16:44:14,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +25: [2023-05-09 16:44:14,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +25: [2023-05-09 16:44:14,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +25: [2023-05-09 16:44:14,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +25: [2023-05-09 16:44:14,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 6: [2023-05-09 16:44:14,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 6: [2023-05-09 16:44:14,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 6: [2023-05-09 16:44:14,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 2: [2023-05-09 16:44:14,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 2: [2023-05-09 16:44:14,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 6: [2023-05-09 16:44:14,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +12: [2023-05-09 16:44:14,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +12: [2023-05-09 16:44:14,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +12: [2023-05-09 16:44:14,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 2: [2023-05-09 16:44:14,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +12: [2023-05-09 16:44:14,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 2: [2023-05-09 16:44:14,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +17: [2023-05-09 16:44:14,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +17: [2023-05-09 16:44:14,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +17: [2023-05-09 16:44:14,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +25: [2023-05-09 16:44:14,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +17: [2023-05-09 16:44:14,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +28: [2023-05-09 16:44:14,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +25: [2023-05-09 16:44:14,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +28: [2023-05-09 16:44:14,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +28: [2023-05-09 16:44:14,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +25: [2023-05-09 16:44:14,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +25: [2023-05-09 16:44:14,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +28: [2023-05-09 16:44:14,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 2: [2023-05-09 16:44:14,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +12: [2023-05-09 16:44:14,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 2: [2023-05-09 16:44:14,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +12: [2023-05-09 16:44:14,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 2: [2023-05-09 16:44:14,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +12: [2023-05-09 16:44:14,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +12: [2023-05-09 16:44:14,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 2: [2023-05-09 16:44:14,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +30: [2023-05-09 16:44:14,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +30: [2023-05-09 16:44:14,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +30: [2023-05-09 16:44:14,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +30: [2023-05-09 16:44:14,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 3: [2023-05-09 16:44:14,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 3: [2023-05-09 16:44:14,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 3: [2023-05-09 16:44:14,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 3: [2023-05-09 16:44:14,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 3: [2023-05-09 16:44:14,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 3: [2023-05-09 16:44:14,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +30: [2023-05-09 16:44:14,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +30: [2023-05-09 16:44:14,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +30: [2023-05-09 16:44:14,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +30: [2023-05-09 16:44:14,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 3: [2023-05-09 16:44:14,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +12: [2023-05-09 16:44:14,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +12: [2023-05-09 16:44:14,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +12: [2023-05-09 16:44:14,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +12: [2023-05-09 16:44:14,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 3: [2023-05-09 16:44:14,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +26: [2023-05-09 16:44:14,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 3: [2023-05-09 16:44:14,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +26: [2023-05-09 16:44:14,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +26: [2023-05-09 16:44:14,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +26: [2023-05-09 16:44:14,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +26: [2023-05-09 16:44:14,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +26: [2023-05-09 16:44:14,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +26: [2023-05-09 16:44:14,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +26: [2023-05-09 16:44:14,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +26: [2023-05-09 16:44:14,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +26: [2023-05-09 16:44:14,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +26: [2023-05-09 16:44:14,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +26: [2023-05-09 16:44:14,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +26: [2023-05-09 16:44:14,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +26: [2023-05-09 16:44:14,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +26: [2023-05-09 16:44:14,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +10: [2023-05-09 16:44:14,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +10: [2023-05-09 16:44:14,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +10: [2023-05-09 16:44:14,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +10: [2023-05-09 16:44:14,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +10: [2023-05-09 16:44:14,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +10: [2023-05-09 16:44:14,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +10: [2023-05-09 16:44:14,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +10: [2023-05-09 16:44:14,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +26: [2023-05-09 16:44:14,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +31: [2023-05-09 16:44:14,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +31: [2023-05-09 16:44:14,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +31: [2023-05-09 16:44:14,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +31: [2023-05-09 16:44:14,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 8: [2023-05-09 16:44:14,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 8: [2023-05-09 16:44:14,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 8: [2023-05-09 16:44:14,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 8: [2023-05-09 16:44:14,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 8: [2023-05-09 16:44:14,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +12: [2023-05-09 16:44:14,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +12: [2023-05-09 16:44:14,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +12: [2023-05-09 16:44:14,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +12: [2023-05-09 16:44:14,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 3: [2023-05-09 16:44:14,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +10: [2023-05-09 16:44:14,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 6: [2023-05-09 16:44:14,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 6: [2023-05-09 16:44:14,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 6: [2023-05-09 16:44:14,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 7: [2023-05-09 16:44:14,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 7: [2023-05-09 16:44:14,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 7: [2023-05-09 16:44:14,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 6: [2023-05-09 16:44:14,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +31: [2023-05-09 16:44:14,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +31: [2023-05-09 16:44:14,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +31: [2023-05-09 16:44:14,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +31: [2023-05-09 16:44:14,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +26: [2023-05-09 16:44:14,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +26: [2023-05-09 16:44:14,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +26: [2023-05-09 16:44:14,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 4: [2023-05-09 16:44:14,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 4: [2023-05-09 16:44:14,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 4: [2023-05-09 16:44:14,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 3: [2023-05-09 16:44:14,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +10: [2023-05-09 16:44:14,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +10: [2023-05-09 16:44:14,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +10: [2023-05-09 16:44:14,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 9: [2023-05-09 16:44:14,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 9: [2023-05-09 16:44:14,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 9: [2023-05-09 16:44:14,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 5: [2023-05-09 16:44:14,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 3: [2023-05-09 16:44:14,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 5: [2023-05-09 16:44:14,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 3: [2023-05-09 16:44:14,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +10: [2023-05-09 16:44:14,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 3: [2023-05-09 16:44:14,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +26: [2023-05-09 16:44:14,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 3: [2023-05-09 16:44:14,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 3: [2023-05-09 16:44:14,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 3: [2023-05-09 16:44:14,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +26: [2023-05-09 16:44:14,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 0: [2023-05-09 16:44:14,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 1: [2023-05-09 16:44:14,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 1: [2023-05-09 16:44:14,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 0: [2023-05-09 16:44:14,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 0: [2023-05-09 16:44:14,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 0: [2023-05-09 16:44:14,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 0: [2023-05-09 16:44:14,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 0: [2023-05-09 16:44:14,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 0: [2023-05-09 16:44:14,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 0: [2023-05-09 16:44:14,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +26: [2023-05-09 16:44:14,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 0: [2023-05-09 16:44:14,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 0: [2023-05-09 16:44:14,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 0: [2023-05-09 16:44:14,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +26: [2023-05-09 16:44:14,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +11: [2023-05-09 16:44:14,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +11: [2023-05-09 16:44:14,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 4: [2023-05-09 16:44:14,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +11: [2023-05-09 16:44:14,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +11: [2023-05-09 16:44:14,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 4: [2023-05-09 16:44:14,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +23: [2023-05-09 16:44:14,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +23: [2023-05-09 16:44:14,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +23: [2023-05-09 16:44:14,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +23: [2023-05-09 16:44:14,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +26: [2023-05-09 16:44:14,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +15: [2023-05-09 16:44:14,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +15: [2023-05-09 16:44:14,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 9: [2023-05-09 16:44:14,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +15: [2023-05-09 16:44:14,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +15: [2023-05-09 16:44:14,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +15: [2023-05-09 16:44:14,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +15: [2023-05-09 16:44:14,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +15: [2023-05-09 16:44:14,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 3: [2023-05-09 16:44:14,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 3: [2023-05-09 16:44:14,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +10: [2023-05-09 16:44:14,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +10: [2023-05-09 16:44:14,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +10: [2023-05-09 16:44:14,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +24: [2023-05-09 16:44:14,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 3: [2023-05-09 16:44:14,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +24: [2023-05-09 16:44:14,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +24: [2023-05-09 16:44:14,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +24: [2023-05-09 16:44:14,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +24: [2023-05-09 16:44:14,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +24: [2023-05-09 16:44:14,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 3: [2023-05-09 16:44:14,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +23: [2023-05-09 16:44:14,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +23: [2023-05-09 16:44:14,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +23: [2023-05-09 16:44:14,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +23: [2023-05-09 16:44:14,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +25: [2023-05-09 16:44:14,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +24: [2023-05-09 16:44:14,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +25: [2023-05-09 16:44:14,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +24: [2023-05-09 16:44:14,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 9: [2023-05-09 16:44:14,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 9: [2023-05-09 16:44:14,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +15: [2023-05-09 16:44:14,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 9: [2023-05-09 16:44:14,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +25: [2023-05-09 16:44:14,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +25: [2023-05-09 16:44:14,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 9: [2023-05-09 16:44:14,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +30: [2023-05-09 16:44:14,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +30: [2023-05-09 16:44:14,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +30: [2023-05-09 16:44:14,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +30: [2023-05-09 16:44:14,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +30: [2023-05-09 16:44:14,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +30: [2023-05-09 16:44:14,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +30: [2023-05-09 16:44:14,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +30: [2023-05-09 16:44:14,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +30: [2023-05-09 16:44:14,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +30: [2023-05-09 16:44:14,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +30: [2023-05-09 16:44:14,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +30: [2023-05-09 16:44:14,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +30: [2023-05-09 16:44:14,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +30: [2023-05-09 16:44:14,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +30: [2023-05-09 16:44:14,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +30: [2023-05-09 16:44:14,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +13: [2023-05-09 16:44:14,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +13: [2023-05-09 16:44:14,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +11: [2023-05-09 16:44:14,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +13: [2023-05-09 16:44:14,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +11: [2023-05-09 16:44:14,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +13: [2023-05-09 16:44:14,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +11: [2023-05-09 16:44:14,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +25: [2023-05-09 16:44:14,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +25: [2023-05-09 16:44:14,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +25: [2023-05-09 16:44:14,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +25: [2023-05-09 16:44:14,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +25: [2023-05-09 16:44:14,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +25: [2023-05-09 16:44:14,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +25: [2023-05-09 16:44:14,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +10: [2023-05-09 16:44:14,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +10: [2023-05-09 16:44:14,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +10: [2023-05-09 16:44:14,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +25: [2023-05-09 16:44:14,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +10: [2023-05-09 16:44:14,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +19: [2023-05-09 16:44:14,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +19: [2023-05-09 16:44:14,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 4: [2023-05-09 16:44:14,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +19: [2023-05-09 16:44:14,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +19: [2023-05-09 16:44:14,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 6: [2023-05-09 16:44:14,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 9: [2023-05-09 16:44:14,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +19: [2023-05-09 16:44:14,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +19: [2023-05-09 16:44:14,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +19: [2023-05-09 16:44:14,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +19: [2023-05-09 16:44:14,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +11: [2023-05-09 16:44:14,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +23: [2023-05-09 16:44:14,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +29: [2023-05-09 16:44:14,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +23: [2023-05-09 16:44:14,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +25: [2023-05-09 16:44:14,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +29: [2023-05-09 16:44:14,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +29: [2023-05-09 16:44:14,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +29: [2023-05-09 16:44:14,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 8: [2023-05-09 16:44:14,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 8: [2023-05-09 16:44:14,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 8: [2023-05-09 16:44:14,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 8: [2023-05-09 16:44:14,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 8: [2023-05-09 16:44:14,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 8: [2023-05-09 16:44:14,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 8: [2023-05-09 16:44:14,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 8: [2023-05-09 16:44:14,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +27: [2023-05-09 16:44:14,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +27: [2023-05-09 16:44:14,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +23: [2023-05-09 16:44:14,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +25: [2023-05-09 16:44:14,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +27: [2023-05-09 16:44:14,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +25: [2023-05-09 16:44:14,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 6: [2023-05-09 16:44:14,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +27: [2023-05-09 16:44:14,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +15: [2023-05-09 16:44:14,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +15: [2023-05-09 16:44:14,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +15: [2023-05-09 16:44:14,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +25: [2023-05-09 16:44:14,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +15: [2023-05-09 16:44:14,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +23: [2023-05-09 16:44:14,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 4: [2023-05-09 16:44:14,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 9: [2023-05-09 16:44:14,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +19: [2023-05-09 16:44:14,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +19: [2023-05-09 16:44:14,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +25: [2023-05-09 16:44:14,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +20: [2023-05-09 16:44:14,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +20: [2023-05-09 16:44:14,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 9: [2023-05-09 16:44:14,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +20: [2023-05-09 16:44:14,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +20: [2023-05-09 16:44:14,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +24: [2023-05-09 16:44:14,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +24: [2023-05-09 16:44:14,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +24: [2023-05-09 16:44:14,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +15: [2023-05-09 16:44:14,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +23: [2023-05-09 16:44:14,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +23: [2023-05-09 16:44:14,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +13: [2023-05-09 16:44:14,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +13: [2023-05-09 16:44:14,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +13: [2023-05-09 16:44:14,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +24: [2023-05-09 16:44:14,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +30: [2023-05-09 16:44:14,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +30: [2023-05-09 16:44:14,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 0: [2023-05-09 16:44:14,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +13: [2023-05-09 16:44:14,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +17: [2023-05-09 16:44:14,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +17: [2023-05-09 16:44:14,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +17: [2023-05-09 16:44:14,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +17: [2023-05-09 16:44:14,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +17: [2023-05-09 16:44:14,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +17: [2023-05-09 16:44:14,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +17: [2023-05-09 16:44:14,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +17: [2023-05-09 16:44:14,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +17: [2023-05-09 16:44:14,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +17: [2023-05-09 16:44:14,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +15: [2023-05-09 16:44:14,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +10: [2023-05-09 16:44:14,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +24: [2023-05-09 16:44:14,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +10: [2023-05-09 16:44:14,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +10: [2023-05-09 16:44:14,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +23: [2023-05-09 16:44:14,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +19: [2023-05-09 16:44:14,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +23: [2023-05-09 16:44:14,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +15: [2023-05-09 16:44:14,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +19: [2023-05-09 16:44:14,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +19: [2023-05-09 16:44:14,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +21: [2023-05-09 16:44:14,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +21: [2023-05-09 16:44:14,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +21: [2023-05-09 16:44:14,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +21: [2023-05-09 16:44:14,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +30: [2023-05-09 16:44:14,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +30: [2023-05-09 16:44:14,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +10: [2023-05-09 16:44:14,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +19: [2023-05-09 16:44:14,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +21: [2023-05-09 16:44:14,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +29: [2023-05-09 16:44:14,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +21: [2023-05-09 16:44:14,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +29: [2023-05-09 16:44:14,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +21: [2023-05-09 16:44:14,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. + 1: [2023-05-09 16:44:14,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +21: [2023-05-09 16:44:14,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +17: [2023-05-09 16:44:14,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +17: [2023-05-09 16:44:14,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +17: [2023-05-09 16:44:14,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 1: [2023-05-09 16:44:14,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +17: [2023-05-09 16:44:14,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +23: [2023-05-09 16:44:14,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 9: [2023-05-09 16:44:14,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +17: [2023-05-09 16:44:14,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +21: [2023-05-09 16:44:14,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +27: [2023-05-09 16:44:14,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +27: [2023-05-09 16:44:14,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 9: [2023-05-09 16:44:14,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +21: [2023-05-09 16:44:14,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt. +29: [2023-05-09 16:44:14,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +19: [2023-05-09 16:44:14,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +19: [2023-05-09 16:44:14,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +27: [2023-05-09 16:44:14,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +19: [2023-05-09 16:44:14,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +27: [2023-05-09 16:44:14,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +23: [2023-05-09 16:44:14,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +29: [2023-05-09 16:44:14,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +15: [2023-05-09 16:44:14,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +23: [2023-05-09 16:44:14,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +21: [2023-05-09 16:44:14,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +15: [2023-05-09 16:44:14,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +25: [2023-05-09 16:44:14,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +17: [2023-05-09 16:44:14,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +23: [2023-05-09 16:44:14,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 9: [2023-05-09 16:44:14,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +15: [2023-05-09 16:44:14,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +21: [2023-05-09 16:44:14,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 9: [2023-05-09 16:44:14,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +15: [2023-05-09 16:44:14,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +30: [2023-05-09 16:44:14,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +24: [2023-05-09 16:44:14,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +24: [2023-05-09 16:44:14,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +24: [2023-05-09 16:44:14,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +24: [2023-05-09 16:44:14,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 4: [2023-05-09 16:44:14,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +30: [2023-05-09 16:44:14,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +25: [2023-05-09 16:44:14,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +30: [2023-05-09 16:44:14,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +15: [2023-05-09 16:44:14,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +20: [2023-05-09 16:44:14,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +25: [2023-05-09 16:44:14,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +13: [2023-05-09 16:44:14,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +13: [2023-05-09 16:44:14,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +13: [2023-05-09 16:44:14,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +13: [2023-05-09 16:44:14,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 0: [2023-05-09 16:44:14,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +25: [2023-05-09 16:44:14,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +24: [2023-05-09 16:44:14,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +13: [2023-05-09 16:44:14,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +13: [2023-05-09 16:44:14,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +13: [2023-05-09 16:44:14,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +17: [2023-05-09 16:44:14,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +24: [2023-05-09 16:44:14,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 1: [2023-05-09 16:44:14,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 4: [2023-05-09 16:44:14,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +24: [2023-05-09 16:44:14,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +17: [2023-05-09 16:44:14,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +20: [2023-05-09 16:44:14,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +20: [2023-05-09 16:44:14,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +10: [2023-05-09 16:44:14,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +30: [2023-05-09 16:44:14,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +20: [2023-05-09 16:44:14,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +15: [2023-05-09 16:44:14,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +24: [2023-05-09 16:44:14,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +13: [2023-05-09 16:44:14,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +13: [2023-05-09 16:44:14,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +10: [2023-05-09 16:44:14,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +13: [2023-05-09 16:44:14,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +21: [2023-05-09 16:44:14,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +13: [2023-05-09 16:44:14,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +13: [2023-05-09 16:44:14,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +21: [2023-05-09 16:44:14,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +23: [2023-05-09 16:44:14,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +23: [2023-05-09 16:44:14,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +23: [2023-05-09 16:44:14,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +19: [2023-05-09 16:44:14,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +15: [2023-05-09 16:44:14,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +23: [2023-05-09 16:44:14,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +14: [2023-05-09 16:44:14,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +14: [2023-05-09 16:44:14,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +21: [2023-05-09 16:44:14,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +21: [2023-05-09 16:44:14,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +17: [2023-05-09 16:44:14,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +13: [2023-05-09 16:44:14,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +13: [2023-05-09 16:44:14,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +13: [2023-05-09 16:44:14,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +13: [2023-05-09 16:44:14,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +15: [2023-05-09 16:44:14,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 9: [2023-05-09 16:44:14,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +14: [2023-05-09 16:44:14,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +22: [2023-05-09 16:44:14,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +22: [2023-05-09 16:44:14,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +10: [2023-05-09 16:44:14,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +10: [2023-05-09 16:44:14,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +10: [2023-05-09 16:44:14,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +10: [2023-05-09 16:44:14,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 4: [2023-05-09 16:44:14,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +23: [2023-05-09 16:44:14,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +14: [2023-05-09 16:44:14,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +25: [2023-05-09 16:44:14,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +23: [2023-05-09 16:44:14,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +15: [2023-05-09 16:44:14,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +12: [2023-05-09 16:44:14,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +12: [2023-05-09 16:44:14,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +12: [2023-05-09 16:44:14,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +23: [2023-05-09 16:44:14,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +23: [2023-05-09 16:44:14,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 1: [2023-05-09 16:44:14,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 1: [2023-05-09 16:44:14,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +25: [2023-05-09 16:44:14,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +25: [2023-05-09 16:44:14,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +23: [2023-05-09 16:44:14,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +15: [2023-05-09 16:44:14,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +23: [2023-05-09 16:44:14,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +12: [2023-05-09 16:44:14,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +22: [2023-05-09 16:44:14,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +10: [2023-05-09 16:44:14,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +19: [2023-05-09 16:44:14,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +14: [2023-05-09 16:44:14,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +14: [2023-05-09 16:44:14,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +10: [2023-05-09 16:44:14,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +19: [2023-05-09 16:44:14,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +17: [2023-05-09 16:44:14,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +19: [2023-05-09 16:44:14,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +19: [2023-05-09 16:44:14,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +24: [2023-05-09 16:44:14,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +13: [2023-05-09 16:44:14,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +22: [2023-05-09 16:44:14,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +22: [2023-05-09 16:44:14,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +12: [2023-05-09 16:44:14,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +12: [2023-05-09 16:44:14,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +23: [2023-05-09 16:44:14,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +15: [2023-05-09 16:44:14,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +14: [2023-05-09 16:44:14,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +24: [2023-05-09 16:44:14,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +23: [2023-05-09 16:44:14,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 9: [2023-05-09 16:44:14,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +14: [2023-05-09 16:44:14,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +24: [2023-05-09 16:44:14,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +19: [2023-05-09 16:44:14,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +22: [2023-05-09 16:44:14,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +12: [2023-05-09 16:44:14,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +22: [2023-05-09 16:44:14,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +22: [2023-05-09 16:44:14,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +12: [2023-05-09 16:44:14,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 4: [2023-05-09 16:44:14,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +24: [2023-05-09 16:44:14,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +15: [2023-05-09 16:44:14,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +24: [2023-05-09 16:44:14,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +17: [2023-05-09 16:44:14,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +14: [2023-05-09 16:44:14,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +15: [2023-05-09 16:44:14,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 4: [2023-05-09 16:44:14,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +21: [2023-05-09 16:44:14,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +21: [2023-05-09 16:44:14,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +21: [2023-05-09 16:44:14,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +22: [2023-05-09 16:44:14,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +21: [2023-05-09 16:44:14,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +16: [2023-05-09 16:44:14,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +13: [2023-05-09 16:44:14,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +13: [2023-05-09 16:44:14,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +16: [2023-05-09 16:44:14,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +16: [2023-05-09 16:44:14,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 4: [2023-05-09 16:44:14,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +16: [2023-05-09 16:44:14,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +17: [2023-05-09 16:44:14,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +19: [2023-05-09 16:44:14,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +17: [2023-05-09 16:44:14,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +19: [2023-05-09 16:44:14,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +19: [2023-05-09 16:44:14,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +15: [2023-05-09 16:44:14,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +15: [2023-05-09 16:44:14,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +19: [2023-05-09 16:44:14,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +19: [2023-05-09 16:44:14,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +15: [2023-05-09 16:44:14,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +13: [2023-05-09 16:44:14,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +15: [2023-05-09 16:44:14,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +24: [2023-05-09 16:44:14,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +24: [2023-05-09 16:44:14,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +16: [2023-05-09 16:44:14,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +16: [2023-05-09 16:44:14,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +24: [2023-05-09 16:44:14,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +16: [2023-05-09 16:44:14,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +16: [2023-05-09 16:44:14,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 1: [2023-05-09 16:44:14,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +24: [2023-05-09 16:44:14,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +21: [2023-05-09 16:44:14,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +19: [2023-05-09 16:44:14,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +21: [2023-05-09 16:44:14,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +21: [2023-05-09 16:44:14,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +24: [2023-05-09 16:44:14,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +24: [2023-05-09 16:44:14,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +12: [2023-05-09 16:44:14,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +29: [2023-05-09 16:44:14,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +29: [2023-05-09 16:44:14,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +29: [2023-05-09 16:44:14,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +29: [2023-05-09 16:44:14,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +29: [2023-05-09 16:44:14,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +22: [2023-05-09 16:44:14,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +29: [2023-05-09 16:44:14,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +14: [2023-05-09 16:44:14,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +28: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +28: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +28: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +28: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +28: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +28: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +28: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +29: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +28: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +28: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +28: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +28: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +28: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +29: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +29: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +28: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +28: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +29: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +28: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +29: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +29: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +29: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +19: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +29: [2023-05-09 16:44:14,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +29: [2023-05-09 16:44:14,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +13: [2023-05-09 16:44:14,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +21: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +21: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +17: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +29: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 2: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 2: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 2: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 2: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +28: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 2: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 2: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 2: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 2: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 2: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 2: [2023-05-09 16:44:14,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 2: [2023-05-09 16:44:14,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 2: [2023-05-09 16:44:14,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 2: [2023-05-09 16:44:14,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 2: [2023-05-09 16:44:14,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 2: [2023-05-09 16:44:14,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 7: [2023-05-09 16:44:14,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 7: [2023-05-09 16:44:14,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 7: [2023-05-09 16:44:14,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 7: [2023-05-09 16:44:14,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +13: [2023-05-09 16:44:14,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +19: [2023-05-09 16:44:14,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 5: [2023-05-09 16:44:14,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 5: [2023-05-09 16:44:14,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 5: [2023-05-09 16:44:14,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... + 2: [2023-05-09 16:44:14,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +18: [2023-05-09 16:44:14,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +18: [2023-05-09 16:44:14,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +18: [2023-05-09 16:44:14,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +18: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +18: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +18: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +18: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +18: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +18: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +18: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +18: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +18: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +27: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +27: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +18: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +18: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +27: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +27: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +18: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +27: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +27: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +27: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +27: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +27: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +27: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +19: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +27: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 9: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +27: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +27: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +27: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +27: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +27: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +14: [2023-05-09 16:44:14,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +20: [2023-05-09 16:44:14,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +20: [2023-05-09 16:44:14,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +20: [2023-05-09 16:44:14,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +20: [2023-05-09 16:44:14,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +20: [2023-05-09 16:44:14,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +21: [2023-05-09 16:44:14,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +13: [2023-05-09 16:44:14,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +20: [2023-05-09 16:44:14,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +20: [2023-05-09 16:44:14,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +20: [2023-05-09 16:44:14,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +20: [2023-05-09 16:44:14,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +20: [2023-05-09 16:44:14,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +20: [2023-05-09 16:44:14,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +20: [2023-05-09 16:44:14,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +20: [2023-05-09 16:44:14,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +20: [2023-05-09 16:44:14,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +20: [2023-05-09 16:44:14,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +20: [2023-05-09 16:44:14,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +18: [2023-05-09 16:44:14,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +11: [2023-05-09 16:44:14,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +11: [2023-05-09 16:44:14,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +11: [2023-05-09 16:44:14,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +11: [2023-05-09 16:44:14,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +11: [2023-05-09 16:44:14,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +11: [2023-05-09 16:44:14,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +11: [2023-05-09 16:44:14,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +11: [2023-05-09 16:44:14,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +11: [2023-05-09 16:44:14,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +11: [2023-05-09 16:44:14,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +11: [2023-05-09 16:44:14,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +11: [2023-05-09 16:44:14,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +11: [2023-05-09 16:44:14,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +11: [2023-05-09 16:44:14,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +11: [2023-05-09 16:44:14,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +11: [2023-05-09 16:44:14,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +21: [2023-05-09 16:44:14,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +21: [2023-05-09 16:44:14,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +21: [2023-05-09 16:44:14,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +21: [2023-05-09 16:44:14,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +29: [2023-05-09 16:44:14,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +29: [2023-05-09 16:44:14,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +14: [2023-05-09 16:44:14,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +31: [2023-05-09 16:44:14,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +31: [2023-05-09 16:44:14,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +31: [2023-05-09 16:44:14,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +31: [2023-05-09 16:44:14,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +31: [2023-05-09 16:44:14,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +31: [2023-05-09 16:44:14,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +31: [2023-05-09 16:44:14,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +31: [2023-05-09 16:44:14,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +31: [2023-05-09 16:44:14,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +31: [2023-05-09 16:44:14,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +31: [2023-05-09 16:44:14,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +31: [2023-05-09 16:44:14,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +31: [2023-05-09 16:44:14,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +31: [2023-05-09 16:44:14,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +31: [2023-05-09 16:44:14,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt... +31: [2023-05-09 16:44:14,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt... +12: [2023-05-09 16:44:14,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +22: [2023-05-09 16:44:14,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +12: [2023-05-09 16:44:14,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +29: [2023-05-09 16:44:14,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +14: [2023-05-09 16:44:14,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +21: [2023-05-09 16:44:14,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +12: [2023-05-09 16:44:14,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +22: [2023-05-09 16:44:14,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 2: [2023-05-09 16:44:14,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 2: [2023-05-09 16:44:14,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +22: [2023-05-09 16:44:14,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +28: [2023-05-09 16:44:14,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +14: [2023-05-09 16:44:14,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +28: [2023-05-09 16:44:14,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +28: [2023-05-09 16:44:14,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +16: [2023-05-09 16:44:14,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 2: [2023-05-09 16:44:14,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 2: [2023-05-09 16:44:14,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 5: [2023-05-09 16:44:14,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +13: [2023-05-09 16:44:14,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +18: [2023-05-09 16:44:14,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +28: [2023-05-09 16:44:14,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +18: [2023-05-09 16:44:14,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +27: [2023-05-09 16:44:14,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +20: [2023-05-09 16:44:14,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +27: [2023-05-09 16:44:14,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +16: [2023-05-09 16:44:14,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +14: [2023-05-09 16:44:14,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +31: [2023-05-09 16:44:14,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +20: [2023-05-09 16:44:14,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +27: [2023-05-09 16:44:14,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +27: [2023-05-09 16:44:14,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +12: [2023-05-09 16:44:14,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +29: [2023-05-09 16:44:14,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +18: [2023-05-09 16:44:14,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +20: [2023-05-09 16:44:14,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +16: [2023-05-09 16:44:14,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +16: [2023-05-09 16:44:14,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +29: [2023-05-09 16:44:14,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +21: [2023-05-09 16:44:14,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +14: [2023-05-09 16:44:14,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +12: [2023-05-09 16:44:14,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +29: [2023-05-09 16:44:14,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +22: [2023-05-09 16:44:14,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +18: [2023-05-09 16:44:14,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +22: [2023-05-09 16:44:14,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +12: [2023-05-09 16:44:14,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +22: [2023-05-09 16:44:14,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +16: [2023-05-09 16:44:14,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +31: [2023-05-09 16:44:14,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +31: [2023-05-09 16:44:14,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +16: [2023-05-09 16:44:14,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 5: [2023-05-09 16:44:14,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +29: [2023-05-09 16:44:14,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +27: [2023-05-09 16:44:14,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +31: [2023-05-09 16:44:14,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +28: [2023-05-09 16:44:14,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +28: [2023-05-09 16:44:14,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +27: [2023-05-09 16:44:14,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +28: [2023-05-09 16:44:14,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +20: [2023-05-09 16:44:14,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +20: [2023-05-09 16:44:14,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 7: [2023-05-09 16:44:14,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 7: [2023-05-09 16:44:14,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +28: [2023-05-09 16:44:14,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +16: [2023-05-09 16:44:14,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +16: [2023-05-09 16:44:14,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +18: [2023-05-09 16:44:14,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +18: [2023-05-09 16:44:14,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 2: [2023-05-09 16:44:14,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +12: [2023-05-09 16:44:14,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 2: [2023-05-09 16:44:14,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +11: [2023-05-09 16:44:14,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +11: [2023-05-09 16:44:14,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +11: [2023-05-09 16:44:14,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. + 2: [2023-05-09 16:44:14,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 2: [2023-05-09 16:44:14,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +27: [2023-05-09 16:44:14,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +27: [2023-05-09 16:44:14,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +31: [2023-05-09 16:44:14,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +18: [2023-05-09 16:44:14,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +20: [2023-05-09 16:44:14,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +20: [2023-05-09 16:44:14,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +29: [2023-05-09 16:44:14,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +18: [2023-05-09 16:44:14,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +31: [2023-05-09 16:44:14,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +11: [2023-05-09 16:44:14,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt. +31: [2023-05-09 16:44:14,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +11: [2023-05-09 16:44:14,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +31: [2023-05-09 16:44:14,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +20: [2023-05-09 16:44:14,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +11: [2023-05-09 16:44:14,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +11: [2023-05-09 16:44:14,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +11: [2023-05-09 16:44:14,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +26: [2023-05-09 16:44:14,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 3: [2023-05-09 16:44:14,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 3: [2023-05-09 16:44:14,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +26: [2023-05-09 16:44:14,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +26: [2023-05-09 16:44:14,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +26: [2023-05-09 16:44:14,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 3: [2023-05-09 16:44:14,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 3: [2023-05-09 16:44:14,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 3: [2023-05-09 16:44:14,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +14: [2023-05-09 16:44:14,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +14: [2023-05-09 16:44:14,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +14: [2023-05-09 16:44:14,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +14: [2023-05-09 16:44:14,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +10: [2023-05-09 16:44:14,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +10: [2023-05-09 16:44:14,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +10: [2023-05-09 16:44:14,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +10: [2023-05-09 16:44:14,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +10: [2023-05-09 16:44:14,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +26: [2023-05-09 16:44:14,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +10: [2023-05-09 16:44:14,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +10: [2023-05-09 16:44:14,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +10: [2023-05-09 16:44:14,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +26: [2023-05-09 16:44:14,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +26: [2023-05-09 16:44:14,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +26: [2023-05-09 16:44:14,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +14: [2023-05-09 16:44:15,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +14: [2023-05-09 16:44:15,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +14: [2023-05-09 16:44:15,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +16: [2023-05-09 16:44:15,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +16: [2023-05-09 16:44:15,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +16: [2023-05-09 16:44:15,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +16: [2023-05-09 16:44:15,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +14: [2023-05-09 16:44:15,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +22: [2023-05-09 16:44:15,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +22: [2023-05-09 16:44:15,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +22: [2023-05-09 16:44:15,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +22: [2023-05-09 16:44:15,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 3: [2023-05-09 16:44:15,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +14: [2023-05-09 16:44:15,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +14: [2023-05-09 16:44:15,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +16: [2023-05-09 16:44:15,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +25: [2023-05-09 16:44:15,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +25: [2023-05-09 16:44:15,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +25: [2023-05-09 16:44:15,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +25: [2023-05-09 16:44:15,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +16: [2023-05-09 16:44:15,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +16: [2023-05-09 16:44:15,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +16: [2023-05-09 16:44:15,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +25: [2023-05-09 16:44:15,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +25: [2023-05-09 16:44:15,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +25: [2023-05-09 16:44:15,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +25: [2023-05-09 16:44:15,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +22: [2023-05-09 16:44:15,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +22: [2023-05-09 16:44:15,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +10: [2023-05-09 16:44:15,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +14: [2023-05-09 16:44:15,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +14: [2023-05-09 16:44:15,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 3: [2023-05-09 16:44:15,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +22: [2023-05-09 16:44:15,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +22: [2023-05-09 16:44:15,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +14: [2023-05-09 16:44:15,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +14: [2023-05-09 16:44:15,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +10: [2023-05-09 16:44:15,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +10: [2023-05-09 16:44:15,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +10: [2023-05-09 16:44:15,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +24: [2023-05-09 16:44:15,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +24: [2023-05-09 16:44:15,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +24: [2023-05-09 16:44:15,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +24: [2023-05-09 16:44:15,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +10: [2023-05-09 16:44:15,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +24: [2023-05-09 16:44:15,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +24: [2023-05-09 16:44:15,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +24: [2023-05-09 16:44:15,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +24: [2023-05-09 16:44:15,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +17: [2023-05-09 16:44:15,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +17: [2023-05-09 16:44:15,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +17: [2023-05-09 16:44:15,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +17: [2023-05-09 16:44:15,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +17: [2023-05-09 16:44:15,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +17: [2023-05-09 16:44:15,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +22: [2023-05-09 16:44:15,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +22: [2023-05-09 16:44:15,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +14: [2023-05-09 16:44:15,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +22: [2023-05-09 16:44:15,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +22: [2023-05-09 16:44:15,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +10: [2023-05-09 16:44:15,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +10: [2023-05-09 16:44:15,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +10: [2023-05-09 16:44:15,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +17: [2023-05-09 16:44:15,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +14: [2023-05-09 16:44:15,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +17: [2023-05-09 16:44:15,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +13: [2023-05-09 16:44:15,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +25: [2023-05-09 16:44:15,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +13: [2023-05-09 16:44:15,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +13: [2023-05-09 16:44:15,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +13: [2023-05-09 16:44:15,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +16: [2023-05-09 16:44:15,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +16: [2023-05-09 16:44:15,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +16: [2023-05-09 16:44:15,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +16: [2023-05-09 16:44:15,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +16: [2023-05-09 16:44:15,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +25: [2023-05-09 16:44:15,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +25: [2023-05-09 16:44:15,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +25: [2023-05-09 16:44:15,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +22: [2023-05-09 16:44:15,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +16: [2023-05-09 16:44:15,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 4: [2023-05-09 16:44:15,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +17: [2023-05-09 16:44:15,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +22: [2023-05-09 16:44:15,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +16: [2023-05-09 16:44:15,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +16: [2023-05-09 16:44:15,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +17: [2023-05-09 16:44:15,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +25: [2023-05-09 16:44:15,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +13: [2023-05-09 16:44:15,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +22: [2023-05-09 16:44:15,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +13: [2023-05-09 16:44:15,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +22: [2023-05-09 16:44:15,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +13: [2023-05-09 16:44:15,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +13: [2023-05-09 16:44:15,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +24: [2023-05-09 16:44:15,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 3: [2023-05-09 16:44:15,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 3: [2023-05-09 16:44:15,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 3: [2023-05-09 16:44:15,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 0: [2023-05-09 16:44:15,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +25: [2023-05-09 16:44:15,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +25: [2023-05-09 16:44:15,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +25: [2023-05-09 16:44:15,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 8: [2023-05-09 16:44:15,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 8: [2023-05-09 16:44:15,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 8: [2023-05-09 16:44:15,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +17: [2023-05-09 16:44:15,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +24: [2023-05-09 16:44:15,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +24: [2023-05-09 16:44:15,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +24: [2023-05-09 16:44:15,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +24: [2023-05-09 16:44:15,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +17: [2023-05-09 16:44:15,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +17: [2023-05-09 16:44:15,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +17: [2023-05-09 16:44:15,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +24: [2023-05-09 16:44:15,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +24: [2023-05-09 16:44:15,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +24: [2023-05-09 16:44:15,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +17: [2023-05-09 16:44:15,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +17: [2023-05-09 16:44:15,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +30: [2023-05-09 16:44:15,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +30: [2023-05-09 16:44:15,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +30: [2023-05-09 16:44:15,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +30: [2023-05-09 16:44:15,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +30: [2023-05-09 16:44:15,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +30: [2023-05-09 16:44:15,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +30: [2023-05-09 16:44:15,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +30: [2023-05-09 16:44:15,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +21: [2023-05-09 16:44:15,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +21: [2023-05-09 16:44:15,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +12: [2023-05-09 16:44:15,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +12: [2023-05-09 16:44:15,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 9: [2023-05-09 16:44:15,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +12: [2023-05-09 16:44:15,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +12: [2023-05-09 16:44:15,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +21: [2023-05-09 16:44:15,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +21: [2023-05-09 16:44:15,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +21: [2023-05-09 16:44:15,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +21: [2023-05-09 16:44:15,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +21: [2023-05-09 16:44:15,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +21: [2023-05-09 16:44:15,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +15: [2023-05-09 16:44:15,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +15: [2023-05-09 16:44:15,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +15: [2023-05-09 16:44:15,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +15: [2023-05-09 16:44:15,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +15: [2023-05-09 16:44:15,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +19: [2023-05-09 16:44:15,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +19: [2023-05-09 16:44:15,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +19: [2023-05-09 16:44:15,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +19: [2023-05-09 16:44:15,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +15: [2023-05-09 16:44:15,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +19: [2023-05-09 16:44:15,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +23: [2023-05-09 16:44:15,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +19: [2023-05-09 16:44:15,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +23: [2023-05-09 16:44:15,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +23: [2023-05-09 16:44:15,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +15: [2023-05-09 16:44:15,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +23: [2023-05-09 16:44:15,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 9: [2023-05-09 16:44:15,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +19: [2023-05-09 16:44:15,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +15: [2023-05-09 16:44:15,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +26: [2023-05-09 16:44:15,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +26: [2023-05-09 16:44:15,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +19: [2023-05-09 16:44:15,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +26: [2023-05-09 16:44:15,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +26: [2023-05-09 16:44:15,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +26: [2023-05-09 16:44:15,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +26: [2023-05-09 16:44:15,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +26: [2023-05-09 16:44:15,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +26: [2023-05-09 16:44:15,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +26: [2023-05-09 16:44:15,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +26: [2023-05-09 16:44:15,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +26: [2023-05-09 16:44:15,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +23: [2023-05-09 16:44:15,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +23: [2023-05-09 16:44:15,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +23: [2023-05-09 16:44:15,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +23: [2023-05-09 16:44:15,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +26: [2023-05-09 16:44:15,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +26: [2023-05-09 16:44:15,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +26: [2023-05-09 16:44:15,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +23: [2023-05-09 16:44:15,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +23: [2023-05-09 16:44:15,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +26: [2023-05-09 16:44:15,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +26: [2023-05-09 16:44:15,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +30: [2023-05-09 16:44:15,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +12: [2023-05-09 16:44:15,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +30: [2023-05-09 16:44:15,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +23: [2023-05-09 16:44:15,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +23: [2023-05-09 16:44:15,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +21: [2023-05-09 16:44:15,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +12: [2023-05-09 16:44:15,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +15: [2023-05-09 16:44:15,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 3: [2023-05-09 16:44:15,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 8: [2023-05-09 16:44:15,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +12: [2023-05-09 16:44:15,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +12: [2023-05-09 16:44:15,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +16: [2023-05-09 16:44:15,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +23: [2023-05-09 16:44:15,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +16: [2023-05-09 16:44:15,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +16: [2023-05-09 16:44:15,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +16: [2023-05-09 16:44:15,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +19: [2023-05-09 16:44:15,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +19: [2023-05-09 16:44:15,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +19: [2023-05-09 16:44:15,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +30: [2023-05-09 16:44:15,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +19: [2023-05-09 16:44:15,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +23: [2023-05-09 16:44:15,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +16: [2023-05-09 16:44:15,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +16: [2023-05-09 16:44:15,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +16: [2023-05-09 16:44:15,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +16: [2023-05-09 16:44:15,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +23: [2023-05-09 16:44:15,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +21: [2023-05-09 16:44:15,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +21: [2023-05-09 16:44:15,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +21: [2023-05-09 16:44:15,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +15: [2023-05-09 16:44:15,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +26: [2023-05-09 16:44:15,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +26: [2023-05-09 16:44:15,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +30: [2023-05-09 16:44:15,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +23: [2023-05-09 16:44:15,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 7: [2023-05-09 16:44:15,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +12: [2023-05-09 16:44:15,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +14: [2023-05-09 16:44:15,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +14: [2023-05-09 16:44:15,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +14: [2023-05-09 16:44:15,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +14: [2023-05-09 16:44:15,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +14: [2023-05-09 16:44:15,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +14: [2023-05-09 16:44:15,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +14: [2023-05-09 16:44:15,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +14: [2023-05-09 16:44:15,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +21: [2023-05-09 16:44:15,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +23: [2023-05-09 16:44:15,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +12: [2023-05-09 16:44:15,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +19: [2023-05-09 16:44:15,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +19: [2023-05-09 16:44:15,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +29: [2023-05-09 16:44:15,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +29: [2023-05-09 16:44:15,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +29: [2023-05-09 16:44:15,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +29: [2023-05-09 16:44:15,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +30: [2023-05-09 16:44:15,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +30: [2023-05-09 16:44:15,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +30: [2023-05-09 16:44:15,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +30: [2023-05-09 16:44:15,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +30: [2023-05-09 16:44:15,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +12: [2023-05-09 16:44:15,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +12: [2023-05-09 16:44:15,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +26: [2023-05-09 16:44:15,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +26: [2023-05-09 16:44:15,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +16: [2023-05-09 16:44:15,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +15: [2023-05-09 16:44:15,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +13: [2023-05-09 16:44:15,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +13: [2023-05-09 16:44:15,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +13: [2023-05-09 16:44:15,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +13: [2023-05-09 16:44:15,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +13: [2023-05-09 16:44:15,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +13: [2023-05-09 16:44:15,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +13: [2023-05-09 16:44:15,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +13: [2023-05-09 16:44:15,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +13: [2023-05-09 16:44:15,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +13: [2023-05-09 16:44:15,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +13: [2023-05-09 16:44:15,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +13: [2023-05-09 16:44:15,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +13: [2023-05-09 16:44:15,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +29: [2023-05-09 16:44:15,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +29: [2023-05-09 16:44:15,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +19: [2023-05-09 16:44:15,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +19: [2023-05-09 16:44:15,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +13: [2023-05-09 16:44:15,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +21: [2023-05-09 16:44:15,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +16: [2023-05-09 16:44:15,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +16: [2023-05-09 16:44:15,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +19: [2023-05-09 16:44:15,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +21: [2023-05-09 16:44:15,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +29: [2023-05-09 16:44:15,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +15: [2023-05-09 16:44:15,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +29: [2023-05-09 16:44:15,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +13: [2023-05-09 16:44:15,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +21: [2023-05-09 16:44:15,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +30: [2023-05-09 16:44:15,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +13: [2023-05-09 16:44:15,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +23: [2023-05-09 16:44:15,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +15: [2023-05-09 16:44:15,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +10: [2023-05-09 16:44:15,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +10: [2023-05-09 16:44:15,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +10: [2023-05-09 16:44:15,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +10: [2023-05-09 16:44:15,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 1: [2023-05-09 16:44:15,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +14: [2023-05-09 16:44:15,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +14: [2023-05-09 16:44:15,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +16: [2023-05-09 16:44:15,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +23: [2023-05-09 16:44:15,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +30: [2023-05-09 16:44:15,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +26: [2023-05-09 16:44:15,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +26: [2023-05-09 16:44:15,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +16: [2023-05-09 16:44:15,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +14: [2023-05-09 16:44:15,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +14: [2023-05-09 16:44:15,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +30: [2023-05-09 16:44:15,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +30: [2023-05-09 16:44:15,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +30: [2023-05-09 16:44:15,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +30: [2023-05-09 16:44:15,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +15: [2023-05-09 16:44:15,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +16: [2023-05-09 16:44:15,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +16: [2023-05-09 16:44:15,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +13: [2023-05-09 16:44:15,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +19: [2023-05-09 16:44:15,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +19: [2023-05-09 16:44:15,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +30: [2023-05-09 16:44:15,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +19: [2023-05-09 16:44:15,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +12: [2023-05-09 16:44:15,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +12: [2023-05-09 16:44:15,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +15: [2023-05-09 16:44:15,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +23: [2023-05-09 16:44:15,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +20: [2023-05-09 16:44:15,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +20: [2023-05-09 16:44:15,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +20: [2023-05-09 16:44:15,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +15: [2023-05-09 16:44:15,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +23: [2023-05-09 16:44:15,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +23: [2023-05-09 16:44:15,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +16: [2023-05-09 16:44:15,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +10: [2023-05-09 16:44:15,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +29: [2023-05-09 16:44:15,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +29: [2023-05-09 16:44:15,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +20: [2023-05-09 16:44:15,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +20: [2023-05-09 16:44:15,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +20: [2023-05-09 16:44:15,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +12: [2023-05-09 16:44:15,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +12: [2023-05-09 16:44:15,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 1: [2023-05-09 16:44:15,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +19: [2023-05-09 16:44:15,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +19: [2023-05-09 16:44:15,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +19: [2023-05-09 16:44:15,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +26: [2023-05-09 16:44:15,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +20: [2023-05-09 16:44:15,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +14: [2023-05-09 16:44:15,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +26: [2023-05-09 16:44:15,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +19: [2023-05-09 16:44:15,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +19: [2023-05-09 16:44:15,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +10: [2023-05-09 16:44:15,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +23: [2023-05-09 16:44:15,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +23: [2023-05-09 16:44:15,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +10: [2023-05-09 16:44:15,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +23: [2023-05-09 16:44:15,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +10: [2023-05-09 16:44:15,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +23: [2023-05-09 16:44:15,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +14: [2023-05-09 16:44:15,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +14: [2023-05-09 16:44:15,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +13: [2023-05-09 16:44:15,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +20: [2023-05-09 16:44:15,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +23: [2023-05-09 16:44:15,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +13: [2023-05-09 16:44:15,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +23: [2023-05-09 16:44:15,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +14: [2023-05-09 16:44:15,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 6: [2023-05-09 16:44:15,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 6: [2023-05-09 16:44:15,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 6: [2023-05-09 16:44:15,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +13: [2023-05-09 16:44:15,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +29: [2023-05-09 16:44:15,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +29: [2023-05-09 16:44:15,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +23: [2023-05-09 16:44:15,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 0: [2023-05-09 16:44:15,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 0: [2023-05-09 16:44:15,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +28: [2023-05-09 16:44:15,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +28: [2023-05-09 16:44:15,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +28: [2023-05-09 16:44:15,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +28: [2023-05-09 16:44:15,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +23: [2023-05-09 16:44:15,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +27: [2023-05-09 16:44:15,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +27: [2023-05-09 16:44:15,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +27: [2023-05-09 16:44:15,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +23: [2023-05-09 16:44:15,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +27: [2023-05-09 16:44:15,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +29: [2023-05-09 16:44:15,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +19: [2023-05-09 16:44:15,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +28: [2023-05-09 16:44:15,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +19: [2023-05-09 16:44:15,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +19: [2023-05-09 16:44:15,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +20: [2023-05-09 16:44:15,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +23: [2023-05-09 16:44:15,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +20: [2023-05-09 16:44:15,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +11: [2023-05-09 16:44:15,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +11: [2023-05-09 16:44:15,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +11: [2023-05-09 16:44:15,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +11: [2023-05-09 16:44:15,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +27: [2023-05-09 16:44:15,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +27: [2023-05-09 16:44:15,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +18: [2023-05-09 16:44:15,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +18: [2023-05-09 16:44:15,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +18: [2023-05-09 16:44:15,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +11: [2023-05-09 16:44:15,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +28: [2023-05-09 16:44:15,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +11: [2023-05-09 16:44:15,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +18: [2023-05-09 16:44:15,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +18: [2023-05-09 16:44:15,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +28: [2023-05-09 16:44:15,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +27: [2023-05-09 16:44:15,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +27: [2023-05-09 16:44:15,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +28: [2023-05-09 16:44:15,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +13: [2023-05-09 16:44:15,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +13: [2023-05-09 16:44:15,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +18: [2023-05-09 16:44:15,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +18: [2023-05-09 16:44:15,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +11: [2023-05-09 16:44:15,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +18: [2023-05-09 16:44:15,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +10: [2023-05-09 16:44:15,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +11: [2023-05-09 16:44:15,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +10: [2023-05-09 16:44:15,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 6: [2023-05-09 16:44:15,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +29: [2023-05-09 16:44:15,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +29: [2023-05-09 16:44:15,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +31: [2023-05-09 16:44:15,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +31: [2023-05-09 16:44:15,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +30: [2023-05-09 16:44:15,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +30: [2023-05-09 16:44:15,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +13: [2023-05-09 16:44:15,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +30: [2023-05-09 16:44:15,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +30: [2023-05-09 16:44:15,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +31: [2023-05-09 16:44:15,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +31: [2023-05-09 16:44:15,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +31: [2023-05-09 16:44:15,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +31: [2023-05-09 16:44:15,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +31: [2023-05-09 16:44:15,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +31: [2023-05-09 16:44:15,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +19: [2023-05-09 16:44:15,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +19: [2023-05-09 16:44:15,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +30: [2023-05-09 16:44:15,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +20: [2023-05-09 16:44:15,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +30: [2023-05-09 16:44:15,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +31: [2023-05-09 16:44:15,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +30: [2023-05-09 16:44:15,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +30: [2023-05-09 16:44:15,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +29: [2023-05-09 16:44:15,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +18: [2023-05-09 16:44:15,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +31: [2023-05-09 16:44:15,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +19: [2023-05-09 16:44:15,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +19: [2023-05-09 16:44:15,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +20: [2023-05-09 16:44:15,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +22: [2023-05-09 16:44:15,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +22: [2023-05-09 16:44:15,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +22: [2023-05-09 16:44:15,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +22: [2023-05-09 16:44:15,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +22: [2023-05-09 16:44:15,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +22: [2023-05-09 16:44:15,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +22: [2023-05-09 16:44:15,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +22: [2023-05-09 16:44:15,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +20: [2023-05-09 16:44:15,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +31: [2023-05-09 16:44:15,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +31: [2023-05-09 16:44:15,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +11: [2023-05-09 16:44:15,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +11: [2023-05-09 16:44:15,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +10: [2023-05-09 16:44:15,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +13: [2023-05-09 16:44:15,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +10: [2023-05-09 16:44:15,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 5: [2023-05-09 16:44:15,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +18: [2023-05-09 16:44:15,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +28: [2023-05-09 16:44:15,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +31: [2023-05-09 16:44:15,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +20: [2023-05-09 16:44:15,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +31: [2023-05-09 16:44:15,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +10: [2023-05-09 16:44:15,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +10: [2023-05-09 16:44:15,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +20: [2023-05-09 16:44:15,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +27: [2023-05-09 16:44:15,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +31: [2023-05-09 16:44:15,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +11: [2023-05-09 16:44:15,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +11: [2023-05-09 16:44:15,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +10: [2023-05-09 16:44:15,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +10: [2023-05-09 16:44:15,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +27: [2023-05-09 16:44:15,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +22: [2023-05-09 16:44:15,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +22: [2023-05-09 16:44:15,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +31: [2023-05-09 16:44:15,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +28: [2023-05-09 16:44:15,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +27: [2023-05-09 16:44:15,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +28: [2023-05-09 16:44:15,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +28: [2023-05-09 16:44:15,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +18: [2023-05-09 16:44:15,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +18: [2023-05-09 16:44:15,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +18: [2023-05-09 16:44:15,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 5: [2023-05-09 16:44:15,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +27: [2023-05-09 16:44:15,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +28: [2023-05-09 16:44:15,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +22: [2023-05-09 16:44:15,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +22: [2023-05-09 16:44:15,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 6: [2023-05-09 16:44:15,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +20: [2023-05-09 16:44:15,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +27: [2023-05-09 16:44:15,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +22: [2023-05-09 16:44:15,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +11: [2023-05-09 16:44:15,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +11: [2023-05-09 16:44:15,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +27: [2023-05-09 16:44:15,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +28: [2023-05-09 16:44:15,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +27: [2023-05-09 16:44:15,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +22: [2023-05-09 16:44:15,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +28: [2023-05-09 16:44:15,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +28: [2023-05-09 16:44:15,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +27: [2023-05-09 16:44:15,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +18: [2023-05-09 16:44:15,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +18: [2023-05-09 16:44:15,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +18: [2023-05-09 16:44:15,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +22: [2023-05-09 16:44:15,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +22: [2023-05-09 16:44:15,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 5: [2023-05-09 16:44:15,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 0: [2023-05-09 16:44:15,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 5: [2023-05-09 16:44:15,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 0: [2023-05-09 16:44:15,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 0: [2023-05-09 16:44:15,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +11: [2023-05-09 16:44:15,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +11: [2023-05-09 16:44:15,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 5: [2023-05-09 16:44:15,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +12: [2023-05-09 16:44:15,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +12: [2023-05-09 16:44:15,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +12: [2023-05-09 16:44:15,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +12: [2023-05-09 16:44:15,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +12: [2023-05-09 16:44:15,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +12: [2023-05-09 16:44:15,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +12: [2023-05-09 16:44:15,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +12: [2023-05-09 16:44:15,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +10: [2023-05-09 16:44:15,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +10: [2023-05-09 16:44:15,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +10: [2023-05-09 16:44:15,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +10: [2023-05-09 16:44:15,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +10: [2023-05-09 16:44:15,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +10: [2023-05-09 16:44:15,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +10: [2023-05-09 16:44:15,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +10: [2023-05-09 16:44:15,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +12: [2023-05-09 16:44:15,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 4: [2023-05-09 16:44:15,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +10: [2023-05-09 16:44:15,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +25: [2023-05-09 16:44:15,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +25: [2023-05-09 16:44:15,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +17: [2023-05-09 16:44:15,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +17: [2023-05-09 16:44:15,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +25: [2023-05-09 16:44:15,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +25: [2023-05-09 16:44:15,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +25: [2023-05-09 16:44:15,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +17: [2023-05-09 16:44:15,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +17: [2023-05-09 16:44:15,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +25: [2023-05-09 16:44:15,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +25: [2023-05-09 16:44:15,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +25: [2023-05-09 16:44:15,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +17: [2023-05-09 16:44:15,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +17: [2023-05-09 16:44:15,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +18: [2023-05-09 16:44:15,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +17: [2023-05-09 16:44:15,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +12: [2023-05-09 16:44:15,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +12: [2023-05-09 16:44:15,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +18: [2023-05-09 16:44:15,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +18: [2023-05-09 16:44:15,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +17: [2023-05-09 16:44:15,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +12: [2023-05-09 16:44:15,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +18: [2023-05-09 16:44:15,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 8: [2023-05-09 16:44:15,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +12: [2023-05-09 16:44:15,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +10: [2023-05-09 16:44:15,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +15: [2023-05-09 16:44:15,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +15: [2023-05-09 16:44:15,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +15: [2023-05-09 16:44:15,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +15: [2023-05-09 16:44:15,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +15: [2023-05-09 16:44:15,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +15: [2023-05-09 16:44:15,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +15: [2023-05-09 16:44:15,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +15: [2023-05-09 16:44:15,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +10: [2023-05-09 16:44:15,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +10: [2023-05-09 16:44:15,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +10: [2023-05-09 16:44:15,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +25: [2023-05-09 16:44:15,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 5: [2023-05-09 16:44:15,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 5: [2023-05-09 16:44:15,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +17: [2023-05-09 16:44:15,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +17: [2023-05-09 16:44:15,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +18: [2023-05-09 16:44:15,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +10: [2023-05-09 16:44:15,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +12: [2023-05-09 16:44:15,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +15: [2023-05-09 16:44:15,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +15: [2023-05-09 16:44:15,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +18: [2023-05-09 16:44:15,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +17: [2023-05-09 16:44:15,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +12: [2023-05-09 16:44:15,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +18: [2023-05-09 16:44:15,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +12: [2023-05-09 16:44:15,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +15: [2023-05-09 16:44:15,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +15: [2023-05-09 16:44:15,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +25: [2023-05-09 16:44:15,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +25: [2023-05-09 16:44:15,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +18: [2023-05-09 16:44:15,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +25: [2023-05-09 16:44:15,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +25: [2023-05-09 16:44:15,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +10: [2023-05-09 16:44:15,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +10: [2023-05-09 16:44:15,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +17: [2023-05-09 16:44:15,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +17: [2023-05-09 16:44:15,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +15: [2023-05-09 16:44:15,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +17: [2023-05-09 16:44:15,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +15: [2023-05-09 16:44:15,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +17: [2023-05-09 16:44:15,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +25: [2023-05-09 16:44:15,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +25: [2023-05-09 16:44:15,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +25: [2023-05-09 16:44:15,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +15: [2023-05-09 16:44:15,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +15: [2023-05-09 16:44:15,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +18: [2023-05-09 16:44:15,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +17: [2023-05-09 16:44:15,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +18: [2023-05-09 16:44:15,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +30: [2023-05-09 16:44:15,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +30: [2023-05-09 16:44:15,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +30: [2023-05-09 16:44:15,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +30: [2023-05-09 16:44:15,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +30: [2023-05-09 16:44:15,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +30: [2023-05-09 16:44:15,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +30: [2023-05-09 16:44:15,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +30: [2023-05-09 16:44:15,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +18: [2023-05-09 16:44:15,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +18: [2023-05-09 16:44:15,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +18: [2023-05-09 16:44:15,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +30: [2023-05-09 16:44:15,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +30: [2023-05-09 16:44:15,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +18: [2023-05-09 16:44:15,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +18: [2023-05-09 16:44:15,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +18: [2023-05-09 16:44:15,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +20: [2023-05-09 16:44:15,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +20: [2023-05-09 16:44:15,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +30: [2023-05-09 16:44:15,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +30: [2023-05-09 16:44:15,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +20: [2023-05-09 16:44:15,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 6: [2023-05-09 16:44:15,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +20: [2023-05-09 16:44:15,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 6: [2023-05-09 16:44:15,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +30: [2023-05-09 16:44:15,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +30: [2023-05-09 16:44:15,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +30: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +26: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +26: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +26: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +26: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +19: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +26: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +23: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +23: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +19: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +23: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +26: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +26: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +26: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +19: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +19: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +23: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +23: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +19: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +24: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +19: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +23: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +23: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +19: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +24: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +23: [2023-05-09 16:44:15,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +19: [2023-05-09 16:44:15,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +24: [2023-05-09 16:44:15,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +24: [2023-05-09 16:44:15,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +24: [2023-05-09 16:44:15,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +13: [2023-05-09 16:44:15,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +13: [2023-05-09 16:44:15,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +13: [2023-05-09 16:44:15,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +24: [2023-05-09 16:44:15,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +24: [2023-05-09 16:44:15,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +24: [2023-05-09 16:44:15,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +13: [2023-05-09 16:44:15,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +13: [2023-05-09 16:44:15,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +13: [2023-05-09 16:44:15,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +13: [2023-05-09 16:44:15,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +13: [2023-05-09 16:44:15,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +30: [2023-05-09 16:44:15,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +28: [2023-05-09 16:44:15,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +28: [2023-05-09 16:44:15,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +28: [2023-05-09 16:44:15,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +28: [2023-05-09 16:44:15,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +27: [2023-05-09 16:44:15,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +27: [2023-05-09 16:44:15,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +27: [2023-05-09 16:44:15,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +20: [2023-05-09 16:44:15,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +20: [2023-05-09 16:44:15,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +29: [2023-05-09 16:44:15,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +29: [2023-05-09 16:44:15,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +29: [2023-05-09 16:44:15,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +20: [2023-05-09 16:44:15,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +29: [2023-05-09 16:44:15,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 2: [2023-05-09 16:44:15,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +27: [2023-05-09 16:44:15,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 7: [2023-05-09 16:44:15,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +24: [2023-05-09 16:44:15,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +20: [2023-05-09 16:44:15,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +19: [2023-05-09 16:44:15,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +23: [2023-05-09 16:44:15,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +13: [2023-05-09 16:44:15,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +23: [2023-05-09 16:44:15,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +28: [2023-05-09 16:44:15,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +26: [2023-05-09 16:44:15,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +26: [2023-05-09 16:44:15,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +27: [2023-05-09 16:44:15,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +26: [2023-05-09 16:44:15,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +26: [2023-05-09 16:44:15,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +23: [2023-05-09 16:44:15,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +23: [2023-05-09 16:44:15,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +27: [2023-05-09 16:44:15,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +28: [2023-05-09 16:44:15,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +27: [2023-05-09 16:44:15,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +28: [2023-05-09 16:44:15,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +28: [2023-05-09 16:44:15,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +29: [2023-05-09 16:44:15,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +29: [2023-05-09 16:44:15,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +29: [2023-05-09 16:44:15,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +29: [2023-05-09 16:44:15,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +13: [2023-05-09 16:44:15,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +13: [2023-05-09 16:44:15,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +13: [2023-05-09 16:44:15,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +24: [2023-05-09 16:44:15,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +24: [2023-05-09 16:44:15,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +19: [2023-05-09 16:44:15,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +19: [2023-05-09 16:44:15,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +19: [2023-05-09 16:44:15,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +27: [2023-05-09 16:44:15,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +24: [2023-05-09 16:44:15,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +25: [2023-05-09 16:44:15,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +24: [2023-05-09 16:44:15,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +25: [2023-05-09 16:44:15,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +25: [2023-05-09 16:44:15,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +20: [2023-05-09 16:44:15,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +25: [2023-05-09 16:44:15,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 6: [2023-05-09 16:44:15,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +20: [2023-05-09 16:44:15,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +20: [2023-05-09 16:44:15,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +13: [2023-05-09 16:44:15,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +20: [2023-05-09 16:44:15,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +26: [2023-05-09 16:44:15,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +26: [2023-05-09 16:44:15,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +19: [2023-05-09 16:44:15,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +23: [2023-05-09 16:44:15,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +23: [2023-05-09 16:44:15,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +23: [2023-05-09 16:44:15,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +28: [2023-05-09 16:44:15,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +26: [2023-05-09 16:44:15,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +26: [2023-05-09 16:44:15,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +23: [2023-05-09 16:44:15,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +27: [2023-05-09 16:44:15,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +27: [2023-05-09 16:44:15,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +13: [2023-05-09 16:44:15,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +24: [2023-05-09 16:44:15,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +24: [2023-05-09 16:44:15,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +25: [2023-05-09 16:44:15,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +13: [2023-05-09 16:44:15,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +19: [2023-05-09 16:44:15,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +24: [2023-05-09 16:44:15,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +13: [2023-05-09 16:44:15,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +19: [2023-05-09 16:44:15,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +25: [2023-05-09 16:44:15,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +19: [2023-05-09 16:44:15,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +25: [2023-05-09 16:44:15,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +25: [2023-05-09 16:44:15,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +28: [2023-05-09 16:44:15,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +29: [2023-05-09 16:44:15,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +29: [2023-05-09 16:44:15,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +29: [2023-05-09 16:44:15,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +29: [2023-05-09 16:44:15,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 2: [2023-05-09 16:44:15,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +29: [2023-05-09 16:44:15,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +17: [2023-05-09 16:44:15,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +29: [2023-05-09 16:44:15,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +17: [2023-05-09 16:44:15,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +17: [2023-05-09 16:44:15,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +17: [2023-05-09 16:44:15,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +27: [2023-05-09 16:44:15,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +27: [2023-05-09 16:44:15,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 2: [2023-05-09 16:44:15,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +20: [2023-05-09 16:44:15,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +20: [2023-05-09 16:44:15,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +20: [2023-05-09 16:44:15,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +20: [2023-05-09 16:44:15,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 2: [2023-05-09 16:44:15,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +28: [2023-05-09 16:44:15,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 2: [2023-05-09 16:44:15,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 2: [2023-05-09 16:44:15,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +29: [2023-05-09 16:44:15,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +28: [2023-05-09 16:44:15,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +29: [2023-05-09 16:44:15,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +17: [2023-05-09 16:44:15,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 1: [2023-05-09 16:44:15,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 1: [2023-05-09 16:44:15,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +17: [2023-05-09 16:44:15,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +27: [2023-05-09 16:44:15,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +27: [2023-05-09 16:44:15,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +28: [2023-05-09 16:44:15,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +28: [2023-05-09 16:44:15,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +21: [2023-05-09 16:44:15,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +21: [2023-05-09 16:44:15,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +21: [2023-05-09 16:44:15,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +21: [2023-05-09 16:44:15,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +25: [2023-05-09 16:44:15,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +17: [2023-05-09 16:44:15,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +17: [2023-05-09 16:44:15,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +25: [2023-05-09 16:44:15,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 4: [2023-05-09 16:44:15,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 4: [2023-05-09 16:44:15,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +27: [2023-05-09 16:44:15,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 4: [2023-05-09 16:44:15,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +27: [2023-05-09 16:44:15,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 4: [2023-05-09 16:44:15,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +28: [2023-05-09 16:44:15,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +28: [2023-05-09 16:44:15,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +31: [2023-05-09 16:44:15,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +31: [2023-05-09 16:44:15,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +31: [2023-05-09 16:44:15,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +31: [2023-05-09 16:44:15,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +25: [2023-05-09 16:44:15,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +25: [2023-05-09 16:44:15,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +29: [2023-05-09 16:44:15,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +29: [2023-05-09 16:44:15,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +29: [2023-05-09 16:44:15,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +29: [2023-05-09 16:44:15,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +29: [2023-05-09 16:44:15,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +29: [2023-05-09 16:44:15,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +29: [2023-05-09 16:44:15,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +29: [2023-05-09 16:44:15,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +21: [2023-05-09 16:44:15,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +17: [2023-05-09 16:44:15,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +21: [2023-05-09 16:44:15,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +21: [2023-05-09 16:44:15,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +25: [2023-05-09 16:44:15,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +17: [2023-05-09 16:44:15,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +21: [2023-05-09 16:44:15,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +25: [2023-05-09 16:44:15,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +31: [2023-05-09 16:44:15,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +25: [2023-05-09 16:44:15,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +31: [2023-05-09 16:44:15,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 7: [2023-05-09 16:44:15,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 7: [2023-05-09 16:44:15,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 7: [2023-05-09 16:44:15,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +31: [2023-05-09 16:44:15,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +31: [2023-05-09 16:44:15,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +25: [2023-05-09 16:44:15,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +14: [2023-05-09 16:44:15,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +14: [2023-05-09 16:44:15,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +14: [2023-05-09 16:44:15,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +14: [2023-05-09 16:44:15,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +29: [2023-05-09 16:44:15,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +29: [2023-05-09 16:44:15,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +29: [2023-05-09 16:44:15,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 3: [2023-05-09 16:44:15,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 3: [2023-05-09 16:44:15,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +29: [2023-05-09 16:44:15,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 3: [2023-05-09 16:44:15,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 3: [2023-05-09 16:44:15,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +21: [2023-05-09 16:44:15,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +21: [2023-05-09 16:44:15,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +16: [2023-05-09 16:44:15,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +16: [2023-05-09 16:44:15,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +17: [2023-05-09 16:44:15,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +16: [2023-05-09 16:44:15,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +21: [2023-05-09 16:44:15,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +21: [2023-05-09 16:44:15,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +16: [2023-05-09 16:44:15,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +17: [2023-05-09 16:44:15,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +14: [2023-05-09 16:44:15,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +14: [2023-05-09 16:44:15,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +14: [2023-05-09 16:44:15,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +29: [2023-05-09 16:44:15,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +14: [2023-05-09 16:44:15,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +17: [2023-05-09 16:44:15,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +17: [2023-05-09 16:44:15,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +17: [2023-05-09 16:44:15,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +17: [2023-05-09 16:44:15,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 7: [2023-05-09 16:44:15,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +19: [2023-05-09 16:44:15,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +19: [2023-05-09 16:44:15,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +29: [2023-05-09 16:44:15,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +29: [2023-05-09 16:44:15,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +19: [2023-05-09 16:44:15,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +19: [2023-05-09 16:44:15,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 3: [2023-05-09 16:44:15,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +21: [2023-05-09 16:44:15,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +16: [2023-05-09 16:44:15,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +20: [2023-05-09 16:44:15,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +20: [2023-05-09 16:44:15,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +20: [2023-05-09 16:44:15,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +20: [2023-05-09 16:44:15,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +20: [2023-05-09 16:44:15,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +20: [2023-05-09 16:44:15,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +21: [2023-05-09 16:44:15,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +20: [2023-05-09 16:44:15,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +29: [2023-05-09 16:44:15,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +16: [2023-05-09 16:44:15,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +16: [2023-05-09 16:44:15,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +16: [2023-05-09 16:44:15,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +20: [2023-05-09 16:44:15,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +31: [2023-05-09 16:44:15,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 1: [2023-05-09 16:44:15,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 2: [2023-05-09 16:44:15,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +14: [2023-05-09 16:44:15,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +21: [2023-05-09 16:44:15,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +14: [2023-05-09 16:44:15,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +11: [2023-05-09 16:44:15,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +11: [2023-05-09 16:44:15,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +11: [2023-05-09 16:44:15,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +11: [2023-05-09 16:44:15,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +31: [2023-05-09 16:44:15,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +21: [2023-05-09 16:44:15,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +19: [2023-05-09 16:44:15,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +19: [2023-05-09 16:44:15,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 1: [2023-05-09 16:44:15,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +19: [2023-05-09 16:44:15,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +19: [2023-05-09 16:44:15,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +31: [2023-05-09 16:44:15,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +31: [2023-05-09 16:44:15,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +31: [2023-05-09 16:44:15,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +16: [2023-05-09 16:44:15,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +16: [2023-05-09 16:44:15,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +20: [2023-05-09 16:44:15,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +14: [2023-05-09 16:44:15,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +14: [2023-05-09 16:44:15,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +14: [2023-05-09 16:44:15,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +14: [2023-05-09 16:44:15,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +14: [2023-05-09 16:44:15,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +14: [2023-05-09 16:44:15,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 7: [2023-05-09 16:44:15,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +20: [2023-05-09 16:44:15,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 4: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +16: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +16: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +18: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +18: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +31: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +31: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +18: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +18: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +31: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +31: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +31: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +31: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +18: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +18: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +31: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +18: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +31: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +18: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +31: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +31: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +16: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +16: [2023-05-09 16:44:15,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +31: [2023-05-09 16:44:15,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 3: [2023-05-09 16:44:15,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 3: [2023-05-09 16:44:15,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 3: [2023-05-09 16:44:15,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +11: [2023-05-09 16:44:15,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +30: [2023-05-09 16:44:15,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +30: [2023-05-09 16:44:15,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +11: [2023-05-09 16:44:15,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +30: [2023-05-09 16:44:15,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +30: [2023-05-09 16:44:15,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +28: [2023-05-09 16:44:15,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +28: [2023-05-09 16:44:15,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +28: [2023-05-09 16:44:15,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +28: [2023-05-09 16:44:15,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +28: [2023-05-09 16:44:15,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +28: [2023-05-09 16:44:15,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +11: [2023-05-09 16:44:15,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +28: [2023-05-09 16:44:15,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +11: [2023-05-09 16:44:15,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +28: [2023-05-09 16:44:15,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 4: [2023-05-09 16:44:15,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 3: [2023-05-09 16:44:15,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +20: [2023-05-09 16:44:15,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +16: [2023-05-09 16:44:15,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +16: [2023-05-09 16:44:15,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +11: [2023-05-09 16:44:15,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +11: [2023-05-09 16:44:15,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +11: [2023-05-09 16:44:15,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +11: [2023-05-09 16:44:15,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +11: [2023-05-09 16:44:15,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +11: [2023-05-09 16:44:15,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +11: [2023-05-09 16:44:15,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +11: [2023-05-09 16:44:15,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +19: [2023-05-09 16:44:15,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +19: [2023-05-09 16:44:15,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +19: [2023-05-09 16:44:15,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +19: [2023-05-09 16:44:15,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +24: [2023-05-09 16:44:15,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +24: [2023-05-09 16:44:15,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +24: [2023-05-09 16:44:15,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +19: [2023-05-09 16:44:15,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +19: [2023-05-09 16:44:15,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +24: [2023-05-09 16:44:15,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 7: [2023-05-09 16:44:15,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +20: [2023-05-09 16:44:15,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +30: [2023-05-09 16:44:15,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +30: [2023-05-09 16:44:15,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +27: [2023-05-09 16:44:15,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +27: [2023-05-09 16:44:15,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +27: [2023-05-09 16:44:15,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +27: [2023-05-09 16:44:15,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +12: [2023-05-09 16:44:15,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +12: [2023-05-09 16:44:15,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +12: [2023-05-09 16:44:15,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +12: [2023-05-09 16:44:15,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 7: [2023-05-09 16:44:15,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +30: [2023-05-09 16:44:15,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +30: [2023-05-09 16:44:15,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 7: [2023-05-09 16:44:15,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +27: [2023-05-09 16:44:15,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +27: [2023-05-09 16:44:15,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +27: [2023-05-09 16:44:15,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +27: [2023-05-09 16:44:15,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +31: [2023-05-09 16:44:15,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +20: [2023-05-09 16:44:15,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +18: [2023-05-09 16:44:15,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +18: [2023-05-09 16:44:15,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +10: [2023-05-09 16:44:15,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +31: [2023-05-09 16:44:15,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +31: [2023-05-09 16:44:15,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +31: [2023-05-09 16:44:15,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +10: [2023-05-09 16:44:15,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +28: [2023-05-09 16:44:15,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +19: [2023-05-09 16:44:15,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +19: [2023-05-09 16:44:15,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 1: [2023-05-09 16:44:15,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +23: [2023-05-09 16:44:15,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +23: [2023-05-09 16:44:15,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +23: [2023-05-09 16:44:15,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 1: [2023-05-09 16:44:15,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +23: [2023-05-09 16:44:15,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +22: [2023-05-09 16:44:15,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +22: [2023-05-09 16:44:15,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 5: [2023-05-09 16:44:15,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 8: [2023-05-09 16:44:15,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 6: [2023-05-09 16:44:15,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 1: [2023-05-09 16:44:15,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +13: [2023-05-09 16:44:15,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +13: [2023-05-09 16:44:15,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +18: [2023-05-09 16:44:15,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +10: [2023-05-09 16:44:15,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +20: [2023-05-09 16:44:15,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +22: [2023-05-09 16:44:15,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +22: [2023-05-09 16:44:15,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +20: [2023-05-09 16:44:15,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +10: [2023-05-09 16:44:15,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 6: [2023-05-09 16:44:15,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 6: [2023-05-09 16:44:15,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +26: [2023-05-09 16:44:15,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +13: [2023-05-09 16:44:15,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +13: [2023-05-09 16:44:15,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +24: [2023-05-09 16:44:15,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +24: [2023-05-09 16:44:15,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +28: [2023-05-09 16:44:15,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +28: [2023-05-09 16:44:15,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +26: [2023-05-09 16:44:15,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +24: [2023-05-09 16:44:15,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +12: [2023-05-09 16:44:15,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +12: [2023-05-09 16:44:15,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +12: [2023-05-09 16:44:15,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +18: [2023-05-09 16:44:15,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +31: [2023-05-09 16:44:15,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +12: [2023-05-09 16:44:15,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +26: [2023-05-09 16:44:15,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +26: [2023-05-09 16:44:15,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 8: [2023-05-09 16:44:15,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +27: [2023-05-09 16:44:15,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +18: [2023-05-09 16:44:15,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +18: [2023-05-09 16:44:15,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +18: [2023-05-09 16:44:15,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +24: [2023-05-09 16:44:15,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +10: [2023-05-09 16:44:15,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +31: [2023-05-09 16:44:15,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +31: [2023-05-09 16:44:15,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +31: [2023-05-09 16:44:15,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +18: [2023-05-09 16:44:15,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +18: [2023-05-09 16:44:15,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +30: [2023-05-09 16:44:15,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +18: [2023-05-09 16:44:15,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +10: [2023-05-09 16:44:15,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +30: [2023-05-09 16:44:15,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +11: [2023-05-09 16:44:15,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +23: [2023-05-09 16:44:15,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +23: [2023-05-09 16:44:15,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +30: [2023-05-09 16:44:15,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +23: [2023-05-09 16:44:15,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +30: [2023-05-09 16:44:15,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 6: [2023-05-09 16:44:15,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 6: [2023-05-09 16:44:15,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +22: [2023-05-09 16:44:15,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +28: [2023-05-09 16:44:15,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +22: [2023-05-09 16:44:15,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +30: [2023-05-09 16:44:15,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +30: [2023-05-09 16:44:15,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +22: [2023-05-09 16:44:15,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +30: [2023-05-09 16:44:15,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +30: [2023-05-09 16:44:15,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +27: [2023-05-09 16:44:15,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +11: [2023-05-09 16:44:15,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +27: [2023-05-09 16:44:15,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 8: [2023-05-09 16:44:15,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 5: [2023-05-09 16:44:15,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +10: [2023-05-09 16:44:15,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +10: [2023-05-09 16:44:15,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +27: [2023-05-09 16:44:15,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +13: [2023-05-09 16:44:15,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +13: [2023-05-09 16:44:15,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +13: [2023-05-09 16:44:15,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +23: [2023-05-09 16:44:15,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +22: [2023-05-09 16:44:15,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +28: [2023-05-09 16:44:15,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +13: [2023-05-09 16:44:15,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +28: [2023-05-09 16:44:15,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +20: [2023-05-09 16:44:15,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +28: [2023-05-09 16:44:15,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +11: [2023-05-09 16:44:15,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +24: [2023-05-09 16:44:15,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +24: [2023-05-09 16:44:15,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +24: [2023-05-09 16:44:15,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +24: [2023-05-09 16:44:15,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +12: [2023-05-09 16:44:15,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +27: [2023-05-09 16:44:15,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +11: [2023-05-09 16:44:15,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 5: [2023-05-09 16:44:15,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +12: [2023-05-09 16:44:15,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +18: [2023-05-09 16:44:15,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +12: [2023-05-09 16:44:15,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +12: [2023-05-09 16:44:15,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +16: [2023-05-09 16:44:15,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +16: [2023-05-09 16:44:15,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +16: [2023-05-09 16:44:15,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +16: [2023-05-09 16:44:15,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +16: [2023-05-09 16:44:15,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +16: [2023-05-09 16:44:15,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +16: [2023-05-09 16:44:15,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +16: [2023-05-09 16:44:15,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +26: [2023-05-09 16:44:15,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +26: [2023-05-09 16:44:15,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +12: [2023-05-09 16:44:15,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +12: [2023-05-09 16:44:15,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +11: [2023-05-09 16:44:15,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +11: [2023-05-09 16:44:15,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +11: [2023-05-09 16:44:15,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +11: [2023-05-09 16:44:15,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +28: [2023-05-09 16:44:15,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +18: [2023-05-09 16:44:15,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +18: [2023-05-09 16:44:15,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +27: [2023-05-09 16:44:15,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +23: [2023-05-09 16:44:15,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +27: [2023-05-09 16:44:15,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +23: [2023-05-09 16:44:15,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +18: [2023-05-09 16:44:15,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +26: [2023-05-09 16:44:15,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +18: [2023-05-09 16:44:15,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +18: [2023-05-09 16:44:15,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +24: [2023-05-09 16:44:15,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +24: [2023-05-09 16:44:15,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +23: [2023-05-09 16:44:15,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +27: [2023-05-09 16:44:15,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +23: [2023-05-09 16:44:15,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +26: [2023-05-09 16:44:15,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 5: [2023-05-09 16:44:15,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +23: [2023-05-09 16:44:15,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +23: [2023-05-09 16:44:15,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 8: [2023-05-09 16:44:15,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +12: [2023-05-09 16:44:15,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +12: [2023-05-09 16:44:15,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 6: [2023-05-09 16:44:15,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 6: [2023-05-09 16:44:15,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 8: [2023-05-09 16:44:15,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +10: [2023-05-09 16:44:15,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +10: [2023-05-09 16:44:15,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +10: [2023-05-09 16:44:15,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +13: [2023-05-09 16:44:15,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +22: [2023-05-09 16:44:15,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +10: [2023-05-09 16:44:15,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +10: [2023-05-09 16:44:15,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +22: [2023-05-09 16:44:15,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +22: [2023-05-09 16:44:15,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +22: [2023-05-09 16:44:15,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +10: [2023-05-09 16:44:15,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +13: [2023-05-09 16:44:15,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +16: [2023-05-09 16:44:15,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +22: [2023-05-09 16:44:15,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +22: [2023-05-09 16:44:15,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 6: [2023-05-09 16:44:15,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 6: [2023-05-09 16:44:15,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +13: [2023-05-09 16:44:15,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +13: [2023-05-09 16:44:15,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +11: [2023-05-09 16:44:15,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +11: [2023-05-09 16:44:15,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +11: [2023-05-09 16:44:15,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +16: [2023-05-09 16:44:15,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +16: [2023-05-09 16:44:15,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +13: [2023-05-09 16:44:15,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +13: [2023-05-09 16:44:15,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +13: [2023-05-09 16:44:15,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +13: [2023-05-09 16:44:15,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +11: [2023-05-09 16:44:15,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +16: [2023-05-09 16:44:15,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +18: [2023-05-09 16:44:15,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +23: [2023-05-09 16:44:15,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +23: [2023-05-09 16:44:15,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +18: [2023-05-09 16:44:15,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +24: [2023-05-09 16:44:15,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +24: [2023-05-09 16:44:15,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 8: [2023-05-09 16:44:15,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 5: [2023-05-09 16:44:15,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +22: [2023-05-09 16:44:15,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +22: [2023-05-09 16:44:15,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +22: [2023-05-09 16:44:15,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +10: [2023-05-09 16:44:15,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +10: [2023-05-09 16:44:15,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 9: [2023-05-09 16:44:15,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +22: [2023-05-09 16:44:15,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 9: [2023-05-09 16:44:15,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +22: [2023-05-09 16:44:15,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +22: [2023-05-09 16:44:15,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +11: [2023-05-09 16:44:15,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +11: [2023-05-09 16:44:15,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. + 5: [2023-05-09 16:44:15,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +18: [2023-05-09 16:44:15,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +22: [2023-05-09 16:44:15,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +18: [2023-05-09 16:44:15,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +15: [2023-05-09 16:44:15,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +15: [2023-05-09 16:44:15,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +18: [2023-05-09 16:44:15,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +15: [2023-05-09 16:44:15,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +18: [2023-05-09 16:44:15,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +15: [2023-05-09 16:44:15,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt. +22: [2023-05-09 16:44:15,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +11: [2023-05-09 16:44:15,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +22: [2023-05-09 16:44:15,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +11: [2023-05-09 16:44:15,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +22: [2023-05-09 16:44:15,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +16: [2023-05-09 16:44:15,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +16: [2023-05-09 16:44:15,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +26: [2023-05-09 16:44:15,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +18: [2023-05-09 16:44:15,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +18: [2023-05-09 16:44:15,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +26: [2023-05-09 16:44:15,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +16: [2023-05-09 16:44:15,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +16: [2023-05-09 16:44:15,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +26: [2023-05-09 16:44:15,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +26: [2023-05-09 16:44:15,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +21: [2023-05-09 16:44:15,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +21: [2023-05-09 16:44:15,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +21: [2023-05-09 16:44:15,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +21: [2023-05-09 16:44:15,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +21: [2023-05-09 16:44:15,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +21: [2023-05-09 16:44:15,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +26: [2023-05-09 16:44:15,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +26: [2023-05-09 16:44:15,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +26: [2023-05-09 16:44:15,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +22: [2023-05-09 16:44:15,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +26: [2023-05-09 16:44:15,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +15: [2023-05-09 16:44:15,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +15: [2023-05-09 16:44:15,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +15: [2023-05-09 16:44:15,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +21: [2023-05-09 16:44:15,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +21: [2023-05-09 16:44:15,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +15: [2023-05-09 16:44:15,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +22: [2023-05-09 16:44:15,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt... +21: [2023-05-09 16:44:15,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +21: [2023-05-09 16:44:15,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +15: [2023-05-09 16:44:15,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 0: [2023-05-09 16:44:15,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 0: [2023-05-09 16:44:15,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +15: [2023-05-09 16:44:15,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 0: [2023-05-09 16:44:15,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +10: [2023-05-09 16:44:15,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +10: [2023-05-09 16:44:15,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +10: [2023-05-09 16:44:15,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +10: [2023-05-09 16:44:15,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +10: [2023-05-09 16:44:15,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +14: [2023-05-09 16:44:15,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +22: [2023-05-09 16:44:15,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +14: [2023-05-09 16:44:15,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +15: [2023-05-09 16:44:15,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +15: [2023-05-09 16:44:15,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +14: [2023-05-09 16:44:15,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +10: [2023-05-09 16:44:15,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +15: [2023-05-09 16:44:15,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +15: [2023-05-09 16:44:15,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +22: [2023-05-09 16:44:15,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +22: [2023-05-09 16:44:15,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +14: [2023-05-09 16:44:15,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +10: [2023-05-09 16:44:15,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +10: [2023-05-09 16:44:15,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +14: [2023-05-09 16:44:15,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +14: [2023-05-09 16:44:15,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +17: [2023-05-09 16:44:15,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +17: [2023-05-09 16:44:15,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +17: [2023-05-09 16:44:15,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +27: [2023-05-09 16:44:15,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +17: [2023-05-09 16:44:15,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +27: [2023-05-09 16:44:15,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +27: [2023-05-09 16:44:15,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +27: [2023-05-09 16:44:15,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 9: [2023-05-09 16:44:15,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 9: [2023-05-09 16:44:15,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... + 2: [2023-05-09 16:44:15,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 2: [2023-05-09 16:44:15,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +14: [2023-05-09 16:44:15,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +14: [2023-05-09 16:44:15,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +12: [2023-05-09 16:44:15,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +12: [2023-05-09 16:44:15,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +20: [2023-05-09 16:44:15,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +20: [2023-05-09 16:44:15,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +21: [2023-05-09 16:44:15,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +12: [2023-05-09 16:44:15,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +15: [2023-05-09 16:44:15,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +21: [2023-05-09 16:44:15,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +21: [2023-05-09 16:44:15,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +20: [2023-05-09 16:44:15,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +20: [2023-05-09 16:44:15,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +12: [2023-05-09 16:44:15,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +12: [2023-05-09 16:44:15,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +12: [2023-05-09 16:44:15,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +12: [2023-05-09 16:44:15,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +12: [2023-05-09 16:44:15,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +15: [2023-05-09 16:44:15,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +14: [2023-05-09 16:44:15,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +27: [2023-05-09 16:44:15,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +22: [2023-05-09 16:44:15,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +22: [2023-05-09 16:44:15,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +17: [2023-05-09 16:44:15,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +22: [2023-05-09 16:44:15,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +17: [2023-05-09 16:44:15,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +27: [2023-05-09 16:44:15,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +17: [2023-05-09 16:44:15,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +17: [2023-05-09 16:44:15,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +27: [2023-05-09 16:44:15,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt. +21: [2023-05-09 16:44:15,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +27: [2023-05-09 16:44:15,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +21: [2023-05-09 16:44:15,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +20: [2023-05-09 16:44:15,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +20: [2023-05-09 16:44:15,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt... +14: [2023-05-09 16:44:15,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +20: [2023-05-09 16:44:15,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +12: [2023-05-09 16:44:15,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +20: [2023-05-09 16:44:15,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +10: [2023-05-09 16:44:15,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +14: [2023-05-09 16:44:15,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +10: [2023-05-09 16:44:15,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +17: [2023-05-09 16:44:15,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +17: [2023-05-09 16:44:15,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +21: [2023-05-09 16:44:15,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +27: [2023-05-09 16:44:15,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +27: [2023-05-09 16:44:15,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +14: [2023-05-09 16:44:15,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +25: [2023-05-09 16:44:15,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +25: [2023-05-09 16:44:15,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +25: [2023-05-09 16:44:15,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +25: [2023-05-09 16:44:15,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +29: [2023-05-09 16:44:15,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +29: [2023-05-09 16:44:15,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +29: [2023-05-09 16:44:15,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +29: [2023-05-09 16:44:15,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 2: [2023-05-09 16:44:15,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 2: [2023-05-09 16:44:15,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +15: [2023-05-09 16:44:15,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +15: [2023-05-09 16:44:15,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +10: [2023-05-09 16:44:15,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +10: [2023-05-09 16:44:15,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +10: [2023-05-09 16:44:15,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +17: [2023-05-09 16:44:15,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +15: [2023-05-09 16:44:15,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +17: [2023-05-09 16:44:15,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +14: [2023-05-09 16:44:15,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +25: [2023-05-09 16:44:15,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +17: [2023-05-09 16:44:15,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +14: [2023-05-09 16:44:15,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +17: [2023-05-09 16:44:15,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +17: [2023-05-09 16:44:15,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +17: [2023-05-09 16:44:15,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 0: [2023-05-09 16:44:15,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 2: [2023-05-09 16:44:15,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +25: [2023-05-09 16:44:15,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +25: [2023-05-09 16:44:15,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +25: [2023-05-09 16:44:15,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +17: [2023-05-09 16:44:15,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +17: [2023-05-09 16:44:15,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +27: [2023-05-09 16:44:15,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +15: [2023-05-09 16:44:15,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +15: [2023-05-09 16:44:15,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +27: [2023-05-09 16:44:15,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +12: [2023-05-09 16:44:15,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +15: [2023-05-09 16:44:15,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +10: [2023-05-09 16:44:15,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +17: [2023-05-09 16:44:15,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +17: [2023-05-09 16:44:15,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +17: [2023-05-09 16:44:15,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 0: [2023-05-09 16:44:15,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 0: [2023-05-09 16:44:15,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 0: [2023-05-09 16:44:15,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +27: [2023-05-09 16:44:15,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +27: [2023-05-09 16:44:15,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +27: [2023-05-09 16:44:15,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +27: [2023-05-09 16:44:15,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +15: [2023-05-09 16:44:15,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +17: [2023-05-09 16:44:15,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +20: [2023-05-09 16:44:15,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +12: [2023-05-09 16:44:15,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +15: [2023-05-09 16:44:15,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +20: [2023-05-09 16:44:15,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +14: [2023-05-09 16:44:15,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +12: [2023-05-09 16:44:15,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +29: [2023-05-09 16:44:15,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +29: [2023-05-09 16:44:15,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +29: [2023-05-09 16:44:15,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +17: [2023-05-09 16:44:15,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +17: [2023-05-09 16:44:15,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +14: [2023-05-09 16:44:15,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +10: [2023-05-09 16:44:15,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +10: [2023-05-09 16:44:15,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +20: [2023-05-09 16:44:15,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +20: [2023-05-09 16:44:15,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +29: [2023-05-09 16:44:15,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +20: [2023-05-09 16:44:15,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +20: [2023-05-09 16:44:15,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +20: [2023-05-09 16:44:15,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +20: [2023-05-09 16:44:15,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +17: [2023-05-09 16:44:15,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +15: [2023-05-09 16:44:15,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +12: [2023-05-09 16:44:15,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +12: [2023-05-09 16:44:15,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +17: [2023-05-09 16:44:15,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +12: [2023-05-09 16:44:15,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +30: [2023-05-09 16:44:15,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +30: [2023-05-09 16:44:15,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +30: [2023-05-09 16:44:15,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +30: [2023-05-09 16:44:15,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +17: [2023-05-09 16:44:15,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +30: [2023-05-09 16:44:15,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +30: [2023-05-09 16:44:15,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +29: [2023-05-09 16:44:15,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +13: [2023-05-09 16:44:15,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +13: [2023-05-09 16:44:15,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +13: [2023-05-09 16:44:15,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +13: [2023-05-09 16:44:15,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +13: [2023-05-09 16:44:15,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +13: [2023-05-09 16:44:15,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +29: [2023-05-09 16:44:15,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +29: [2023-05-09 16:44:15,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +29: [2023-05-09 16:44:15,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +30: [2023-05-09 16:44:15,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +30: [2023-05-09 16:44:15,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +15: [2023-05-09 16:44:15,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +13: [2023-05-09 16:44:15,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +17: [2023-05-09 16:44:15,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +13: [2023-05-09 16:44:15,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +31: [2023-05-09 16:44:15,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +31: [2023-05-09 16:44:15,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +31: [2023-05-09 16:44:15,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +31: [2023-05-09 16:44:15,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +29: [2023-05-09 16:44:15,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +29: [2023-05-09 16:44:15,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 4: [2023-05-09 16:44:15,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +25: [2023-05-09 16:44:15,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +12: [2023-05-09 16:44:15,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +17: [2023-05-09 16:44:15,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +29: [2023-05-09 16:44:15,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +28: [2023-05-09 16:44:15,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +29: [2023-05-09 16:44:15,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +28: [2023-05-09 16:44:15,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +28: [2023-05-09 16:44:15,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +28: [2023-05-09 16:44:15,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +17: [2023-05-09 16:44:15,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +21: [2023-05-09 16:44:15,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +21: [2023-05-09 16:44:15,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 3: [2023-05-09 16:44:15,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +15: [2023-05-09 16:44:15,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +30: [2023-05-09 16:44:15,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +21: [2023-05-09 16:44:15,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +25: [2023-05-09 16:44:15,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +21: [2023-05-09 16:44:15,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 3: [2023-05-09 16:44:15,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +25: [2023-05-09 16:44:15,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +25: [2023-05-09 16:44:15,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +31: [2023-05-09 16:44:15,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +13: [2023-05-09 16:44:15,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 1: [2023-05-09 16:44:15,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 1: [2023-05-09 16:44:15,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +30: [2023-05-09 16:44:15,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +25: [2023-05-09 16:44:15,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 1: [2023-05-09 16:44:15,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 4: [2023-05-09 16:44:15,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 3: [2023-05-09 16:44:15,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +31: [2023-05-09 16:44:15,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 4: [2023-05-09 16:44:15,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 4: [2023-05-09 16:44:15,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +17: [2023-05-09 16:44:15,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +31: [2023-05-09 16:44:15,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +31: [2023-05-09 16:44:15,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +25: [2023-05-09 16:44:15,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +15: [2023-05-09 16:44:15,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +25: [2023-05-09 16:44:15,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +25: [2023-05-09 16:44:15,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +25: [2023-05-09 16:44:15,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +15: [2023-05-09 16:44:15,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +13: [2023-05-09 16:44:15,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +28: [2023-05-09 16:44:15,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +28: [2023-05-09 16:44:15,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +28: [2023-05-09 16:44:15,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +30: [2023-05-09 16:44:15,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +28: [2023-05-09 16:44:15,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +13: [2023-05-09 16:44:15,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +25: [2023-05-09 16:44:15,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +15: [2023-05-09 16:44:15,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +30: [2023-05-09 16:44:15,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +21: [2023-05-09 16:44:15,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +21: [2023-05-09 16:44:15,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +21: [2023-05-09 16:44:15,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +21: [2023-05-09 16:44:15,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +17: [2023-05-09 16:44:15,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +31: [2023-05-09 16:44:15,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +31: [2023-05-09 16:44:15,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +25: [2023-05-09 16:44:15,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +15: [2023-05-09 16:44:15,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +25: [2023-05-09 16:44:15,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +13: [2023-05-09 16:44:15,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +15: [2023-05-09 16:44:15,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +25: [2023-05-09 16:44:15,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +25: [2023-05-09 16:44:15,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +28: [2023-05-09 16:44:15,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +30: [2023-05-09 16:44:15,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +28: [2023-05-09 16:44:15,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 4: [2023-05-09 16:44:15,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +25: [2023-05-09 16:44:15,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +25: [2023-05-09 16:44:15,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +30: [2023-05-09 16:44:15,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +13: [2023-05-09 16:44:15,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +13: [2023-05-09 16:44:15,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +31: [2023-05-09 16:44:15,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +31: [2023-05-09 16:44:15,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +21: [2023-05-09 16:44:15,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +21: [2023-05-09 16:44:15,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +23: [2023-05-09 16:44:15,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +20: [2023-05-09 16:44:15,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +23: [2023-05-09 16:44:15,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +20: [2023-05-09 16:44:15,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +20: [2023-05-09 16:44:15,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +23: [2023-05-09 16:44:15,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +20: [2023-05-09 16:44:15,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +23: [2023-05-09 16:44:15,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +20: [2023-05-09 16:44:15,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +23: [2023-05-09 16:44:15,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +21: [2023-05-09 16:44:15,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 7: [2023-05-09 16:44:15,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +21: [2023-05-09 16:44:15,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +23: [2023-05-09 16:44:15,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +20: [2023-05-09 16:44:15,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +23: [2023-05-09 16:44:15,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +20: [2023-05-09 16:44:15,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +19: [2023-05-09 16:44:15,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +19: [2023-05-09 16:44:15,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +19: [2023-05-09 16:44:15,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +19: [2023-05-09 16:44:15,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +30: [2023-05-09 16:44:15,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +31: [2023-05-09 16:44:15,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +31: [2023-05-09 16:44:15,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +31: [2023-05-09 16:44:15,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +31: [2023-05-09 16:44:15,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +20: [2023-05-09 16:44:15,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +28: [2023-05-09 16:44:15,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +28: [2023-05-09 16:44:15,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +28: [2023-05-09 16:44:15,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +28: [2023-05-09 16:44:15,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 2: [2023-05-09 16:44:15,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +19: [2023-05-09 16:44:15,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +19: [2023-05-09 16:44:15,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +19: [2023-05-09 16:44:15,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +23: [2023-05-09 16:44:15,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +19: [2023-05-09 16:44:15,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +25: [2023-05-09 16:44:15,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +30: [2023-05-09 16:44:15,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +25: [2023-05-09 16:44:15,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 1: [2023-05-09 16:44:15,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 3: [2023-05-09 16:44:15,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +21: [2023-05-09 16:44:15,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +21: [2023-05-09 16:44:15,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +21: [2023-05-09 16:44:15,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +21: [2023-05-09 16:44:15,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +24: [2023-05-09 16:44:15,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 1: [2023-05-09 16:44:15,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 2: [2023-05-09 16:44:15,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +24: [2023-05-09 16:44:15,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +24: [2023-05-09 16:44:15,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +24: [2023-05-09 16:44:15,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +24: [2023-05-09 16:44:15,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +24: [2023-05-09 16:44:15,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +13: [2023-05-09 16:44:15,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +13: [2023-05-09 16:44:15,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +28: [2023-05-09 16:44:15,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +28: [2023-05-09 16:44:15,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 8: [2023-05-09 16:44:15,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +24: [2023-05-09 16:44:15,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +24: [2023-05-09 16:44:15,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +25: [2023-05-09 16:44:15,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +25: [2023-05-09 16:44:15,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +25: [2023-05-09 16:44:15,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +25: [2023-05-09 16:44:15,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 9: [2023-05-09 16:44:15,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +29: [2023-05-09 16:44:15,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +29: [2023-05-09 16:44:15,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +29: [2023-05-09 16:44:15,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +29: [2023-05-09 16:44:15,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +29: [2023-05-09 16:44:15,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +29: [2023-05-09 16:44:15,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +29: [2023-05-09 16:44:15,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +29: [2023-05-09 16:44:15,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +20: [2023-05-09 16:44:15,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 9: [2023-05-09 16:44:15,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +23: [2023-05-09 16:44:15,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +23: [2023-05-09 16:44:15,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +25: [2023-05-09 16:44:15,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +25: [2023-05-09 16:44:15,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 2: [2023-05-09 16:44:15,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +20: [2023-05-09 16:44:15,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +20: [2023-05-09 16:44:15,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +23: [2023-05-09 16:44:15,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +31: [2023-05-09 16:44:15,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +31: [2023-05-09 16:44:15,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +31: [2023-05-09 16:44:15,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +31: [2023-05-09 16:44:15,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +31: [2023-05-09 16:44:15,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +31: [2023-05-09 16:44:15,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +31: [2023-05-09 16:44:15,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +31: [2023-05-09 16:44:15,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 4: [2023-05-09 16:44:15,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 8: [2023-05-09 16:44:15,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +29: [2023-05-09 16:44:15,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +20: [2023-05-09 16:44:15,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +27: [2023-05-09 16:44:15,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +29: [2023-05-09 16:44:15,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +27: [2023-05-09 16:44:15,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +27: [2023-05-09 16:44:15,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +27: [2023-05-09 16:44:15,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +27: [2023-05-09 16:44:15,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +27: [2023-05-09 16:44:15,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +23: [2023-05-09 16:44:15,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +27: [2023-05-09 16:44:15,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +23: [2023-05-09 16:44:15,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +23: [2023-05-09 16:44:15,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +20: [2023-05-09 16:44:15,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +29: [2023-05-09 16:44:15,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +24: [2023-05-09 16:44:15,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +24: [2023-05-09 16:44:15,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +29: [2023-05-09 16:44:15,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +27: [2023-05-09 16:44:15,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +20: [2023-05-09 16:44:15,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +19: [2023-05-09 16:44:15,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +31: [2023-05-09 16:44:15,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +31: [2023-05-09 16:44:15,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +31: [2023-05-09 16:44:15,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +20: [2023-05-09 16:44:15,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +31: [2023-05-09 16:44:15,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +19: [2023-05-09 16:44:15,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +19: [2023-05-09 16:44:15,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +19: [2023-05-09 16:44:15,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +29: [2023-05-09 16:44:15,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +23: [2023-05-09 16:44:15,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +29: [2023-05-09 16:44:15,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +29: [2023-05-09 16:44:15,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +24: [2023-05-09 16:44:15,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +24: [2023-05-09 16:44:15,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +27: [2023-05-09 16:44:15,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +27: [2023-05-09 16:44:15,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +24: [2023-05-09 16:44:15,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +27: [2023-05-09 16:44:15,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +29: [2023-05-09 16:44:15,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +11: [2023-05-09 16:44:15,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +11: [2023-05-09 16:44:15,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +11: [2023-05-09 16:44:15,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +11: [2023-05-09 16:44:15,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +11: [2023-05-09 16:44:15,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +11: [2023-05-09 16:44:15,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +20: [2023-05-09 16:44:15,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +19: [2023-05-09 16:44:15,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +11: [2023-05-09 16:44:15,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +31: [2023-05-09 16:44:15,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +31: [2023-05-09 16:44:15,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +31: [2023-05-09 16:44:15,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +19: [2023-05-09 16:44:15,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +31: [2023-05-09 16:44:15,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +19: [2023-05-09 16:44:15,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +19: [2023-05-09 16:44:15,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +23: [2023-05-09 16:44:15,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +11: [2023-05-09 16:44:15,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +18: [2023-05-09 16:44:15,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +24: [2023-05-09 16:44:15,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +18: [2023-05-09 16:44:15,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +27: [2023-05-09 16:44:15,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +26: [2023-05-09 16:44:15,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +26: [2023-05-09 16:44:15,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +24: [2023-05-09 16:44:15,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +26: [2023-05-09 16:44:15,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +26: [2023-05-09 16:44:15,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +18: [2023-05-09 16:44:15,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +18: [2023-05-09 16:44:15,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +18: [2023-05-09 16:44:15,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +18: [2023-05-09 16:44:15,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +27: [2023-05-09 16:44:15,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +27: [2023-05-09 16:44:15,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +28: [2023-05-09 16:44:15,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +28: [2023-05-09 16:44:15,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +28: [2023-05-09 16:44:15,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +28: [2023-05-09 16:44:15,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +28: [2023-05-09 16:44:15,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +28: [2023-05-09 16:44:15,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +18: [2023-05-09 16:44:15,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +18: [2023-05-09 16:44:15,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +28: [2023-05-09 16:44:15,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +26: [2023-05-09 16:44:15,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +26: [2023-05-09 16:44:15,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +28: [2023-05-09 16:44:15,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +26: [2023-05-09 16:44:15,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +26: [2023-05-09 16:44:15,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +27: [2023-05-09 16:44:15,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +24: [2023-05-09 16:44:15,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +24: [2023-05-09 16:44:15,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +24: [2023-05-09 16:44:15,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +24: [2023-05-09 16:44:15,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +24: [2023-05-09 16:44:15,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +11: [2023-05-09 16:44:15,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +11: [2023-05-09 16:44:15,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +26: [2023-05-09 16:44:15,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +26: [2023-05-09 16:44:15,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +28: [2023-05-09 16:44:15,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +15: [2023-05-09 16:44:15,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +15: [2023-05-09 16:44:15,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +15: [2023-05-09 16:44:15,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +15: [2023-05-09 16:44:15,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +27: [2023-05-09 16:44:15,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +28: [2023-05-09 16:44:15,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +28: [2023-05-09 16:44:15,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +24: [2023-05-09 16:44:15,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +24: [2023-05-09 16:44:15,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +24: [2023-05-09 16:44:15,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +24: [2023-05-09 16:44:15,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +11: [2023-05-09 16:44:15,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +11: [2023-05-09 16:44:15,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +11: [2023-05-09 16:44:15,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +11: [2023-05-09 16:44:15,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 1: [2023-05-09 16:44:15,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +18: [2023-05-09 16:44:15,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +11: [2023-05-09 16:44:15,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +18: [2023-05-09 16:44:15,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +15: [2023-05-09 16:44:15,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +11: [2023-05-09 16:44:15,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +15: [2023-05-09 16:44:15,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +15: [2023-05-09 16:44:15,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +28: [2023-05-09 16:44:15,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +30: [2023-05-09 16:44:15,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +30: [2023-05-09 16:44:15,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +30: [2023-05-09 16:44:15,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +30: [2023-05-09 16:44:15,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +15: [2023-05-09 16:44:15,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +11: [2023-05-09 16:44:15,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +18: [2023-05-09 16:44:15,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +18: [2023-05-09 16:44:15,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +28: [2023-05-09 16:44:15,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +11: [2023-05-09 16:44:15,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +28: [2023-05-09 16:44:15,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +24: [2023-05-09 16:44:15,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +24: [2023-05-09 16:44:15,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +28: [2023-05-09 16:44:15,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +26: [2023-05-09 16:44:15,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +24: [2023-05-09 16:44:15,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +24: [2023-05-09 16:44:15,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +26: [2023-05-09 16:44:15,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +11: [2023-05-09 16:44:15,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +24: [2023-05-09 16:44:15,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +24: [2023-05-09 16:44:15,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +11: [2023-05-09 16:44:15,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +11: [2023-05-09 16:44:15,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +18: [2023-05-09 16:44:15,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +18: [2023-05-09 16:44:15,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +30: [2023-05-09 16:44:15,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +26: [2023-05-09 16:44:15,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +18: [2023-05-09 16:44:15,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +24: [2023-05-09 16:44:15,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +18: [2023-05-09 16:44:15,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +24: [2023-05-09 16:44:15,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +30: [2023-05-09 16:44:15,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +28: [2023-05-09 16:44:15,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +11: [2023-05-09 16:44:15,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +30: [2023-05-09 16:44:15,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +30: [2023-05-09 16:44:15,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +15: [2023-05-09 16:44:15,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +26: [2023-05-09 16:44:15,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +15: [2023-05-09 16:44:15,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +15: [2023-05-09 16:44:15,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +15: [2023-05-09 16:44:15,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +11: [2023-05-09 16:44:15,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +11: [2023-05-09 16:44:15,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +11: [2023-05-09 16:44:15,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 3: [2023-05-09 16:44:15,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 3: [2023-05-09 16:44:15,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 3: [2023-05-09 16:44:15,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 3: [2023-05-09 16:44:15,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +15: [2023-05-09 16:44:15,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +15: [2023-05-09 16:44:15,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +11: [2023-05-09 16:44:15,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +11: [2023-05-09 16:44:15,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +15: [2023-05-09 16:44:15,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +15: [2023-05-09 16:44:15,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +26: [2023-05-09 16:44:15,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 9: [2023-05-09 16:44:15,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +26: [2023-05-09 16:44:15,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +30: [2023-05-09 16:44:15,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +30: [2023-05-09 16:44:15,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 9: [2023-05-09 16:44:15,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +11: [2023-05-09 16:44:15,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +11: [2023-05-09 16:44:15,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +11: [2023-05-09 16:44:15,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. +11: [2023-05-09 16:44:15,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 9: [2023-05-09 16:44:15,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. +11: [2023-05-09 16:44:15,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 7: [2023-05-09 16:44:15,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 7: [2023-05-09 16:44:15,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 7: [2023-05-09 16:44:15,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt. + 9: [2023-05-09 16:44:15,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... +30: [2023-05-09 16:44:15,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +30: [2023-05-09 16:44:15,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +30: [2023-05-09 16:44:15,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +30: [2023-05-09 16:44:15,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +30: [2023-05-09 16:44:15,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +30: [2023-05-09 16:44:15,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 3: [2023-05-09 16:44:15,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 7: [2023-05-09 16:44:15,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 9: [2023-05-09 16:44:15,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 9: [2023-05-09 16:44:15,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 9: [2023-05-09 16:44:15,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 9: [2023-05-09 16:44:15,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 9: [2023-05-09 16:44:15,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 3: [2023-05-09 16:44:15,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 3: [2023-05-09 16:44:15,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 7: [2023-05-09 16:44:15,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 7: [2023-05-09 16:44:15,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... + 3: [2023-05-09 16:44:15,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 7: [2023-05-09 16:44:15,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt... +10: [2023-05-09 16:44:15,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +10: [2023-05-09 16:44:15,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +10: [2023-05-09 16:44:15,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +10: [2023-05-09 16:44:15,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +10: [2023-05-09 16:44:15,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +10: [2023-05-09 16:44:15,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +23: [2023-05-09 16:44:15,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +23: [2023-05-09 16:44:15,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +23: [2023-05-09 16:44:15,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +23: [2023-05-09 16:44:15,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +10: [2023-05-09 16:44:15,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +10: [2023-05-09 16:44:15,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +23: [2023-05-09 16:44:15,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +23: [2023-05-09 16:44:15,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +10: [2023-05-09 16:44:15,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +10: [2023-05-09 16:44:15,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +23: [2023-05-09 16:44:15,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +23: [2023-05-09 16:44:15,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +16: [2023-05-09 16:44:15,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +16: [2023-05-09 16:44:15,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +16: [2023-05-09 16:44:15,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +13: [2023-05-09 16:44:15,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +13: [2023-05-09 16:44:15,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +13: [2023-05-09 16:44:15,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +16: [2023-05-09 16:44:15,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +13: [2023-05-09 16:44:15,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +23: [2023-05-09 16:44:15,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +23: [2023-05-09 16:44:15,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +13: [2023-05-09 16:44:15,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +16: [2023-05-09 16:44:15,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +13: [2023-05-09 16:44:15,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +13: [2023-05-09 16:44:15,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +10: [2023-05-09 16:44:15,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +16: [2023-05-09 16:44:15,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +13: [2023-05-09 16:44:15,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +10: [2023-05-09 16:44:15,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +10: [2023-05-09 16:44:15,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +10: [2023-05-09 16:44:15,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +16: [2023-05-09 16:44:15,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +10: [2023-05-09 16:44:15,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +16: [2023-05-09 16:44:15,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +10: [2023-05-09 16:44:15,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +12: [2023-05-09 16:44:15,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +12: [2023-05-09 16:44:15,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +12: [2023-05-09 16:44:15,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +12: [2023-05-09 16:44:15,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +23: [2023-05-09 16:44:15,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +23: [2023-05-09 16:44:15,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +23: [2023-05-09 16:44:15,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +23: [2023-05-09 16:44:15,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +16: [2023-05-09 16:44:15,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +23: [2023-05-09 16:44:15,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +23: [2023-05-09 16:44:15,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +16: [2023-05-09 16:44:15,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +13: [2023-05-09 16:44:15,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +12: [2023-05-09 16:44:15,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +13: [2023-05-09 16:44:15,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +12: [2023-05-09 16:44:15,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +13: [2023-05-09 16:44:15,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +12: [2023-05-09 16:44:15,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +13: [2023-05-09 16:44:15,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +12: [2023-05-09 16:44:15,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +13: [2023-05-09 16:44:15,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +13: [2023-05-09 16:44:15,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +16: [2023-05-09 16:44:15,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +16: [2023-05-09 16:44:15,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +12: [2023-05-09 16:44:15,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +12: [2023-05-09 16:44:15,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +16: [2023-05-09 16:44:15,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 5: [2023-05-09 16:44:15,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 5: [2023-05-09 16:44:15,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +13: [2023-05-09 16:44:15,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +16: [2023-05-09 16:44:15,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +12: [2023-05-09 16:44:15,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +12: [2023-05-09 16:44:15,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +12: [2023-05-09 16:44:15,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +12: [2023-05-09 16:44:15,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +13: [2023-05-09 16:44:15,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +22: [2023-05-09 16:44:15,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +22: [2023-05-09 16:44:15,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +18: [2023-05-09 16:44:15,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +22: [2023-05-09 16:44:15,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +18: [2023-05-09 16:44:15,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +18: [2023-05-09 16:44:15,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +22: [2023-05-09 16:44:15,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +18: [2023-05-09 16:44:15,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +26: [2023-05-09 16:44:15,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +26: [2023-05-09 16:44:15,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +26: [2023-05-09 16:44:15,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +26: [2023-05-09 16:44:15,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +16: [2023-05-09 16:44:15,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +16: [2023-05-09 16:44:15,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 5: [2023-05-09 16:44:15,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +18: [2023-05-09 16:44:15,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 8: [2023-05-09 16:44:15,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 8: [2023-05-09 16:44:15,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 8: [2023-05-09 16:44:15,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +18: [2023-05-09 16:44:15,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +22: [2023-05-09 16:44:15,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +22: [2023-05-09 16:44:15,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +18: [2023-05-09 16:44:15,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +18: [2023-05-09 16:44:15,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +26: [2023-05-09 16:44:15,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +12: [2023-05-09 16:44:15,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +26: [2023-05-09 16:44:15,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +12: [2023-05-09 16:44:15,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +26: [2023-05-09 16:44:15,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +22: [2023-05-09 16:44:15,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +22: [2023-05-09 16:44:15,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +26: [2023-05-09 16:44:15,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +18: [2023-05-09 16:44:15,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +18: [2023-05-09 16:44:15,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 8: [2023-05-09 16:44:15,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 5: [2023-05-09 16:44:15,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 5: [2023-05-09 16:44:15,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +26: [2023-05-09 16:44:15,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +26: [2023-05-09 16:44:15,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +22: [2023-05-09 16:44:15,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +22: [2023-05-09 16:44:15,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 6: [2023-05-09 16:44:15,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 6: [2023-05-09 16:44:15,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 6: [2023-05-09 16:44:15,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 6: [2023-05-09 16:44:15,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +22: [2023-05-09 16:44:15,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +22: [2023-05-09 16:44:15,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +17: [2023-05-09 16:44:15,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +17: [2023-05-09 16:44:15,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +17: [2023-05-09 16:44:15,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +17: [2023-05-09 16:44:15,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 8: [2023-05-09 16:44:15,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 5: [2023-05-09 16:44:15,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 5: [2023-05-09 16:44:15,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +26: [2023-05-09 16:44:15,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +26: [2023-05-09 16:44:15,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 5: [2023-05-09 16:44:15,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 5: [2023-05-09 16:44:15,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +18: [2023-05-09 16:44:15,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +18: [2023-05-09 16:44:15,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 2: [2023-05-09 16:44:15,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 2: [2023-05-09 16:44:15,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 2: [2023-05-09 16:44:15,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 2: [2023-05-09 16:44:15,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 6: [2023-05-09 16:44:15,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +18: [2023-05-09 16:44:15,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +18: [2023-05-09 16:44:15,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +17: [2023-05-09 16:44:15,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +18: [2023-05-09 16:44:15,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +18: [2023-05-09 16:44:15,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +17: [2023-05-09 16:44:15,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +17: [2023-05-09 16:44:15,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +17: [2023-05-09 16:44:15,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 8: [2023-05-09 16:44:15,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +22: [2023-05-09 16:44:15,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +26: [2023-05-09 16:44:15,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +26: [2023-05-09 16:44:15,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 8: [2023-05-09 16:44:15,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 8: [2023-05-09 16:44:15,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +22: [2023-05-09 16:44:15,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +22: [2023-05-09 16:44:15,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 8: [2023-05-09 16:44:15,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +26: [2023-05-09 16:44:15,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +26: [2023-05-09 16:44:15,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 2: [2023-05-09 16:44:15,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +22: [2023-05-09 16:44:15,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 6: [2023-05-09 16:44:15,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 2: [2023-05-09 16:44:15,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 6: [2023-05-09 16:44:15,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 6: [2023-05-09 16:44:15,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +17: [2023-05-09 16:44:15,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +17: [2023-05-09 16:44:15,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +17: [2023-05-09 16:44:15,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +17: [2023-05-09 16:44:15,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 6: [2023-05-09 16:44:15,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 0: [2023-05-09 16:44:15,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 0: [2023-05-09 16:44:15,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 0: [2023-05-09 16:44:15,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +17: [2023-05-09 16:44:15,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 6: [2023-05-09 16:44:15,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 2: [2023-05-09 16:44:15,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 2: [2023-05-09 16:44:15,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 2: [2023-05-09 16:44:15,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 2: [2023-05-09 16:44:15,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +17: [2023-05-09 16:44:15,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +17: [2023-05-09 16:44:15,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +17: [2023-05-09 16:44:15,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 2: [2023-05-09 16:44:15,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 2: [2023-05-09 16:44:15,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 0: [2023-05-09 16:44:15,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +27: [2023-05-09 16:44:15,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +27: [2023-05-09 16:44:15,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 0: [2023-05-09 16:44:15,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +27: [2023-05-09 16:44:15,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +27: [2023-05-09 16:44:15,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +14: [2023-05-09 16:44:15,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +14: [2023-05-09 16:44:15,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +14: [2023-05-09 16:44:15,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +14: [2023-05-09 16:44:15,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +27: [2023-05-09 16:44:15,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +27: [2023-05-09 16:44:15,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +14: [2023-05-09 16:44:15,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +27: [2023-05-09 16:44:15,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +27: [2023-05-09 16:44:15,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +14: [2023-05-09 16:44:15,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +14: [2023-05-09 16:44:15,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +14: [2023-05-09 16:44:15,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 0: [2023-05-09 16:44:15,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 0: [2023-05-09 16:44:15,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 3: [2023-05-09 16:44:15,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 3: [2023-05-09 16:44:15,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 3: [2023-05-09 16:44:15,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +14: [2023-05-09 16:44:15,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 3: [2023-05-09 16:44:15,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +14: [2023-05-09 16:44:15,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +27: [2023-05-09 16:44:15,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +27: [2023-05-09 16:44:15,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +27: [2023-05-09 16:44:15,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +27: [2023-05-09 16:44:15,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +27: [2023-05-09 16:44:15,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +27: [2023-05-09 16:44:15,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +27: [2023-05-09 16:44:15,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +27: [2023-05-09 16:44:15,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 1: [2023-05-09 16:44:15,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 1: [2023-05-09 16:44:15,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 1: [2023-05-09 16:44:15,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +11: [2023-05-09 16:44:15,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +11: [2023-05-09 16:44:15,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 1: [2023-05-09 16:44:15,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +11: [2023-05-09 16:44:15,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +11: [2023-05-09 16:44:15,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +14: [2023-05-09 16:44:15,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +14: [2023-05-09 16:44:15,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +14: [2023-05-09 16:44:15,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +14: [2023-05-09 16:44:15,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 3: [2023-05-09 16:44:15,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 3: [2023-05-09 16:44:15,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +11: [2023-05-09 16:44:15,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +11: [2023-05-09 16:44:15,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +11: [2023-05-09 16:44:15,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +31: [2023-05-09 16:44:15,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +11: [2023-05-09 16:44:15,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +31: [2023-05-09 16:44:15,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +31: [2023-05-09 16:44:15,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +31: [2023-05-09 16:44:15,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +14: [2023-05-09 16:44:15,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +25: [2023-05-09 16:44:15,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +25: [2023-05-09 16:44:15,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +25: [2023-05-09 16:44:15,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +25: [2023-05-09 16:44:15,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +14: [2023-05-09 16:44:15,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +15: [2023-05-09 16:44:15,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +15: [2023-05-09 16:44:15,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +15: [2023-05-09 16:44:15,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +15: [2023-05-09 16:44:15,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +11: [2023-05-09 16:44:15,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +11: [2023-05-09 16:44:15,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +31: [2023-05-09 16:44:15,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +31: [2023-05-09 16:44:15,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +31: [2023-05-09 16:44:15,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +31: [2023-05-09 16:44:15,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +25: [2023-05-09 16:44:15,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +25: [2023-05-09 16:44:15,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +25: [2023-05-09 16:44:15,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +25: [2023-05-09 16:44:15,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +15: [2023-05-09 16:44:15,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +15: [2023-05-09 16:44:15,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +15: [2023-05-09 16:44:15,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +15: [2023-05-09 16:44:15,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +19: [2023-05-09 16:44:15,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +19: [2023-05-09 16:44:15,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +19: [2023-05-09 16:44:15,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +19: [2023-05-09 16:44:15,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +28: [2023-05-09 16:44:15,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +28: [2023-05-09 16:44:15,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +28: [2023-05-09 16:44:15,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +28: [2023-05-09 16:44:15,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +23: [2023-05-09 16:44:15,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +23: [2023-05-09 16:44:15,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +23: [2023-05-09 16:44:15,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +23: [2023-05-09 16:44:15,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +11: [2023-05-09 16:44:15,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +21: [2023-05-09 16:44:15,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +21: [2023-05-09 16:44:15,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +21: [2023-05-09 16:44:15,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +21: [2023-05-09 16:44:15,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +25: [2023-05-09 16:44:15,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +25: [2023-05-09 16:44:15,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +11: [2023-05-09 16:44:15,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 9: [2023-05-09 16:44:15,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 9: [2023-05-09 16:44:15,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 9: [2023-05-09 16:44:15,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +20: [2023-05-09 16:44:15,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +20: [2023-05-09 16:44:15,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +20: [2023-05-09 16:44:15,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 9: [2023-05-09 16:44:15,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +20: [2023-05-09 16:44:15,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +29: [2023-05-09 16:44:15,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 7: [2023-05-09 16:44:15,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 7: [2023-05-09 16:44:15,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 7: [2023-05-09 16:44:15,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +29: [2023-05-09 16:44:15,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +29: [2023-05-09 16:44:15,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 7: [2023-05-09 16:44:15,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +29: [2023-05-09 16:44:15,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +11: [2023-05-09 16:44:15,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +11: [2023-05-09 16:44:15,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +24: [2023-05-09 16:44:15,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +24: [2023-05-09 16:44:15,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +19: [2023-05-09 16:44:15,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +24: [2023-05-09 16:44:15,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +24: [2023-05-09 16:44:15,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 1: [2023-05-09 16:44:15,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 1: [2023-05-09 16:44:15,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +11: [2023-05-09 16:44:15,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 1: [2023-05-09 16:44:15,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +11: [2023-05-09 16:44:15,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +19: [2023-05-09 16:44:15,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +28: [2023-05-09 16:44:15,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 1: [2023-05-09 16:44:15,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 1: [2023-05-09 16:44:15,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +31: [2023-05-09 16:44:15,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +19: [2023-05-09 16:44:15,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +19: [2023-05-09 16:44:15,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +31: [2023-05-09 16:44:15,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +23: [2023-05-09 16:44:15,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +23: [2023-05-09 16:44:15,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +28: [2023-05-09 16:44:15,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +21: [2023-05-09 16:44:15,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +23: [2023-05-09 16:44:15,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +23: [2023-05-09 16:44:15,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +25: [2023-05-09 16:44:15,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +25: [2023-05-09 16:44:15,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +28: [2023-05-09 16:44:15,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +28: [2023-05-09 16:44:15,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +25: [2023-05-09 16:44:15,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +25: [2023-05-09 16:44:15,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +15: [2023-05-09 16:44:15,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +15: [2023-05-09 16:44:15,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +15: [2023-05-09 16:44:15,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +15: [2023-05-09 16:44:15,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +15: [2023-05-09 16:44:15,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +15: [2023-05-09 16:44:15,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +15: [2023-05-09 16:44:15,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +15: [2023-05-09 16:44:15,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +21: [2023-05-09 16:44:15,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +21: [2023-05-09 16:44:15,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +21: [2023-05-09 16:44:15,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +29: [2023-05-09 16:44:15,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +29: [2023-05-09 16:44:15,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +31: [2023-05-09 16:44:15,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +20: [2023-05-09 16:44:15,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +29: [2023-05-09 16:44:15,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +29: [2023-05-09 16:44:15,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +31: [2023-05-09 16:44:15,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +20: [2023-05-09 16:44:15,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +20: [2023-05-09 16:44:15,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +24: [2023-05-09 16:44:15,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +24: [2023-05-09 16:44:15,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +24: [2023-05-09 16:44:15,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 7: [2023-05-09 16:44:15,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +20: [2023-05-09 16:44:15,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +31: [2023-05-09 16:44:15,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +31: [2023-05-09 16:44:15,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +19: [2023-05-09 16:44:15,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 4: [2023-05-09 16:44:15,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 4: [2023-05-09 16:44:15,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. + 4: [2023-05-09 16:44:15,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +19: [2023-05-09 16:44:15,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +24: [2023-05-09 16:44:15,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt. +31: [2023-05-09 16:44:15,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +31: [2023-05-09 16:44:15,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +28: [2023-05-09 16:44:15,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +28: [2023-05-09 16:44:15,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +25: [2023-05-09 16:44:15,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +21: [2023-05-09 16:44:15,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +21: [2023-05-09 16:44:15,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +25: [2023-05-09 16:44:15,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +29: [2023-05-09 16:44:15,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +21: [2023-05-09 16:44:15,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +29: [2023-05-09 16:44:15,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +21: [2023-05-09 16:44:15,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +28: [2023-05-09 16:44:15,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +28: [2023-05-09 16:44:15,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +29: [2023-05-09 16:44:15,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +20: [2023-05-09 16:44:15,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +29: [2023-05-09 16:44:15,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 7: [2023-05-09 16:44:15,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +24: [2023-05-09 16:44:15,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +20: [2023-05-09 16:44:15,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 7: [2023-05-09 16:44:15,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 9: [2023-05-09 16:44:15,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +24: [2023-05-09 16:44:15,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +24: [2023-05-09 16:44:15,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +24: [2023-05-09 16:44:15,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 7: [2023-05-09 16:44:15,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +19: [2023-05-09 16:44:15,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +24: [2023-05-09 16:44:15,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +24: [2023-05-09 16:44:15,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +19: [2023-05-09 16:44:15,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +21: [2023-05-09 16:44:15,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +21: [2023-05-09 16:44:15,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +16: [2023-05-09 16:44:15,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +16: [2023-05-09 16:44:15,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +16: [2023-05-09 16:44:15,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +16: [2023-05-09 16:44:15,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +20: [2023-05-09 16:44:15,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +20: [2023-05-09 16:44:15,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +20: [2023-05-09 16:44:15,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +20: [2023-05-09 16:44:15,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 4: [2023-05-09 16:44:15,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +16: [2023-05-09 16:44:15,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +16: [2023-05-09 16:44:15,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +16: [2023-05-09 16:44:15,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +16: [2023-05-09 16:44:15,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +28: [2023-05-09 16:44:15,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +28: [2023-05-09 16:44:15,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 4: [2023-05-09 16:44:15,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +29: [2023-05-09 16:44:15,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +29: [2023-05-09 16:44:15,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 0: [2023-05-09 16:44:15,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 0: [2023-05-09 16:44:15,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 0: [2023-05-09 16:44:15,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +21: [2023-05-09 16:44:15,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +21: [2023-05-09 16:44:15,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +21: [2023-05-09 16:44:15,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +21: [2023-05-09 16:44:15,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +19: [2023-05-09 16:44:15,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +19: [2023-05-09 16:44:15,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +21: [2023-05-09 16:44:15,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +21: [2023-05-09 16:44:15,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +21: [2023-05-09 16:44:15,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 4: [2023-05-09 16:44:15,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt... +28: [2023-05-09 16:44:15,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +13: [2023-05-09 16:44:15,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +13: [2023-05-09 16:44:15,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +21: [2023-05-09 16:44:15,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +13: [2023-05-09 16:44:15,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +13: [2023-05-09 16:44:15,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +19: [2023-05-09 16:44:15,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +19: [2023-05-09 16:44:15,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +13: [2023-05-09 16:44:15,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +13: [2023-05-09 16:44:15,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +13: [2023-05-09 16:44:15,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +13: [2023-05-09 16:44:15,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +28: [2023-05-09 16:44:15,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +12: [2023-05-09 16:44:15,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +12: [2023-05-09 16:44:15,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +14: [2023-05-09 16:44:15,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +12: [2023-05-09 16:44:15,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +12: [2023-05-09 16:44:15,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +14: [2023-05-09 16:44:15,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +12: [2023-05-09 16:44:15,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +14: [2023-05-09 16:44:15,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +14: [2023-05-09 16:44:15,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +14: [2023-05-09 16:44:15,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +14: [2023-05-09 16:44:15,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +14: [2023-05-09 16:44:15,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +12: [2023-05-09 16:44:15,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +12: [2023-05-09 16:44:15,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +29: [2023-05-09 16:44:15,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +21: [2023-05-09 16:44:15,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +12: [2023-05-09 16:44:15,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +14: [2023-05-09 16:44:15,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 9: [2023-05-09 16:44:15,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +10: [2023-05-09 16:44:15,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +10: [2023-05-09 16:44:15,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +30: [2023-05-09 16:44:15,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +30: [2023-05-09 16:44:15,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +30: [2023-05-09 16:44:15,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +10: [2023-05-09 16:44:15,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +10: [2023-05-09 16:44:15,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +10: [2023-05-09 16:44:15,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +10: [2023-05-09 16:44:15,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +30: [2023-05-09 16:44:15,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +10: [2023-05-09 16:44:15,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +21: [2023-05-09 16:44:15,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +29: [2023-05-09 16:44:15,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +16: [2023-05-09 16:44:15,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +16: [2023-05-09 16:44:15,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +16: [2023-05-09 16:44:15,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +16: [2023-05-09 16:44:15,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 9: [2023-05-09 16:44:15,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 7: [2023-05-09 16:44:15,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +10: [2023-05-09 16:44:15,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +16: [2023-05-09 16:44:16,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +16: [2023-05-09 16:44:16,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +24: [2023-05-09 16:44:16,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +20: [2023-05-09 16:44:16,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 0: [2023-05-09 16:44:16,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +22: [2023-05-09 16:44:16,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +24: [2023-05-09 16:44:16,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +22: [2023-05-09 16:44:16,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +20: [2023-05-09 16:44:16,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +16: [2023-05-09 16:44:16,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +22: [2023-05-09 16:44:16,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +22: [2023-05-09 16:44:16,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +16: [2023-05-09 16:44:16,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +13: [2023-05-09 16:44:16,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +14: [2023-05-09 16:44:16,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +30: [2023-05-09 16:44:16,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +30: [2023-05-09 16:44:16,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +30: [2023-05-09 16:44:16,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +30: [2023-05-09 16:44:16,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +22: [2023-05-09 16:44:16,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +22: [2023-05-09 16:44:16,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +22: [2023-05-09 16:44:16,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +22: [2023-05-09 16:44:16,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +12: [2023-05-09 16:44:16,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +16: [2023-05-09 16:44:16,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +21: [2023-05-09 16:44:16,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +10: [2023-05-09 16:44:16,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +14: [2023-05-09 16:44:16,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +21: [2023-05-09 16:44:16,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +21: [2023-05-09 16:44:16,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +14: [2023-05-09 16:44:16,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +16: [2023-05-09 16:44:16,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +16: [2023-05-09 16:44:16,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +16: [2023-05-09 16:44:16,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +16: [2023-05-09 16:44:16,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +16: [2023-05-09 16:44:16,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +13: [2023-05-09 16:44:16,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +13: [2023-05-09 16:44:16,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... + 4: [2023-05-09 16:44:16,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +12: [2023-05-09 16:44:16,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +12: [2023-05-09 16:44:16,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +10: [2023-05-09 16:44:16,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +13: [2023-05-09 16:44:16,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +22: [2023-05-09 16:44:16,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +10: [2023-05-09 16:44:16,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +14: [2023-05-09 16:44:16,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +16: [2023-05-09 16:44:16,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +12: [2023-05-09 16:44:16,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +13: [2023-05-09 16:44:16,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +16: [2023-05-09 16:44:16,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +10: [2023-05-09 16:44:16,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +17: [2023-05-09 16:44:16,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +17: [2023-05-09 16:44:16,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +21: [2023-05-09 16:44:16,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +17: [2023-05-09 16:44:16,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +17: [2023-05-09 16:44:16,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +17: [2023-05-09 16:44:16,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +17: [2023-05-09 16:44:16,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +17: [2023-05-09 16:44:16,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +14: [2023-05-09 16:44:16,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +14: [2023-05-09 16:44:16,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +21: [2023-05-09 16:44:16,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +17: [2023-05-09 16:44:16,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +21: [2023-05-09 16:44:16,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +22: [2023-05-09 16:44:16,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +22: [2023-05-09 16:44:16,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +13: [2023-05-09 16:44:16,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +14: [2023-05-09 16:44:16,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +10: [2023-05-09 16:44:16,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +13: [2023-05-09 16:44:16,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +13: [2023-05-09 16:44:16,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +22: [2023-05-09 16:44:16,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +12: [2023-05-09 16:44:16,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +16: [2023-05-09 16:44:16,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +10: [2023-05-09 16:44:16,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +15: [2023-05-09 16:44:16,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +15: [2023-05-09 16:44:16,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +22: [2023-05-09 16:44:16,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +15: [2023-05-09 16:44:16,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +16: [2023-05-09 16:44:16,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +15: [2023-05-09 16:44:16,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +10: [2023-05-09 16:44:16,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +15: [2023-05-09 16:44:16,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +15: [2023-05-09 16:44:16,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +21: [2023-05-09 16:44:16,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +15: [2023-05-09 16:44:16,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +15: [2023-05-09 16:44:16,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +12: [2023-05-09 16:44:16,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +16: [2023-05-09 16:44:16,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +16: [2023-05-09 16:44:16,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +16: [2023-05-09 16:44:16,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +16: [2023-05-09 16:44:16,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 4: [2023-05-09 16:44:16,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt. +12: [2023-05-09 16:44:16,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt... +17: [2023-05-09 16:44:16,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +16: [2023-05-09 16:44:16,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +16: [2023-05-09 16:44:16,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +14: [2023-05-09 16:44:16,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +10: [2023-05-09 16:44:16,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +17: [2023-05-09 16:44:16,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +10: [2023-05-09 16:44:16,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +10: [2023-05-09 16:44:16,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +22: [2023-05-09 16:44:16,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +10: [2023-05-09 16:44:16,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +10: [2023-05-09 16:44:16,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +21: [2023-05-09 16:44:16,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +12: [2023-05-09 16:44:16,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +12: [2023-05-09 16:44:16,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +12: [2023-05-09 16:44:16,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +12: [2023-05-09 16:44:16,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +15: [2023-05-09 16:44:16,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +22: [2023-05-09 16:44:16,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +15: [2023-05-09 16:44:16,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +22: [2023-05-09 16:44:16,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +12: [2023-05-09 16:44:16,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +17: [2023-05-09 16:44:16,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +15: [2023-05-09 16:44:16,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +13: [2023-05-09 16:44:16,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +13: [2023-05-09 16:44:16,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +13: [2023-05-09 16:44:16,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +15: [2023-05-09 16:44:16,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +13: [2023-05-09 16:44:16,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 5: [2023-05-09 16:44:16,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 5: [2023-05-09 16:44:16,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 6: [2023-05-09 16:44:16,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 6: [2023-05-09 16:44:16,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 6: [2023-05-09 16:44:16,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +17: [2023-05-09 16:44:16,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +17: [2023-05-09 16:44:16,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +10: [2023-05-09 16:44:16,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +10: [2023-05-09 16:44:16,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +18: [2023-05-09 16:44:16,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +18: [2023-05-09 16:44:16,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +10: [2023-05-09 16:44:16,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +18: [2023-05-09 16:44:16,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +18: [2023-05-09 16:44:16,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +10: [2023-05-09 16:44:16,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 5: [2023-05-09 16:44:16,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +12: [2023-05-09 16:44:16,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +15: [2023-05-09 16:44:16,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +12: [2023-05-09 16:44:16,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +12: [2023-05-09 16:44:16,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +15: [2023-05-09 16:44:16,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +12: [2023-05-09 16:44:16,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +17: [2023-05-09 16:44:16,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +13: [2023-05-09 16:44:16,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +13: [2023-05-09 16:44:16,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +15: [2023-05-09 16:44:16,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +13: [2023-05-09 16:44:16,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +17: [2023-05-09 16:44:16,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +13: [2023-05-09 16:44:16,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +18: [2023-05-09 16:44:16,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +18: [2023-05-09 16:44:16,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +15: [2023-05-09 16:44:16,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +18: [2023-05-09 16:44:16,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +18: [2023-05-09 16:44:16,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +12: [2023-05-09 16:44:16,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +12: [2023-05-09 16:44:16,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +17: [2023-05-09 16:44:16,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +10: [2023-05-09 16:44:16,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +10: [2023-05-09 16:44:16,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +13: [2023-05-09 16:44:16,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +13: [2023-05-09 16:44:16,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +13: [2023-05-09 16:44:16,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +13: [2023-05-09 16:44:16,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +10: [2023-05-09 16:44:16,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +10: [2023-05-09 16:44:16,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +12: [2023-05-09 16:44:16,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +12: [2023-05-09 16:44:16,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +10: [2023-05-09 16:44:16,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +10: [2023-05-09 16:44:16,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 2: [2023-05-09 16:44:16,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +12: [2023-05-09 16:44:16,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +12: [2023-05-09 16:44:16,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +13: [2023-05-09 16:44:16,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +13: [2023-05-09 16:44:16,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +12: [2023-05-09 16:44:16,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +12: [2023-05-09 16:44:16,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +10: [2023-05-09 16:44:16,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +10: [2023-05-09 16:44:16,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +13: [2023-05-09 16:44:16,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +13: [2023-05-09 16:44:16,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 2: [2023-05-09 16:44:16,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 2: [2023-05-09 16:44:16,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 0: [2023-05-09 16:44:16,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 0: [2023-05-09 16:44:16,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 2: [2023-05-09 16:44:16,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 8: [2023-05-09 16:44:16,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +14: [2023-05-09 16:44:16,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +14: [2023-05-09 16:44:16,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +14: [2023-05-09 16:44:16,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +11: [2023-05-09 16:44:16,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +26: [2023-05-09 16:44:16,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +26: [2023-05-09 16:44:16,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +22: [2023-05-09 16:44:16,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +22: [2023-05-09 16:44:16,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +14: [2023-05-09 16:44:16,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 2: [2023-05-09 16:44:16,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +17: [2023-05-09 16:44:16,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 2: [2023-05-09 16:44:16,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 2: [2023-05-09 16:44:16,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +17: [2023-05-09 16:44:16,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +17: [2023-05-09 16:44:16,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +17: [2023-05-09 16:44:16,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 8: [2023-05-09 16:44:16,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 8: [2023-05-09 16:44:16,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +11: [2023-05-09 16:44:16,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +26: [2023-05-09 16:44:16,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +26: [2023-05-09 16:44:16,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +11: [2023-05-09 16:44:16,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +22: [2023-05-09 16:44:16,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +22: [2023-05-09 16:44:16,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 8: [2023-05-09 16:44:16,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +11: [2023-05-09 16:44:16,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 2: [2023-05-09 16:44:16,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +11: [2023-05-09 16:44:16,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +14: [2023-05-09 16:44:16,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +14: [2023-05-09 16:44:16,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +14: [2023-05-09 16:44:16,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +14: [2023-05-09 16:44:16,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +17: [2023-05-09 16:44:16,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +17: [2023-05-09 16:44:16,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +22: [2023-05-09 16:44:16,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +17: [2023-05-09 16:44:16,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +17: [2023-05-09 16:44:16,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +26: [2023-05-09 16:44:16,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +11: [2023-05-09 16:44:16,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +26: [2023-05-09 16:44:16,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +26: [2023-05-09 16:44:16,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +26: [2023-05-09 16:44:16,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +22: [2023-05-09 16:44:16,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +11: [2023-05-09 16:44:16,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +11: [2023-05-09 16:44:16,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +22: [2023-05-09 16:44:16,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +22: [2023-05-09 16:44:16,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +30: [2023-05-09 16:44:16,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +30: [2023-05-09 16:44:16,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +30: [2023-05-09 16:44:16,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +30: [2023-05-09 16:44:16,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +30: [2023-05-09 16:44:16,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +30: [2023-05-09 16:44:16,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +30: [2023-05-09 16:44:16,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +30: [2023-05-09 16:44:16,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +30: [2023-05-09 16:44:16,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +30: [2023-05-09 16:44:16,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +30: [2023-05-09 16:44:16,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +30: [2023-05-09 16:44:16,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +30: [2023-05-09 16:44:16,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +30: [2023-05-09 16:44:16,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +14: [2023-05-09 16:44:16,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +14: [2023-05-09 16:44:16,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +30: [2023-05-09 16:44:16,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +30: [2023-05-09 16:44:16,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 6: [2023-05-09 16:44:16,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 6: [2023-05-09 16:44:16,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 6: [2023-05-09 16:44:16,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 6: [2023-05-09 16:44:16,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 2: [2023-05-09 16:44:16,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 6: [2023-05-09 16:44:16,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +17: [2023-05-09 16:44:16,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +22: [2023-05-09 16:44:16,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +14: [2023-05-09 16:44:16,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +14: [2023-05-09 16:44:16,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +17: [2023-05-09 16:44:16,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +22: [2023-05-09 16:44:16,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +14: [2023-05-09 16:44:16,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +14: [2023-05-09 16:44:16,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 2: [2023-05-09 16:44:16,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 0: [2023-05-09 16:44:16,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +17: [2023-05-09 16:44:16,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +17: [2023-05-09 16:44:16,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +17: [2023-05-09 16:44:16,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 0: [2023-05-09 16:44:16,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +17: [2023-05-09 16:44:16,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +17: [2023-05-09 16:44:16,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +30: [2023-05-09 16:44:16,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +30: [2023-05-09 16:44:16,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +17: [2023-05-09 16:44:16,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 2: [2023-05-09 16:44:16,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 3: [2023-05-09 16:44:16,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 3: [2023-05-09 16:44:16,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 3: [2023-05-09 16:44:16,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 3: [2023-05-09 16:44:16,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +11: [2023-05-09 16:44:16,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 0: [2023-05-09 16:44:16,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 3: [2023-05-09 16:44:16,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +30: [2023-05-09 16:44:16,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +30: [2023-05-09 16:44:16,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +11: [2023-05-09 16:44:16,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +11: [2023-05-09 16:44:16,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +11: [2023-05-09 16:44:16,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +11: [2023-05-09 16:44:16,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +11: [2023-05-09 16:44:16,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +11: [2023-05-09 16:44:16,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +11: [2023-05-09 16:44:16,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +11: [2023-05-09 16:44:16,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +11: [2023-05-09 16:44:16,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +11: [2023-05-09 16:44:16,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 6: [2023-05-09 16:44:16,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +11: [2023-05-09 16:44:16,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +11: [2023-05-09 16:44:16,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +11: [2023-05-09 16:44:16,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +14: [2023-05-09 16:44:16,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +27: [2023-05-09 16:44:16,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 2: [2023-05-09 16:44:16,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +27: [2023-05-09 16:44:16,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +27: [2023-05-09 16:44:16,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +22: [2023-05-09 16:44:16,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +27: [2023-05-09 16:44:16,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +22: [2023-05-09 16:44:16,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +22: [2023-05-09 16:44:16,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +14: [2023-05-09 16:44:16,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +11: [2023-05-09 16:44:16,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +22: [2023-05-09 16:44:16,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 0: [2023-05-09 16:44:16,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +11: [2023-05-09 16:44:16,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 0: [2023-05-09 16:44:16,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +30: [2023-05-09 16:44:16,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +22: [2023-05-09 16:44:16,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +30: [2023-05-09 16:44:16,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +25: [2023-05-09 16:44:16,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +25: [2023-05-09 16:44:16,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +25: [2023-05-09 16:44:16,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 6: [2023-05-09 16:44:16,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +29: [2023-05-09 16:44:16,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +29: [2023-05-09 16:44:16,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +29: [2023-05-09 16:44:16,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 6: [2023-05-09 16:44:16,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +22: [2023-05-09 16:44:16,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +30: [2023-05-09 16:44:16,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +25: [2023-05-09 16:44:16,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 3: [2023-05-09 16:44:16,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +30: [2023-05-09 16:44:16,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +29: [2023-05-09 16:44:16,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 3: [2023-05-09 16:44:16,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +27: [2023-05-09 16:44:16,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +27: [2023-05-09 16:44:16,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +27: [2023-05-09 16:44:16,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +27: [2023-05-09 16:44:16,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +27: [2023-05-09 16:44:16,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +27: [2023-05-09 16:44:16,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +15: [2023-05-09 16:44:16,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +15: [2023-05-09 16:44:16,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +15: [2023-05-09 16:44:16,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +15: [2023-05-09 16:44:16,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +27: [2023-05-09 16:44:16,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +27: [2023-05-09 16:44:16,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +31: [2023-05-09 16:44:16,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +31: [2023-05-09 16:44:16,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 3: [2023-05-09 16:44:16,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +25: [2023-05-09 16:44:16,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +25: [2023-05-09 16:44:16,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +27: [2023-05-09 16:44:16,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +25: [2023-05-09 16:44:16,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +25: [2023-05-09 16:44:16,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +27: [2023-05-09 16:44:16,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +25: [2023-05-09 16:44:16,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +25: [2023-05-09 16:44:16,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +25: [2023-05-09 16:44:16,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +26: [2023-05-09 16:44:16,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +26: [2023-05-09 16:44:16,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +26: [2023-05-09 16:44:16,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +26: [2023-05-09 16:44:16,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +26: [2023-05-09 16:44:16,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +26: [2023-05-09 16:44:16,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +26: [2023-05-09 16:44:16,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +26: [2023-05-09 16:44:16,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +26: [2023-05-09 16:44:16,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +26: [2023-05-09 16:44:16,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +26: [2023-05-09 16:44:16,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +26: [2023-05-09 16:44:16,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +26: [2023-05-09 16:44:16,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +26: [2023-05-09 16:44:16,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +27: [2023-05-09 16:44:16,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +27: [2023-05-09 16:44:16,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +31: [2023-05-09 16:44:16,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +31: [2023-05-09 16:44:16,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +26: [2023-05-09 16:44:16,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 4: [2023-05-09 16:44:16,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +25: [2023-05-09 16:44:16,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +25: [2023-05-09 16:44:16,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +29: [2023-05-09 16:44:16,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +29: [2023-05-09 16:44:16,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +26: [2023-05-09 16:44:16,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +29: [2023-05-09 16:44:16,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +29: [2023-05-09 16:44:16,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +25: [2023-05-09 16:44:16,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +21: [2023-05-09 16:44:16,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +21: [2023-05-09 16:44:16,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +21: [2023-05-09 16:44:16,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +21: [2023-05-09 16:44:16,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 3: [2023-05-09 16:44:16,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +31: [2023-05-09 16:44:16,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +31: [2023-05-09 16:44:16,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +11: [2023-05-09 16:44:16,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +11: [2023-05-09 16:44:16,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +11: [2023-05-09 16:44:16,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +15: [2023-05-09 16:44:16,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +15: [2023-05-09 16:44:16,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +15: [2023-05-09 16:44:16,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +31: [2023-05-09 16:44:16,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +25: [2023-05-09 16:44:16,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +25: [2023-05-09 16:44:16,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +15: [2023-05-09 16:44:16,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +31: [2023-05-09 16:44:16,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +19: [2023-05-09 16:44:16,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +19: [2023-05-09 16:44:16,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +19: [2023-05-09 16:44:16,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +19: [2023-05-09 16:44:16,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +19: [2023-05-09 16:44:16,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +19: [2023-05-09 16:44:16,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +19: [2023-05-09 16:44:16,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +25: [2023-05-09 16:44:16,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +19: [2023-05-09 16:44:16,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 4: [2023-05-09 16:44:16,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +19: [2023-05-09 16:44:16,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +28: [2023-05-09 16:44:16,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +28: [2023-05-09 16:44:16,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +28: [2023-05-09 16:44:16,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +28: [2023-05-09 16:44:16,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +21: [2023-05-09 16:44:16,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +21: [2023-05-09 16:44:16,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +27: [2023-05-09 16:44:16,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +19: [2023-05-09 16:44:16,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +27: [2023-05-09 16:44:16,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +21: [2023-05-09 16:44:16,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +27: [2023-05-09 16:44:16,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +26: [2023-05-09 16:44:16,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +26: [2023-05-09 16:44:16,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +26: [2023-05-09 16:44:16,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +26: [2023-05-09 16:44:16,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 7: [2023-05-09 16:44:16,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +21: [2023-05-09 16:44:16,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +27: [2023-05-09 16:44:16,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 7: [2023-05-09 16:44:16,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +27: [2023-05-09 16:44:16,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +25: [2023-05-09 16:44:16,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 7: [2023-05-09 16:44:16,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +11: [2023-05-09 16:44:16,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +27: [2023-05-09 16:44:16,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +27: [2023-05-09 16:44:16,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +11: [2023-05-09 16:44:16,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +11: [2023-05-09 16:44:16,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +27: [2023-05-09 16:44:16,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +27: [2023-05-09 16:44:16,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +27: [2023-05-09 16:44:16,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +27: [2023-05-09 16:44:16,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +27: [2023-05-09 16:44:16,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +19: [2023-05-09 16:44:16,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +19: [2023-05-09 16:44:16,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +19: [2023-05-09 16:44:16,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +28: [2023-05-09 16:44:16,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +31: [2023-05-09 16:44:16,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +31: [2023-05-09 16:44:16,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +11: [2023-05-09 16:44:16,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +31: [2023-05-09 16:44:16,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +28: [2023-05-09 16:44:16,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +28: [2023-05-09 16:44:16,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +31: [2023-05-09 16:44:16,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +31: [2023-05-09 16:44:16,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +27: [2023-05-09 16:44:16,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +15: [2023-05-09 16:44:16,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +19: [2023-05-09 16:44:16,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +15: [2023-05-09 16:44:16,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +23: [2023-05-09 16:44:16,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +23: [2023-05-09 16:44:16,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +19: [2023-05-09 16:44:16,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +23: [2023-05-09 16:44:16,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +23: [2023-05-09 16:44:16,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +23: [2023-05-09 16:44:16,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +23: [2023-05-09 16:44:16,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +23: [2023-05-09 16:44:16,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +23: [2023-05-09 16:44:16,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +31: [2023-05-09 16:44:16,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +23: [2023-05-09 16:44:16,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +23: [2023-05-09 16:44:16,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +23: [2023-05-09 16:44:16,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +23: [2023-05-09 16:44:16,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +28: [2023-05-09 16:44:16,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +23: [2023-05-09 16:44:16,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +23: [2023-05-09 16:44:16,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +23: [2023-05-09 16:44:16,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +23: [2023-05-09 16:44:16,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +25: [2023-05-09 16:44:16,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +24: [2023-05-09 16:44:16,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +24: [2023-05-09 16:44:16,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +31: [2023-05-09 16:44:16,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +31: [2023-05-09 16:44:16,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +24: [2023-05-09 16:44:16,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +24: [2023-05-09 16:44:16,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +24: [2023-05-09 16:44:16,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +24: [2023-05-09 16:44:16,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +24: [2023-05-09 16:44:16,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +24: [2023-05-09 16:44:16,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +31: [2023-05-09 16:44:16,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +27: [2023-05-09 16:44:16,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +25: [2023-05-09 16:44:16,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +26: [2023-05-09 16:44:16,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +31: [2023-05-09 16:44:16,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +26: [2023-05-09 16:44:16,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +28: [2023-05-09 16:44:16,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +26: [2023-05-09 16:44:16,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +26: [2023-05-09 16:44:16,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +28: [2023-05-09 16:44:16,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +31: [2023-05-09 16:44:16,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +31: [2023-05-09 16:44:16,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 7: [2023-05-09 16:44:16,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +15: [2023-05-09 16:44:16,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +15: [2023-05-09 16:44:16,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +15: [2023-05-09 16:44:16,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +15: [2023-05-09 16:44:16,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +15: [2023-05-09 16:44:16,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +28: [2023-05-09 16:44:16,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +28: [2023-05-09 16:44:16,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +28: [2023-05-09 16:44:16,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +28: [2023-05-09 16:44:16,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +28: [2023-05-09 16:44:16,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +18: [2023-05-09 16:44:16,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +18: [2023-05-09 16:44:16,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +18: [2023-05-09 16:44:16,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +18: [2023-05-09 16:44:16,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +15: [2023-05-09 16:44:16,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +18: [2023-05-09 16:44:16,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +18: [2023-05-09 16:44:16,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +21: [2023-05-09 16:44:16,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +21: [2023-05-09 16:44:16,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +18: [2023-05-09 16:44:16,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +18: [2023-05-09 16:44:16,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +18: [2023-05-09 16:44:16,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +18: [2023-05-09 16:44:16,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +18: [2023-05-09 16:44:16,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +18: [2023-05-09 16:44:16,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +18: [2023-05-09 16:44:16,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +18: [2023-05-09 16:44:16,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +18: [2023-05-09 16:44:16,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +21: [2023-05-09 16:44:16,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +21: [2023-05-09 16:44:16,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +28: [2023-05-09 16:44:16,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +21: [2023-05-09 16:44:16,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +27: [2023-05-09 16:44:16,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +21: [2023-05-09 16:44:16,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +18: [2023-05-09 16:44:16,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +11: [2023-05-09 16:44:16,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +25: [2023-05-09 16:44:16,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +25: [2023-05-09 16:44:16,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +27: [2023-05-09 16:44:16,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +31: [2023-05-09 16:44:16,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +31: [2023-05-09 16:44:16,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +31: [2023-05-09 16:44:16,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +31: [2023-05-09 16:44:16,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +19: [2023-05-09 16:44:16,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +19: [2023-05-09 16:44:16,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +19: [2023-05-09 16:44:16,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +28: [2023-05-09 16:44:16,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +21: [2023-05-09 16:44:16,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +21: [2023-05-09 16:44:16,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +28: [2023-05-09 16:44:16,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +31: [2023-05-09 16:44:16,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +23: [2023-05-09 16:44:16,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +23: [2023-05-09 16:44:16,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 5: [2023-05-09 16:44:16,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 5: [2023-05-09 16:44:16,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 5: [2023-05-09 16:44:16,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +23: [2023-05-09 16:44:16,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +19: [2023-05-09 16:44:16,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +23: [2023-05-09 16:44:16,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +19: [2023-05-09 16:44:16,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +31: [2023-05-09 16:44:16,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +19: [2023-05-09 16:44:16,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +19: [2023-05-09 16:44:16,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 9: [2023-05-09 16:44:16,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +25: [2023-05-09 16:44:16,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +20: [2023-05-09 16:44:16,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +31: [2023-05-09 16:44:16,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +20: [2023-05-09 16:44:16,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +20: [2023-05-09 16:44:16,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +20: [2023-05-09 16:44:16,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +20: [2023-05-09 16:44:16,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +20: [2023-05-09 16:44:16,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +29: [2023-05-09 16:44:16,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +19: [2023-05-09 16:44:16,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +29: [2023-05-09 16:44:16,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +20: [2023-05-09 16:44:16,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +29: [2023-05-09 16:44:16,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +29: [2023-05-09 16:44:16,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 7: [2023-05-09 16:44:16,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +28: [2023-05-09 16:44:16,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +29: [2023-05-09 16:44:16,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +29: [2023-05-09 16:44:16,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +29: [2023-05-09 16:44:16,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +29: [2023-05-09 16:44:16,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +29: [2023-05-09 16:44:16,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +29: [2023-05-09 16:44:16,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +29: [2023-05-09 16:44:16,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +29: [2023-05-09 16:44:16,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +29: [2023-05-09 16:44:16,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +18: [2023-05-09 16:44:16,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +18: [2023-05-09 16:44:16,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +18: [2023-05-09 16:44:16,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +19: [2023-05-09 16:44:16,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +25: [2023-05-09 16:44:16,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +25: [2023-05-09 16:44:16,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +25: [2023-05-09 16:44:16,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +29: [2023-05-09 16:44:16,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +29: [2023-05-09 16:44:16,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +29: [2023-05-09 16:44:16,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +31: [2023-05-09 16:44:16,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +28: [2023-05-09 16:44:16,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +20: [2023-05-09 16:44:16,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +23: [2023-05-09 16:44:16,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +18: [2023-05-09 16:44:16,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +28: [2023-05-09 16:44:16,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +28: [2023-05-09 16:44:16,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +28: [2023-05-09 16:44:16,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +28: [2023-05-09 16:44:16,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +23: [2023-05-09 16:44:16,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +31: [2023-05-09 16:44:16,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +31: [2023-05-09 16:44:16,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +23: [2023-05-09 16:44:16,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +20: [2023-05-09 16:44:16,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +20: [2023-05-09 16:44:16,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 9: [2023-05-09 16:44:16,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 9: [2023-05-09 16:44:16,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 9: [2023-05-09 16:44:16,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 4: [2023-05-09 16:44:16,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 4: [2023-05-09 16:44:16,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 4: [2023-05-09 16:44:16,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +20: [2023-05-09 16:44:16,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 9: [2023-05-09 16:44:16,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +20: [2023-05-09 16:44:16,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. + 5: [2023-05-09 16:44:16,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +23: [2023-05-09 16:44:16,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +25: [2023-05-09 16:44:16,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +31: [2023-05-09 16:44:16,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +28: [2023-05-09 16:44:16,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +28: [2023-05-09 16:44:16,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +28: [2023-05-09 16:44:16,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +28: [2023-05-09 16:44:16,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +25: [2023-05-09 16:44:16,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +19: [2023-05-09 16:44:16,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +25: [2023-05-09 16:44:16,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 7: [2023-05-09 16:44:16,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +20: [2023-05-09 16:44:16,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +18: [2023-05-09 16:44:16,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +28: [2023-05-09 16:44:16,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +19: [2023-05-09 16:44:16,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +19: [2023-05-09 16:44:16,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +20: [2023-05-09 16:44:16,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +31: [2023-05-09 16:44:16,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 9: [2023-05-09 16:44:16,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +29: [2023-05-09 16:44:16,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +29: [2023-05-09 16:44:16,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +20: [2023-05-09 16:44:16,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +18: [2023-05-09 16:44:16,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +18: [2023-05-09 16:44:16,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +18: [2023-05-09 16:44:16,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +28: [2023-05-09 16:44:16,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +25: [2023-05-09 16:44:16,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +25: [2023-05-09 16:44:16,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 7: [2023-05-09 16:44:16,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 9: [2023-05-09 16:44:16,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +25: [2023-05-09 16:44:16,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 5: [2023-05-09 16:44:16,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +20: [2023-05-09 16:44:16,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +20: [2023-05-09 16:44:16,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +20: [2023-05-09 16:44:16,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +19: [2023-05-09 16:44:16,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +20: [2023-05-09 16:44:16,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +28: [2023-05-09 16:44:16,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +19: [2023-05-09 16:44:16,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +19: [2023-05-09 16:44:16,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +19: [2023-05-09 16:44:16,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +19: [2023-05-09 16:44:16,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 5: [2023-05-09 16:44:16,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +24: [2023-05-09 16:44:16,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +24: [2023-05-09 16:44:16,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +24: [2023-05-09 16:44:16,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +24: [2023-05-09 16:44:16,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +24: [2023-05-09 16:44:16,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +24: [2023-05-09 16:44:16,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +24: [2023-05-09 16:44:16,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +24: [2023-05-09 16:44:16,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt. +24: [2023-05-09 16:44:16,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +24: [2023-05-09 16:44:16,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +24: [2023-05-09 16:44:16,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +24: [2023-05-09 16:44:16,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +24: [2023-05-09 16:44:16,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +24: [2023-05-09 16:44:16,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... +24: [2023-05-09 16:44:16,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +24: [2023-05-09 16:44:16,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +28: [2023-05-09 16:44:16,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +20: [2023-05-09 16:44:16,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +29: [2023-05-09 16:44:16,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +29: [2023-05-09 16:44:16,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +29: [2023-05-09 16:44:16,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +29: [2023-05-09 16:44:16,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +20: [2023-05-09 16:44:16,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +20: [2023-05-09 16:44:16,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 9: [2023-05-09 16:44:16,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +20: [2023-05-09 16:44:16,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +29: [2023-05-09 16:44:16,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +20: [2023-05-09 16:44:16,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +20: [2023-05-09 16:44:16,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +20: [2023-05-09 16:44:16,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +20: [2023-05-09 16:44:16,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +29: [2023-05-09 16:44:16,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +20: [2023-05-09 16:44:16,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +20: [2023-05-09 16:44:16,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 9: [2023-05-09 16:44:16,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 9: [2023-05-09 16:44:16,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 1: [2023-05-09 16:44:16,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 4: [2023-05-09 16:44:16,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +20: [2023-05-09 16:44:16,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +20: [2023-05-09 16:44:16,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 4: [2023-05-09 16:44:16,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 4: [2023-05-09 16:44:16,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +20: [2023-05-09 16:44:16,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 8: [2023-05-09 16:44:16,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 8: [2023-05-09 16:44:16,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 8: [2023-05-09 16:44:16,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +24: [2023-05-09 16:44:16,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +24: [2023-05-09 16:44:16,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +24: [2023-05-09 16:44:16,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +24: [2023-05-09 16:44:16,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +24: [2023-05-09 16:44:16,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +24: [2023-05-09 16:44:16,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... +24: [2023-05-09 16:44:16,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. +24: [2023-05-09 16:44:16,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 8: [2023-05-09 16:44:16,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 1: [2023-05-09 16:44:16,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 1: [2023-05-09 16:44:16,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 1: [2023-05-09 16:44:16,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt... + 1: [2023-05-09 16:44:16,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +12: [2023-05-09 16:44:16,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +12: [2023-05-09 16:44:16,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +12: [2023-05-09 16:44:16,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +12: [2023-05-09 16:44:16,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +16: [2023-05-09 16:44:16,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +13: [2023-05-09 16:44:16,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +13: [2023-05-09 16:44:16,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +13: [2023-05-09 16:44:16,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +13: [2023-05-09 16:44:16,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +16: [2023-05-09 16:44:16,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +16: [2023-05-09 16:44:16,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +16: [2023-05-09 16:44:16,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +16: [2023-05-09 16:44:16,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +16: [2023-05-09 16:44:16,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +16: [2023-05-09 16:44:16,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +16: [2023-05-09 16:44:16,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +16: [2023-05-09 16:44:16,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +16: [2023-05-09 16:44:16,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +16: [2023-05-09 16:44:16,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +16: [2023-05-09 16:44:16,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +12: [2023-05-09 16:44:16,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +12: [2023-05-09 16:44:16,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +16: [2023-05-09 16:44:16,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +13: [2023-05-09 16:44:16,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +12: [2023-05-09 16:44:16,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +12: [2023-05-09 16:44:16,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +13: [2023-05-09 16:44:16,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +16: [2023-05-09 16:44:16,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +13: [2023-05-09 16:44:16,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +16: [2023-05-09 16:44:16,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +16: [2023-05-09 16:44:16,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +13: [2023-05-09 16:44:16,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +21: [2023-05-09 16:44:16,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +21: [2023-05-09 16:44:16,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +21: [2023-05-09 16:44:16,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +21: [2023-05-09 16:44:16,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +21: [2023-05-09 16:44:16,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +21: [2023-05-09 16:44:16,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +21: [2023-05-09 16:44:16,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +21: [2023-05-09 16:44:16,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +13: [2023-05-09 16:44:16,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +13: [2023-05-09 16:44:16,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +13: [2023-05-09 16:44:16,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +13: [2023-05-09 16:44:16,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +13: [2023-05-09 16:44:16,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +13: [2023-05-09 16:44:16,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +13: [2023-05-09 16:44:16,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +13: [2023-05-09 16:44:16,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 5: [2023-05-09 16:44:16,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 5: [2023-05-09 16:44:16,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 5: [2023-05-09 16:44:16,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +10: [2023-05-09 16:44:16,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +10: [2023-05-09 16:44:16,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +10: [2023-05-09 16:44:16,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +10: [2023-05-09 16:44:16,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +10: [2023-05-09 16:44:16,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +10: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +10: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +12: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +12: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +12: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +14: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +14: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +14: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +14: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +12: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +12: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +12: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +12: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +14: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +14: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +14: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 4: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +15: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 2: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 4: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +15: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +15: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +10: [2023-05-09 16:44:16,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +15: [2023-05-09 16:44:16,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +15: [2023-05-09 16:44:16,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +15: [2023-05-09 16:44:16,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +15: [2023-05-09 16:44:16,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +10: [2023-05-09 16:44:16,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +10: [2023-05-09 16:44:16,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +10: [2023-05-09 16:44:16,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +14: [2023-05-09 16:44:16,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +10: [2023-05-09 16:44:16,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +15: [2023-05-09 16:44:16,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +17: [2023-05-09 16:44:16,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +17: [2023-05-09 16:44:16,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +17: [2023-05-09 16:44:16,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 0: [2023-05-09 16:44:16,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +17: [2023-05-09 16:44:16,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +22: [2023-05-09 16:44:16,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +22: [2023-05-09 16:44:16,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +22: [2023-05-09 16:44:16,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +22: [2023-05-09 16:44:16,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +14: [2023-05-09 16:44:16,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +14: [2023-05-09 16:44:16,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +22: [2023-05-09 16:44:16,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +14: [2023-05-09 16:44:16,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +22: [2023-05-09 16:44:16,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +22: [2023-05-09 16:44:16,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +22: [2023-05-09 16:44:16,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +14: [2023-05-09 16:44:16,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +16: [2023-05-09 16:44:16,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +16: [2023-05-09 16:44:16,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +26: [2023-05-09 16:44:16,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +17: [2023-05-09 16:44:16,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +26: [2023-05-09 16:44:16,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +26: [2023-05-09 16:44:16,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +17: [2023-05-09 16:44:16,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +17: [2023-05-09 16:44:16,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +17: [2023-05-09 16:44:16,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +26: [2023-05-09 16:44:16,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 3: [2023-05-09 16:44:16,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +12: [2023-05-09 16:44:16,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +17: [2023-05-09 16:44:16,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +17: [2023-05-09 16:44:16,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +17: [2023-05-09 16:44:16,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +31: [2023-05-09 16:44:16,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +31: [2023-05-09 16:44:16,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +31: [2023-05-09 16:44:16,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +17: [2023-05-09 16:44:16,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +31: [2023-05-09 16:44:16,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +31: [2023-05-09 16:44:16,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +31: [2023-05-09 16:44:16,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +31: [2023-05-09 16:44:16,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +31: [2023-05-09 16:44:16,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +30: [2023-05-09 16:44:16,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +25: [2023-05-09 16:44:16,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +30: [2023-05-09 16:44:16,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +30: [2023-05-09 16:44:16,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +25: [2023-05-09 16:44:16,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +25: [2023-05-09 16:44:16,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +30: [2023-05-09 16:44:16,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +30: [2023-05-09 16:44:16,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +25: [2023-05-09 16:44:16,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +30: [2023-05-09 16:44:16,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +25: [2023-05-09 16:44:16,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +30: [2023-05-09 16:44:16,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +25: [2023-05-09 16:44:16,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +25: [2023-05-09 16:44:16,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +28: [2023-05-09 16:44:16,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +28: [2023-05-09 16:44:16,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +28: [2023-05-09 16:44:16,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +28: [2023-05-09 16:44:16,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +12: [2023-05-09 16:44:16,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +11: [2023-05-09 16:44:16,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +11: [2023-05-09 16:44:16,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +11: [2023-05-09 16:44:16,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +11: [2023-05-09 16:44:16,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +11: [2023-05-09 16:44:16,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +11: [2023-05-09 16:44:16,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +11: [2023-05-09 16:44:16,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +21: [2023-05-09 16:44:16,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +11: [2023-05-09 16:44:16,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 7: [2023-05-09 16:44:16,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 7: [2023-05-09 16:44:16,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +12: [2023-05-09 16:44:16,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +16: [2023-05-09 16:44:16,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +30: [2023-05-09 16:44:16,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +12: [2023-05-09 16:44:16,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +23: [2023-05-09 16:44:16,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +12: [2023-05-09 16:44:16,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +23: [2023-05-09 16:44:16,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +25: [2023-05-09 16:44:16,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +23: [2023-05-09 16:44:16,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +23: [2023-05-09 16:44:16,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +23: [2023-05-09 16:44:16,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +19: [2023-05-09 16:44:16,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +19: [2023-05-09 16:44:16,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +23: [2023-05-09 16:44:16,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +19: [2023-05-09 16:44:16,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +19: [2023-05-09 16:44:16,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +13: [2023-05-09 16:44:16,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +13: [2023-05-09 16:44:16,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +12: [2023-05-09 16:44:16,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +12: [2023-05-09 16:44:16,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +13: [2023-05-09 16:44:16,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +13: [2023-05-09 16:44:16,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 2: [2023-05-09 16:44:16,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +23: [2023-05-09 16:44:16,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +19: [2023-05-09 16:44:16,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +19: [2023-05-09 16:44:16,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +19: [2023-05-09 16:44:16,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +19: [2023-05-09 16:44:16,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +19: [2023-05-09 16:44:16,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +10: [2023-05-09 16:44:16,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +19: [2023-05-09 16:44:16,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +19: [2023-05-09 16:44:16,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +17: [2023-05-09 16:44:16,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +17: [2023-05-09 16:44:16,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +16: [2023-05-09 16:44:16,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +23: [2023-05-09 16:44:16,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +13: [2023-05-09 16:44:16,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +10: [2023-05-09 16:44:16,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +10: [2023-05-09 16:44:16,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +16: [2023-05-09 16:44:16,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +15: [2023-05-09 16:44:16,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +15: [2023-05-09 16:44:16,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +15: [2023-05-09 16:44:16,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +26: [2023-05-09 16:44:16,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +16: [2023-05-09 16:44:16,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +17: [2023-05-09 16:44:16,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +22: [2023-05-09 16:44:16,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +22: [2023-05-09 16:44:16,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +22: [2023-05-09 16:44:16,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +22: [2023-05-09 16:44:16,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +19: [2023-05-09 16:44:16,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +22: [2023-05-09 16:44:16,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 6: [2023-05-09 16:44:16,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +14: [2023-05-09 16:44:16,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 6: [2023-05-09 16:44:16,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 6: [2023-05-09 16:44:16,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +21: [2023-05-09 16:44:16,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +26: [2023-05-09 16:44:16,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +21: [2023-05-09 16:44:16,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +14: [2023-05-09 16:44:16,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +22: [2023-05-09 16:44:16,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +10: [2023-05-09 16:44:16,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +14: [2023-05-09 16:44:16,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +14: [2023-05-09 16:44:16,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 3: [2023-05-09 16:44:16,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 3: [2023-05-09 16:44:16,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +12: [2023-05-09 16:44:16,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +12: [2023-05-09 16:44:16,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 4: [2023-05-09 16:44:16,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 2: [2023-05-09 16:44:16,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +10: [2023-05-09 16:44:16,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +17: [2023-05-09 16:44:16,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +14: [2023-05-09 16:44:16,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +28: [2023-05-09 16:44:16,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +14: [2023-05-09 16:44:16,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +14: [2023-05-09 16:44:16,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +28: [2023-05-09 16:44:16,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +21: [2023-05-09 16:44:16,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +16: [2023-05-09 16:44:16,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +31: [2023-05-09 16:44:16,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +26: [2023-05-09 16:44:16,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +26: [2023-05-09 16:44:16,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +27: [2023-05-09 16:44:16,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +27: [2023-05-09 16:44:16,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +28: [2023-05-09 16:44:16,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +27: [2023-05-09 16:44:16,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +16: [2023-05-09 16:44:16,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +27: [2023-05-09 16:44:16,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +31: [2023-05-09 16:44:16,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +16: [2023-05-09 16:44:16,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +12: [2023-05-09 16:44:16,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +31: [2023-05-09 16:44:16,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +28: [2023-05-09 16:44:16,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +30: [2023-05-09 16:44:16,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +17: [2023-05-09 16:44:16,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +19: [2023-05-09 16:44:16,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +16: [2023-05-09 16:44:16,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +25: [2023-05-09 16:44:16,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +16: [2023-05-09 16:44:16,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 7: [2023-05-09 16:44:16,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +16: [2023-05-09 16:44:16,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +16: [2023-05-09 16:44:16,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +30: [2023-05-09 16:44:16,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +15: [2023-05-09 16:44:16,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +13: [2023-05-09 16:44:16,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +13: [2023-05-09 16:44:16,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +12: [2023-05-09 16:44:16,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +12: [2023-05-09 16:44:16,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +17: [2023-05-09 16:44:16,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +30: [2023-05-09 16:44:16,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +22: [2023-05-09 16:44:16,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +16: [2023-05-09 16:44:16,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +15: [2023-05-09 16:44:16,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +10: [2023-05-09 16:44:16,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +13: [2023-05-09 16:44:16,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +13: [2023-05-09 16:44:16,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +23: [2023-05-09 16:44:16,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +19: [2023-05-09 16:44:16,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +13: [2023-05-09 16:44:16,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +23: [2023-05-09 16:44:16,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +13: [2023-05-09 16:44:16,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +22: [2023-05-09 16:44:16,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +15: [2023-05-09 16:44:16,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +31: [2023-05-09 16:44:16,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +21: [2023-05-09 16:44:16,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +10: [2023-05-09 16:44:16,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +19: [2023-05-09 16:44:16,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +19: [2023-05-09 16:44:16,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +19: [2023-05-09 16:44:16,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +14: [2023-05-09 16:44:16,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +23: [2023-05-09 16:44:16,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +21: [2023-05-09 16:44:16,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +22: [2023-05-09 16:44:16,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +21: [2023-05-09 16:44:16,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +14: [2023-05-09 16:44:16,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +14: [2023-05-09 16:44:16,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +13: [2023-05-09 16:44:16,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +25: [2023-05-09 16:44:16,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +25: [2023-05-09 16:44:16,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +22: [2023-05-09 16:44:16,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +31: [2023-05-09 16:44:16,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +22: [2023-05-09 16:44:16,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +10: [2023-05-09 16:44:16,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +27: [2023-05-09 16:44:16,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +12: [2023-05-09 16:44:16,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +16: [2023-05-09 16:44:16,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +25: [2023-05-09 16:44:16,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +13: [2023-05-09 16:44:16,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +27: [2023-05-09 16:44:16,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +27: [2023-05-09 16:44:16,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +27: [2023-05-09 16:44:16,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +10: [2023-05-09 16:44:16,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 4: [2023-05-09 16:44:16,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 0: [2023-05-09 16:44:16,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 0: [2023-05-09 16:44:16,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +30: [2023-05-09 16:44:16,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +30: [2023-05-09 16:44:16,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 0: [2023-05-09 16:44:16,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +30: [2023-05-09 16:44:16,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +31: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +25: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +30: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +31: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +15: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +15: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 2: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +15: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +20: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +20: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 3: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +17: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +20: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +23: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +23: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +23: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +20: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +15: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +11: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +31: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +17: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +16: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +11: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 2: [2023-05-09 16:44:16,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +30: [2023-05-09 16:44:16,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +25: [2023-05-09 16:44:16,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 3: [2023-05-09 16:44:16,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +31: [2023-05-09 16:44:16,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +31: [2023-05-09 16:44:16,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 3: [2023-05-09 16:44:16,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +24: [2023-05-09 16:44:16,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +17: [2023-05-09 16:44:16,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +22: [2023-05-09 16:44:16,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +31: [2023-05-09 16:44:16,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +24: [2023-05-09 16:44:16,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +24: [2023-05-09 16:44:16,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +22: [2023-05-09 16:44:16,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +24: [2023-05-09 16:44:16,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +18: [2023-05-09 16:44:16,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 2: [2023-05-09 16:44:16,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +18: [2023-05-09 16:44:16,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +18: [2023-05-09 16:44:16,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 2: [2023-05-09 16:44:16,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +12: [2023-05-09 16:44:16,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +22: [2023-05-09 16:44:16,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +18: [2023-05-09 16:44:16,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 1: [2023-05-09 16:44:16,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 1: [2023-05-09 16:44:16,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 1: [2023-05-09 16:44:16,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 1: [2023-05-09 16:44:16,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 2: [2023-05-09 16:44:16,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +23: [2023-05-09 16:44:16,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +15: [2023-05-09 16:44:16,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +13: [2023-05-09 16:44:16,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +11: [2023-05-09 16:44:16,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +11: [2023-05-09 16:44:16,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +17: [2023-05-09 16:44:16,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +11: [2023-05-09 16:44:16,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 3: [2023-05-09 16:44:16,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +30: [2023-05-09 16:44:16,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 9: [2023-05-09 16:44:16,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 2: [2023-05-09 16:44:16,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +11: [2023-05-09 16:44:16,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +30: [2023-05-09 16:44:16,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +14: [2023-05-09 16:44:16,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +17: [2023-05-09 16:44:16,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +11: [2023-05-09 16:44:16,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +14: [2023-05-09 16:44:16,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +23: [2023-05-09 16:44:16,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +29: [2023-05-09 16:44:16,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +13: [2023-05-09 16:44:16,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +10: [2023-05-09 16:44:16,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +29: [2023-05-09 16:44:16,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +29: [2023-05-09 16:44:16,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +29: [2023-05-09 16:44:16,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +13: [2023-05-09 16:44:16,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +21: [2023-05-09 16:44:16,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +21: [2023-05-09 16:44:16,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +21: [2023-05-09 16:44:16,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +23: [2023-05-09 16:44:16,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +21: [2023-05-09 16:44:16,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +12: [2023-05-09 16:44:16,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +21: [2023-05-09 16:44:16,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +19: [2023-05-09 16:44:16,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +17: [2023-05-09 16:44:16,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +19: [2023-05-09 16:44:16,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +19: [2023-05-09 16:44:16,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 6: [2023-05-09 16:44:16,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +17: [2023-05-09 16:44:16,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +17: [2023-05-09 16:44:16,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +31: [2023-05-09 16:44:16,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +10: [2023-05-09 16:44:16,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +30: [2023-05-09 16:44:16,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +30: [2023-05-09 16:44:16,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +11: [2023-05-09 16:44:16,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +30: [2023-05-09 16:44:16,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +25: [2023-05-09 16:44:16,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +17: [2023-05-09 16:44:16,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +14: [2023-05-09 16:44:16,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +14: [2023-05-09 16:44:16,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +14: [2023-05-09 16:44:16,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +14: [2023-05-09 16:44:16,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 0: [2023-05-09 16:44:16,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +17: [2023-05-09 16:44:16,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +14: [2023-05-09 16:44:16,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +25: [2023-05-09 16:44:16,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +25: [2023-05-09 16:44:16,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +22: [2023-05-09 16:44:16,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +23: [2023-05-09 16:44:16,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +25: [2023-05-09 16:44:16,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 6: [2023-05-09 16:44:16,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +24: [2023-05-09 16:44:16,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +31: [2023-05-09 16:44:16,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +23: [2023-05-09 16:44:16,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +23: [2023-05-09 16:44:16,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +30: [2023-05-09 16:44:16,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +11: [2023-05-09 16:44:16,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +22: [2023-05-09 16:44:16,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +20: [2023-05-09 16:44:16,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +31: [2023-05-09 16:44:16,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +24: [2023-05-09 16:44:16,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +23: [2023-05-09 16:44:16,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +19: [2023-05-09 16:44:16,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +19: [2023-05-09 16:44:16,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +20: [2023-05-09 16:44:16,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +31: [2023-05-09 16:44:16,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +18: [2023-05-09 16:44:16,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +15: [2023-05-09 16:44:16,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +18: [2023-05-09 16:44:16,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +15: [2023-05-09 16:44:16,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +31: [2023-05-09 16:44:16,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +25: [2023-05-09 16:44:16,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +15: [2023-05-09 16:44:16,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 6: [2023-05-09 16:44:16,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +24: [2023-05-09 16:44:16,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +25: [2023-05-09 16:44:16,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +20: [2023-05-09 16:44:16,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +20: [2023-05-09 16:44:16,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +30: [2023-05-09 16:44:16,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +21: [2023-05-09 16:44:16,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +29: [2023-05-09 16:44:16,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +29: [2023-05-09 16:44:16,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +29: [2023-05-09 16:44:16,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +23: [2023-05-09 16:44:16,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +24: [2023-05-09 16:44:16,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +15: [2023-05-09 16:44:16,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +18: [2023-05-09 16:44:16,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +18: [2023-05-09 16:44:16,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +21: [2023-05-09 16:44:16,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +29: [2023-05-09 16:44:16,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +10: [2023-05-09 16:44:16,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 1: [2023-05-09 16:44:16,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +22: [2023-05-09 16:44:16,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +22: [2023-05-09 16:44:16,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +17: [2023-05-09 16:44:16,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +22: [2023-05-09 16:44:16,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +22: [2023-05-09 16:44:16,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 9: [2023-05-09 16:44:16,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +11: [2023-05-09 16:44:16,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +12: [2023-05-09 16:44:16,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +14: [2023-05-09 16:44:16,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +21: [2023-05-09 16:44:16,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +22: [2023-05-09 16:44:16,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +11: [2023-05-09 16:44:16,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +11: [2023-05-09 16:44:16,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +15: [2023-05-09 16:44:16,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +22: [2023-05-09 16:44:16,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 3: [2023-05-09 16:44:16,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +21: [2023-05-09 16:44:16,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +11: [2023-05-09 16:44:16,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +11: [2023-05-09 16:44:16,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +14: [2023-05-09 16:44:16,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +14: [2023-05-09 16:44:16,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +17: [2023-05-09 16:44:16,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +22: [2023-05-09 16:44:16,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +22: [2023-05-09 16:44:16,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +23: [2023-05-09 16:44:16,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +10: [2023-05-09 16:44:16,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +10: [2023-05-09 16:44:16,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +25: [2023-05-09 16:44:16,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +25: [2023-05-09 16:44:16,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +10: [2023-05-09 16:44:16,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +11: [2023-05-09 16:44:16,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +19: [2023-05-09 16:44:16,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +30: [2023-05-09 16:44:16,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +30: [2023-05-09 16:44:16,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +19: [2023-05-09 16:44:16,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +19: [2023-05-09 16:44:16,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +19: [2023-05-09 16:44:16,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 5: [2023-05-09 16:44:16,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +11: [2023-05-09 16:44:16,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 5: [2023-05-09 16:44:16,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 5: [2023-05-09 16:44:16,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +23: [2023-05-09 16:44:16,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +27: [2023-05-09 16:44:16,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +27: [2023-05-09 16:44:16,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +27: [2023-05-09 16:44:16,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +27: [2023-05-09 16:44:16,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +27: [2023-05-09 16:44:16,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +27: [2023-05-09 16:44:16,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +27: [2023-05-09 16:44:16,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +27: [2023-05-09 16:44:16,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +27: [2023-05-09 16:44:16,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +27: [2023-05-09 16:44:16,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +27: [2023-05-09 16:44:16,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +27: [2023-05-09 16:44:16,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +27: [2023-05-09 16:44:16,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +30: [2023-05-09 16:44:16,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +30: [2023-05-09 16:44:16,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +27: [2023-05-09 16:44:16,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +27: [2023-05-09 16:44:16,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +30: [2023-05-09 16:44:16,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +30: [2023-05-09 16:44:16,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +23: [2023-05-09 16:44:16,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 5: [2023-05-09 16:44:16,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +11: [2023-05-09 16:44:16,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +30: [2023-05-09 16:44:16,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +23: [2023-05-09 16:44:16,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +27: [2023-05-09 16:44:16,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +23: [2023-05-09 16:44:16,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +10: [2023-05-09 16:44:16,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +11: [2023-05-09 16:44:16,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +11: [2023-05-09 16:44:16,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +11: [2023-05-09 16:44:16,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +26: [2023-05-09 16:44:16,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +26: [2023-05-09 16:44:16,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +26: [2023-05-09 16:44:16,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +26: [2023-05-09 16:44:16,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +26: [2023-05-09 16:44:16,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +26: [2023-05-09 16:44:16,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +26: [2023-05-09 16:44:16,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +26: [2023-05-09 16:44:16,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +26: [2023-05-09 16:44:16,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +26: [2023-05-09 16:44:16,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +26: [2023-05-09 16:44:16,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +26: [2023-05-09 16:44:16,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +26: [2023-05-09 16:44:16,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +26: [2023-05-09 16:44:16,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +24: [2023-05-09 16:44:16,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +24: [2023-05-09 16:44:16,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +24: [2023-05-09 16:44:16,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +24: [2023-05-09 16:44:16,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +26: [2023-05-09 16:44:16,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +24: [2023-05-09 16:44:16,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +25: [2023-05-09 16:44:16,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +24: [2023-05-09 16:44:16,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +15: [2023-05-09 16:44:16,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +12: [2023-05-09 16:44:16,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +24: [2023-05-09 16:44:16,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +23: [2023-05-09 16:44:16,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +23: [2023-05-09 16:44:16,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 6: [2023-05-09 16:44:16,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +15: [2023-05-09 16:44:16,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +21: [2023-05-09 16:44:16,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +25: [2023-05-09 16:44:16,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +24: [2023-05-09 16:44:16,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +23: [2023-05-09 16:44:16,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +26: [2023-05-09 16:44:16,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 3: [2023-05-09 16:44:16,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +21: [2023-05-09 16:44:16,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +24: [2023-05-09 16:44:16,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +24: [2023-05-09 16:44:16,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +24: [2023-05-09 16:44:16,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +24: [2023-05-09 16:44:16,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +17: [2023-05-09 16:44:16,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +15: [2023-05-09 16:44:16,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +15: [2023-05-09 16:44:16,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 3: [2023-05-09 16:44:16,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +19: [2023-05-09 16:44:16,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +19: [2023-05-09 16:44:16,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +19: [2023-05-09 16:44:16,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +19: [2023-05-09 16:44:16,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 3: [2023-05-09 16:44:16,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +25: [2023-05-09 16:44:16,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +21: [2023-05-09 16:44:16,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +21: [2023-05-09 16:44:16,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +17: [2023-05-09 16:44:16,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +19: [2023-05-09 16:44:16,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +10: [2023-05-09 16:44:16,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +10: [2023-05-09 16:44:16,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +24: [2023-05-09 16:44:16,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +24: [2023-05-09 16:44:16,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 3: [2023-05-09 16:44:16,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +10: [2023-05-09 16:44:16,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +31: [2023-05-09 16:44:16,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +31: [2023-05-09 16:44:16,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +31: [2023-05-09 16:44:16,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +31: [2023-05-09 16:44:16,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 0: [2023-05-09 16:44:16,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +15: [2023-05-09 16:44:16,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +10: [2023-05-09 16:44:16,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +15: [2023-05-09 16:44:16,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 0: [2023-05-09 16:44:16,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 0: [2023-05-09 16:44:16,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 0: [2023-05-09 16:44:16,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +18: [2023-05-09 16:44:16,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +18: [2023-05-09 16:44:16,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +18: [2023-05-09 16:44:16,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +18: [2023-05-09 16:44:16,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +18: [2023-05-09 16:44:16,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +18: [2023-05-09 16:44:16,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +18: [2023-05-09 16:44:16,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +18: [2023-05-09 16:44:16,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +18: [2023-05-09 16:44:16,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +18: [2023-05-09 16:44:16,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +18: [2023-05-09 16:44:16,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 7: [2023-05-09 16:44:16,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 7: [2023-05-09 16:44:16,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 7: [2023-05-09 16:44:16,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +27: [2023-05-09 16:44:16,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +18: [2023-05-09 16:44:16,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +18: [2023-05-09 16:44:16,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +27: [2023-05-09 16:44:16,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +18: [2023-05-09 16:44:16,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 7: [2023-05-09 16:44:16,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 7: [2023-05-09 16:44:16,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +15: [2023-05-09 16:44:16,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +27: [2023-05-09 16:44:16,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +15: [2023-05-09 16:44:16,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +21: [2023-05-09 16:44:16,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +21: [2023-05-09 16:44:16,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +31: [2023-05-09 16:44:16,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +18: [2023-05-09 16:44:16,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +30: [2023-05-09 16:44:16,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +31: [2023-05-09 16:44:16,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +23: [2023-05-09 16:44:16,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +11: [2023-05-09 16:44:16,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +11: [2023-05-09 16:44:16,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +19: [2023-05-09 16:44:16,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +18: [2023-05-09 16:44:16,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 4: [2023-05-09 16:44:16,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +30: [2023-05-09 16:44:16,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +25: [2023-05-09 16:44:16,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +23: [2023-05-09 16:44:16,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +26: [2023-05-09 16:44:16,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +26: [2023-05-09 16:44:16,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +26: [2023-05-09 16:44:16,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +26: [2023-05-09 16:44:16,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +24: [2023-05-09 16:44:16,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +31: [2023-05-09 16:44:16,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +25: [2023-05-09 16:44:16,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +25: [2023-05-09 16:44:16,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +25: [2023-05-09 16:44:16,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +25: [2023-05-09 16:44:16,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +27: [2023-05-09 16:44:16,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +24: [2023-05-09 16:44:16,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +24: [2023-05-09 16:44:16,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +24: [2023-05-09 16:44:16,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +31: [2023-05-09 16:44:16,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 4: [2023-05-09 16:44:16,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 5: [2023-05-09 16:44:16,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +21: [2023-05-09 16:44:16,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +27: [2023-05-09 16:44:16,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +18: [2023-05-09 16:44:16,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +11: [2023-05-09 16:44:16,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +27: [2023-05-09 16:44:16,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +29: [2023-05-09 16:44:16,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +21: [2023-05-09 16:44:16,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +29: [2023-05-09 16:44:16,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +29: [2023-05-09 16:44:16,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +29: [2023-05-09 16:44:16,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +29: [2023-05-09 16:44:16,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +29: [2023-05-09 16:44:16,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +29: [2023-05-09 16:44:16,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +29: [2023-05-09 16:44:16,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +29: [2023-05-09 16:44:16,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +29: [2023-05-09 16:44:16,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +29: [2023-05-09 16:44:16,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +29: [2023-05-09 16:44:16,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +29: [2023-05-09 16:44:16,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +29: [2023-05-09 16:44:16,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +29: [2023-05-09 16:44:16,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +29: [2023-05-09 16:44:16,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +11: [2023-05-09 16:44:16,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +26: [2023-05-09 16:44:16,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 1: [2023-05-09 16:44:16,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +25: [2023-05-09 16:44:16,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +26: [2023-05-09 16:44:16,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +26: [2023-05-09 16:44:16,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +24: [2023-05-09 16:44:16,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +24: [2023-05-09 16:44:16,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 9: [2023-05-09 16:44:16,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 9: [2023-05-09 16:44:16,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +26: [2023-05-09 16:44:16,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 9: [2023-05-09 16:44:16,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 9: [2023-05-09 16:44:16,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +25: [2023-05-09 16:44:16,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 9: [2023-05-09 16:44:16,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 9: [2023-05-09 16:44:16,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +24: [2023-05-09 16:44:16,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +27: [2023-05-09 16:44:16,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +18: [2023-05-09 16:44:16,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +18: [2023-05-09 16:44:16,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +24: [2023-05-09 16:44:16,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +27: [2023-05-09 16:44:16,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +18: [2023-05-09 16:44:16,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +18: [2023-05-09 16:44:16,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +24: [2023-05-09 16:44:16,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +29: [2023-05-09 16:44:16,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +29: [2023-05-09 16:44:16,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +29: [2023-05-09 16:44:16,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +29: [2023-05-09 16:44:16,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +18: [2023-05-09 16:44:16,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +24: [2023-05-09 16:44:16,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +18: [2023-05-09 16:44:16,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +18: [2023-05-09 16:44:16,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +29: [2023-05-09 16:44:16,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +29: [2023-05-09 16:44:16,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +29: [2023-05-09 16:44:16,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +29: [2023-05-09 16:44:16,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +28: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +28: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +28: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +28: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +28: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +28: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +20: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +20: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +20: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +20: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +20: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +20: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +20: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +20: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +20: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +28: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +28: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +28: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +28: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +28: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +28: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +28: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +20: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +20: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +20: [2023-05-09 16:44:16,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +20: [2023-05-09 16:44:16,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +20: [2023-05-09 16:44:16,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +20: [2023-05-09 16:44:16,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +20: [2023-05-09 16:44:16,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +28: [2023-05-09 16:44:16,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +28: [2023-05-09 16:44:16,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 1: [2023-05-09 16:44:16,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +28: [2023-05-09 16:44:16,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +20: [2023-05-09 16:44:16,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +20: [2023-05-09 16:44:16,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +28: [2023-05-09 16:44:16,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +28: [2023-05-09 16:44:16,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 8: [2023-05-09 16:44:16,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 8: [2023-05-09 16:44:16,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. + 8: [2023-05-09 16:44:16,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt. +20: [2023-05-09 16:44:16,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +20: [2023-05-09 16:44:16,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +28: [2023-05-09 16:44:16,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +20: [2023-05-09 16:44:16,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +28: [2023-05-09 16:44:16,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +28: [2023-05-09 16:44:16,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +20: [2023-05-09 16:44:16,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt... +28: [2023-05-09 16:44:16,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +20: [2023-05-09 16:44:16,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. +20: [2023-05-09 16:44:16,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +28: [2023-05-09 16:44:16,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +28: [2023-05-09 16:44:16,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 8: [2023-05-09 16:44:16,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 8: [2023-05-09 16:44:16,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... + 8: [2023-05-09 16:44:16,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt... +14: [2023-05-09 16:44:16,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +14: [2023-05-09 16:44:16,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +14: [2023-05-09 16:44:16,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +14: [2023-05-09 16:44:16,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +12: [2023-05-09 16:44:16,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +12: [2023-05-09 16:44:16,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +12: [2023-05-09 16:44:16,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +12: [2023-05-09 16:44:16,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +14: [2023-05-09 16:44:16,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +14: [2023-05-09 16:44:16,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +14: [2023-05-09 16:44:16,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +14: [2023-05-09 16:44:16,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +12: [2023-05-09 16:44:16,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +12: [2023-05-09 16:44:16,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +12: [2023-05-09 16:44:16,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +19: [2023-05-09 16:44:16,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +19: [2023-05-09 16:44:16,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +19: [2023-05-09 16:44:16,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +12: [2023-05-09 16:44:16,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +19: [2023-05-09 16:44:16,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +19: [2023-05-09 16:44:16,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +19: [2023-05-09 16:44:16,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +19: [2023-05-09 16:44:16,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +19: [2023-05-09 16:44:16,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 2: [2023-05-09 16:44:16,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 2: [2023-05-09 16:44:16,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 2: [2023-05-09 16:44:16,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +16: [2023-05-09 16:44:16,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +16: [2023-05-09 16:44:16,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +16: [2023-05-09 16:44:16,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +16: [2023-05-09 16:44:16,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +16: [2023-05-09 16:44:16,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +16: [2023-05-09 16:44:16,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +16: [2023-05-09 16:44:16,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +16: [2023-05-09 16:44:16,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +25: [2023-05-09 16:44:16,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +25: [2023-05-09 16:44:16,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +30: [2023-05-09 16:44:16,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +30: [2023-05-09 16:44:16,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +25: [2023-05-09 16:44:16,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +25: [2023-05-09 16:44:16,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +25: [2023-05-09 16:44:16,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +25: [2023-05-09 16:44:16,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +25: [2023-05-09 16:44:16,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +30: [2023-05-09 16:44:16,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +30: [2023-05-09 16:44:16,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +30: [2023-05-09 16:44:16,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +25: [2023-05-09 16:44:16,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +30: [2023-05-09 16:44:16,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +30: [2023-05-09 16:44:16,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +30: [2023-05-09 16:44:16,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +16: [2023-05-09 16:44:16,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +16: [2023-05-09 16:44:16,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +12: [2023-05-09 16:44:16,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +12: [2023-05-09 16:44:16,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +12: [2023-05-09 16:44:16,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +12: [2023-05-09 16:44:16,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +12: [2023-05-09 16:44:16,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +12: [2023-05-09 16:44:16,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +12: [2023-05-09 16:44:16,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +12: [2023-05-09 16:44:16,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +12: [2023-05-09 16:44:16,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +12: [2023-05-09 16:44:16,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +12: [2023-05-09 16:44:16,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +12: [2023-05-09 16:44:16,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +12: [2023-05-09 16:44:16,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +12: [2023-05-09 16:44:16,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +12: [2023-05-09 16:44:16,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +12: [2023-05-09 16:44:16,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +25: [2023-05-09 16:44:16,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +30: [2023-05-09 16:44:16,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +16: [2023-05-09 16:44:16,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +16: [2023-05-09 16:44:16,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +16: [2023-05-09 16:44:16,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +16: [2023-05-09 16:44:16,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +25: [2023-05-09 16:44:16,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +25: [2023-05-09 16:44:16,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +28: [2023-05-09 16:44:16,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +28: [2023-05-09 16:44:16,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +28: [2023-05-09 16:44:16,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +28: [2023-05-09 16:44:16,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +20: [2023-05-09 16:44:16,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +20: [2023-05-09 16:44:16,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +20: [2023-05-09 16:44:16,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +20: [2023-05-09 16:44:16,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +20: [2023-05-09 16:44:16,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +20: [2023-05-09 16:44:16,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +28: [2023-05-09 16:44:16,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +28: [2023-05-09 16:44:16,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +25: [2023-05-09 16:44:16,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +20: [2023-05-09 16:44:16,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +20: [2023-05-09 16:44:16,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +28: [2023-05-09 16:44:16,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +12: [2023-05-09 16:44:16,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +28: [2023-05-09 16:44:16,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +30: [2023-05-09 16:44:16,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +16: [2023-05-09 16:44:16,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +30: [2023-05-09 16:44:16,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +30: [2023-05-09 16:44:16,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +16: [2023-05-09 16:44:16,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +25: [2023-05-09 16:44:16,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +30: [2023-05-09 16:44:16,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +25: [2023-05-09 16:44:16,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +12: [2023-05-09 16:44:16,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +12: [2023-05-09 16:44:16,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +22: [2023-05-09 16:44:16,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +22: [2023-05-09 16:44:16,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +22: [2023-05-09 16:44:16,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +22: [2023-05-09 16:44:16,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +28: [2023-05-09 16:44:16,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +12: [2023-05-09 16:44:16,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +20: [2023-05-09 16:44:16,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +20: [2023-05-09 16:44:16,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +25: [2023-05-09 16:44:16,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +20: [2023-05-09 16:44:16,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +30: [2023-05-09 16:44:16,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +28: [2023-05-09 16:44:16,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +30: [2023-05-09 16:44:16,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +20: [2023-05-09 16:44:16,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +30: [2023-05-09 16:44:16,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +23: [2023-05-09 16:44:16,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +31: [2023-05-09 16:44:16,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +23: [2023-05-09 16:44:16,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +31: [2023-05-09 16:44:16,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +31: [2023-05-09 16:44:16,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +31: [2023-05-09 16:44:16,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +23: [2023-05-09 16:44:16,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +23: [2023-05-09 16:44:16,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +23: [2023-05-09 16:44:16,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +10: [2023-05-09 16:44:16,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +10: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +23: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +23: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +31: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +18: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +10: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +19: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +18: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +19: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +11: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +11: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +11: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +19: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +10: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +26: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +10: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +11: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +11: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +11: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +26: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +18: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +18: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +26: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +26: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +18: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +18: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +11: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +18: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +15: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +26: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +24: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +15: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +11: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +18: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +10: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +15: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +24: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +15: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +24: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +26: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +24: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +19: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +19: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +19: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +24: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +19: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +24: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +19: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +19: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +19: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +19: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +24: [2023-05-09 16:44:16,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +19: [2023-05-09 16:44:16,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 7: [2023-05-09 16:44:16,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +19: [2023-05-09 16:44:16,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +23: [2023-05-09 16:44:16,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +31: [2023-05-09 16:44:16,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +15: [2023-05-09 16:44:16,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +15: [2023-05-09 16:44:16,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +15: [2023-05-09 16:44:16,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +10: [2023-05-09 16:44:16,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +10: [2023-05-09 16:44:16,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +31: [2023-05-09 16:44:16,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +19: [2023-05-09 16:44:16,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +12: [2023-05-09 16:44:16,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +12: [2023-05-09 16:44:16,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +19: [2023-05-09 16:44:16,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +15: [2023-05-09 16:44:16,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +31: [2023-05-09 16:44:16,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +26: [2023-05-09 16:44:16,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +26: [2023-05-09 16:44:16,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +19: [2023-05-09 16:44:16,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +24: [2023-05-09 16:44:16,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +22: [2023-05-09 16:44:16,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +22: [2023-05-09 16:44:16,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +25: [2023-05-09 16:44:16,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +22: [2023-05-09 16:44:16,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +28: [2023-05-09 16:44:16,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +20: [2023-05-09 16:44:16,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +28: [2023-05-09 16:44:16,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +20: [2023-05-09 16:44:16,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +20: [2023-05-09 16:44:16,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +12: [2023-05-09 16:44:16,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +22: [2023-05-09 16:44:16,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +13: [2023-05-09 16:44:16,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +13: [2023-05-09 16:44:16,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +13: [2023-05-09 16:44:16,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +13: [2023-05-09 16:44:16,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +13: [2023-05-09 16:44:16,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +13: [2023-05-09 16:44:16,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +13: [2023-05-09 16:44:16,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +13: [2023-05-09 16:44:16,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +23: [2023-05-09 16:44:16,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +31: [2023-05-09 16:44:16,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +28: [2023-05-09 16:44:16,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +28: [2023-05-09 16:44:16,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +20: [2023-05-09 16:44:16,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +15: [2023-05-09 16:44:16,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +18: [2023-05-09 16:44:16,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +24: [2023-05-09 16:44:16,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +15: [2023-05-09 16:44:16,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +26: [2023-05-09 16:44:16,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +15: [2023-05-09 16:44:16,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +26: [2023-05-09 16:44:16,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +23: [2023-05-09 16:44:16,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +23: [2023-05-09 16:44:16,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +12: [2023-05-09 16:44:16,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +18: [2023-05-09 16:44:16,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +10: [2023-05-09 16:44:16,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +10: [2023-05-09 16:44:16,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +19: [2023-05-09 16:44:16,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +18: [2023-05-09 16:44:16,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +19: [2023-05-09 16:44:16,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +24: [2023-05-09 16:44:16,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +28: [2023-05-09 16:44:16,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +23: [2023-05-09 16:44:16,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +28: [2023-05-09 16:44:16,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +24: [2023-05-09 16:44:16,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +13: [2023-05-09 16:44:16,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +31: [2023-05-09 16:44:16,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +18: [2023-05-09 16:44:16,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +15: [2023-05-09 16:44:16,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +11: [2023-05-09 16:44:16,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +11: [2023-05-09 16:44:16,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +11: [2023-05-09 16:44:16,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +11: [2023-05-09 16:44:16,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +24: [2023-05-09 16:44:16,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +26: [2023-05-09 16:44:16,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +18: [2023-05-09 16:44:16,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +10: [2023-05-09 16:44:16,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +15: [2023-05-09 16:44:16,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +26: [2023-05-09 16:44:16,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +19: [2023-05-09 16:44:16,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +31: [2023-05-09 16:44:16,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +31: [2023-05-09 16:44:16,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +27: [2023-05-09 16:44:16,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +27: [2023-05-09 16:44:16,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +27: [2023-05-09 16:44:16,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +23: [2023-05-09 16:44:16,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +23: [2023-05-09 16:44:16,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +10: [2023-05-09 16:44:16,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +15: [2023-05-09 16:44:16,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +23: [2023-05-09 16:44:16,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +27: [2023-05-09 16:44:16,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 8: [2023-05-09 16:44:16,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +10: [2023-05-09 16:44:16,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +18: [2023-05-09 16:44:16,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +13: [2023-05-09 16:44:16,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +13: [2023-05-09 16:44:16,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +19: [2023-05-09 16:44:16,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +18: [2023-05-09 16:44:16,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +15: [2023-05-09 16:44:16,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +26: [2023-05-09 16:44:16,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +19: [2023-05-09 16:44:16,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +26: [2023-05-09 16:44:16,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +29: [2023-05-09 16:44:16,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +29: [2023-05-09 16:44:16,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +29: [2023-05-09 16:44:16,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +29: [2023-05-09 16:44:16,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +13: [2023-05-09 16:44:16,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +22: [2023-05-09 16:44:16,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +22: [2023-05-09 16:44:16,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +22: [2023-05-09 16:44:16,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +22: [2023-05-09 16:44:16,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +22: [2023-05-09 16:44:16,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +31: [2023-05-09 16:44:16,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +22: [2023-05-09 16:44:16,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +22: [2023-05-09 16:44:16,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +22: [2023-05-09 16:44:16,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +22: [2023-05-09 16:44:16,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +21: [2023-05-09 16:44:16,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +22: [2023-05-09 16:44:16,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +22: [2023-05-09 16:44:16,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +22: [2023-05-09 16:44:16,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +21: [2023-05-09 16:44:16,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +21: [2023-05-09 16:44:16,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +21: [2023-05-09 16:44:16,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +21: [2023-05-09 16:44:16,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +21: [2023-05-09 16:44:16,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +24: [2023-05-09 16:44:16,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +22: [2023-05-09 16:44:16,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +10: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +17: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +17: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +17: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +18: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +27: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 2: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +11: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +27: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +22: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 0: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +11: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +21: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +14: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +14: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +14: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +14: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +14: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +11: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +14: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +14: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +14: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +14: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +29: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +29: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +14: [2023-05-09 16:44:16,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +14: [2023-05-09 16:44:16,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +14: [2023-05-09 16:44:16,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +22: [2023-05-09 16:44:16,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +14: [2023-05-09 16:44:16,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +14: [2023-05-09 16:44:16,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 9: [2023-05-09 16:44:16,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +10: [2023-05-09 16:44:16,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +17: [2023-05-09 16:44:16,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +31: [2023-05-09 16:44:16,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +22: [2023-05-09 16:44:16,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +17: [2023-05-09 16:44:16,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +17: [2023-05-09 16:44:16,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +11: [2023-05-09 16:44:16,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +29: [2023-05-09 16:44:16,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +14: [2023-05-09 16:44:16,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +31: [2023-05-09 16:44:16,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +29: [2023-05-09 16:44:16,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +23: [2023-05-09 16:44:16,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +19: [2023-05-09 16:44:16,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +15: [2023-05-09 16:44:16,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +27: [2023-05-09 16:44:16,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +14: [2023-05-09 16:44:16,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +17: [2023-05-09 16:44:16,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +17: [2023-05-09 16:44:16,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +27: [2023-05-09 16:44:16,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +27: [2023-05-09 16:44:16,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +27: [2023-05-09 16:44:16,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +13: [2023-05-09 16:44:16,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +19: [2023-05-09 16:44:16,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +13: [2023-05-09 16:44:16,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +24: [2023-05-09 16:44:16,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +27: [2023-05-09 16:44:16,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +21: [2023-05-09 16:44:16,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +27: [2023-05-09 16:44:16,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +24: [2023-05-09 16:44:16,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +27: [2023-05-09 16:44:16,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +27: [2023-05-09 16:44:16,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +26: [2023-05-09 16:44:16,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +31: [2023-05-09 16:44:16,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 5: [2023-05-09 16:44:16,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +27: [2023-05-09 16:44:16,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +26: [2023-05-09 16:44:16,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +27: [2023-05-09 16:44:16,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +13: [2023-05-09 16:44:16,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +22: [2023-05-09 16:44:16,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +10: [2023-05-09 16:44:16,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +22: [2023-05-09 16:44:16,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +29: [2023-05-09 16:44:16,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +14: [2023-05-09 16:44:16,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +21: [2023-05-09 16:44:16,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +14: [2023-05-09 16:44:16,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +19: [2023-05-09 16:44:16,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +22: [2023-05-09 16:44:16,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +14: [2023-05-09 16:44:16,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +24: [2023-05-09 16:44:16,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 2: [2023-05-09 16:44:16,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +29: [2023-05-09 16:44:16,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +17: [2023-05-09 16:44:16,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +13: [2023-05-09 16:44:16,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +27: [2023-05-09 16:44:16,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +27: [2023-05-09 16:44:16,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +29: [2023-05-09 16:44:16,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +27: [2023-05-09 16:44:16,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +14: [2023-05-09 16:44:16,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +27: [2023-05-09 16:44:16,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +21: [2023-05-09 16:44:16,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +27: [2023-05-09 16:44:16,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +21: [2023-05-09 16:44:16,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +22: [2023-05-09 16:44:16,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +16: [2023-05-09 16:44:16,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +16: [2023-05-09 16:44:16,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +16: [2023-05-09 16:44:16,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +16: [2023-05-09 16:44:16,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +11: [2023-05-09 16:44:16,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +11: [2023-05-09 16:44:16,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +11: [2023-05-09 16:44:16,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +17: [2023-05-09 16:44:16,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +11: [2023-05-09 16:44:16,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 1: [2023-05-09 16:44:16,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +22: [2023-05-09 16:44:16,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +14: [2023-05-09 16:44:16,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +14: [2023-05-09 16:44:16,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +17: [2023-05-09 16:44:16,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +21: [2023-05-09 16:44:16,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +29: [2023-05-09 16:44:16,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +17: [2023-05-09 16:44:16,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +22: [2023-05-09 16:44:16,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +29: [2023-05-09 16:44:16,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +27: [2023-05-09 16:44:16,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +27: [2023-05-09 16:44:16,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +17: [2023-05-09 16:44:16,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +27: [2023-05-09 16:44:16,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +14: [2023-05-09 16:44:16,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +22: [2023-05-09 16:44:16,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +16: [2023-05-09 16:44:16,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +16: [2023-05-09 16:44:16,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +11: [2023-05-09 16:44:16,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +16: [2023-05-09 16:44:16,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +29: [2023-05-09 16:44:16,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +17: [2023-05-09 16:44:16,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +11: [2023-05-09 16:44:16,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +11: [2023-05-09 16:44:16,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +11: [2023-05-09 16:44:16,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +16: [2023-05-09 16:44:16,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +21: [2023-05-09 16:44:16,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +17: [2023-05-09 16:44:16,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +27: [2023-05-09 16:44:16,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +27: [2023-05-09 16:44:16,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +29: [2023-05-09 16:44:16,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +27: [2023-05-09 16:44:16,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 5: [2023-05-09 16:44:16,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +21: [2023-05-09 16:44:16,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +27: [2023-05-09 16:44:16,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 5: [2023-05-09 16:44:16,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +27: [2023-05-09 16:44:16,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +17: [2023-05-09 16:44:16,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +14: [2023-05-09 16:44:16,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +22: [2023-05-09 16:44:16,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +29: [2023-05-09 16:44:16,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 5: [2023-05-09 16:44:16,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 5: [2023-05-09 16:44:16,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +21: [2023-05-09 16:44:16,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +27: [2023-05-09 16:44:16,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +11: [2023-05-09 16:44:16,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +11: [2023-05-09 16:44:16,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +16: [2023-05-09 16:44:16,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +16: [2023-05-09 16:44:16,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +27: [2023-05-09 16:44:16,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 2: [2023-05-09 16:44:16,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +18: [2023-05-09 16:44:16,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +18: [2023-05-09 16:44:16,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +18: [2023-05-09 16:44:16,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +18: [2023-05-09 16:44:16,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +27: [2023-05-09 16:44:16,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +11: [2023-05-09 16:44:16,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +11: [2023-05-09 16:44:16,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +16: [2023-05-09 16:44:16,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +16: [2023-05-09 16:44:16,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +16: [2023-05-09 16:44:16,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +16: [2023-05-09 16:44:16,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +11: [2023-05-09 16:44:16,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +11: [2023-05-09 16:44:16,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +18: [2023-05-09 16:44:16,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +21: [2023-05-09 16:44:16,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +18: [2023-05-09 16:44:16,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +11: [2023-05-09 16:44:16,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +18: [2023-05-09 16:44:16,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +18: [2023-05-09 16:44:16,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +16: [2023-05-09 16:44:16,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +16: [2023-05-09 16:44:16,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +11: [2023-05-09 16:44:16,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 7: [2023-05-09 16:44:16,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 7: [2023-05-09 16:44:16,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 7: [2023-05-09 16:44:16,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 7: [2023-05-09 16:44:16,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +21: [2023-05-09 16:44:16,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +21: [2023-05-09 16:44:16,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +21: [2023-05-09 16:44:16,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +21: [2023-05-09 16:44:16,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +18: [2023-05-09 16:44:16,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +18: [2023-05-09 16:44:16,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +18: [2023-05-09 16:44:16,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +18: [2023-05-09 16:44:16,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 7: [2023-05-09 16:44:16,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +21: [2023-05-09 16:44:16,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +21: [2023-05-09 16:44:16,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +21: [2023-05-09 16:44:16,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +21: [2023-05-09 16:44:16,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 9: [2023-05-09 16:44:16,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 9: [2023-05-09 16:44:16,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +18: [2023-05-09 16:44:16,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +18: [2023-05-09 16:44:16,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +18: [2023-05-09 16:44:16,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +18: [2023-05-09 16:44:16,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +26: [2023-05-09 16:44:16,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +28: [2023-05-09 16:44:16,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +26: [2023-05-09 16:44:16,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +26: [2023-05-09 16:44:16,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +26: [2023-05-09 16:44:16,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +28: [2023-05-09 16:44:16,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +28: [2023-05-09 16:44:16,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +28: [2023-05-09 16:44:16,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +10: [2023-05-09 16:44:16,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +10: [2023-05-09 16:44:16,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +10: [2023-05-09 16:44:16,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +10: [2023-05-09 16:44:16,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +30: [2023-05-09 16:44:16,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +30: [2023-05-09 16:44:16,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +30: [2023-05-09 16:44:16,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +30: [2023-05-09 16:44:16,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +20: [2023-05-09 16:44:16,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +20: [2023-05-09 16:44:16,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +20: [2023-05-09 16:44:16,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +20: [2023-05-09 16:44:16,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +31: [2023-05-09 16:44:16,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 1: [2023-05-09 16:44:16,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 1: [2023-05-09 16:44:16,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 9: [2023-05-09 16:44:16,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +31: [2023-05-09 16:44:16,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 9: [2023-05-09 16:44:16,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 8: [2023-05-09 16:44:16,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 8: [2023-05-09 16:44:16,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 7: [2023-05-09 16:44:16,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +28: [2023-05-09 16:44:16,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +31: [2023-05-09 16:44:16,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 8: [2023-05-09 16:44:16,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 1: [2023-05-09 16:44:16,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +31: [2023-05-09 16:44:16,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +21: [2023-05-09 16:44:16,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 7: [2023-05-09 16:44:16,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 0: [2023-05-09 16:44:16,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 0: [2023-05-09 16:44:16,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +21: [2023-05-09 16:44:16,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 7: [2023-05-09 16:44:16,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 1: [2023-05-09 16:44:16,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 0: [2023-05-09 16:44:16,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +28: [2023-05-09 16:44:16,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +26: [2023-05-09 16:44:16,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +26: [2023-05-09 16:44:16,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +26: [2023-05-09 16:44:16,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +28: [2023-05-09 16:44:16,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +28: [2023-05-09 16:44:16,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +26: [2023-05-09 16:44:16,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +24: [2023-05-09 16:44:16,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +24: [2023-05-09 16:44:16,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +24: [2023-05-09 16:44:16,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +24: [2023-05-09 16:44:16,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +10: [2023-05-09 16:44:16,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +10: [2023-05-09 16:44:16,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 4: [2023-05-09 16:44:16,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +10: [2023-05-09 16:44:16,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +30: [2023-05-09 16:44:16,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +30: [2023-05-09 16:44:16,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +10: [2023-05-09 16:44:16,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +21: [2023-05-09 16:44:16,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 6: [2023-05-09 16:44:16,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 6: [2023-05-09 16:44:16,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +30: [2023-05-09 16:44:16,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +23: [2023-05-09 16:44:16,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +23: [2023-05-09 16:44:16,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +23: [2023-05-09 16:44:16,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +15: [2023-05-09 16:44:16,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +15: [2023-05-09 16:44:16,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +15: [2023-05-09 16:44:16,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +30: [2023-05-09 16:44:16,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +23: [2023-05-09 16:44:16,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +21: [2023-05-09 16:44:16,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +15: [2023-05-09 16:44:16,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +21: [2023-05-09 16:44:16,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +21: [2023-05-09 16:44:16,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +21: [2023-05-09 16:44:16,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 6: [2023-05-09 16:44:16,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +21: [2023-05-09 16:44:16,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +25: [2023-05-09 16:44:16,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +31: [2023-05-09 16:44:16,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +25: [2023-05-09 16:44:16,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +25: [2023-05-09 16:44:16,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +31: [2023-05-09 16:44:16,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +25: [2023-05-09 16:44:16,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +31: [2023-05-09 16:44:16,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +20: [2023-05-09 16:44:16,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +20: [2023-05-09 16:44:16,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +17: [2023-05-09 16:44:16,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +17: [2023-05-09 16:44:16,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +29: [2023-05-09 16:44:16,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +29: [2023-05-09 16:44:16,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +29: [2023-05-09 16:44:16,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +29: [2023-05-09 16:44:16,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +20: [2023-05-09 16:44:16,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +20: [2023-05-09 16:44:16,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +31: [2023-05-09 16:44:16,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +17: [2023-05-09 16:44:16,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 8: [2023-05-09 16:44:16,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 3: [2023-05-09 16:44:16,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 1: [2023-05-09 16:44:16,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 7: [2023-05-09 16:44:16,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 0: [2023-05-09 16:44:16,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 9: [2023-05-09 16:44:16,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 7: [2023-05-09 16:44:16,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +28: [2023-05-09 16:44:16,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +28: [2023-05-09 16:44:16,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 8: [2023-05-09 16:44:16,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 8: [2023-05-09 16:44:16,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +13: [2023-05-09 16:44:16,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +13: [2023-05-09 16:44:16,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 9: [2023-05-09 16:44:16,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 9: [2023-05-09 16:44:16,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +13: [2023-05-09 16:44:16,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 9: [2023-05-09 16:44:16,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 9: [2023-05-09 16:44:16,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +13: [2023-05-09 16:44:16,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. +17: [2023-05-09 16:44:16,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt. + 0: [2023-05-09 16:44:16,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 0: [2023-05-09 16:44:16,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 1: [2023-05-09 16:44:16,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +30: [2023-05-09 16:44:16,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +10: [2023-05-09 16:44:16,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +30: [2023-05-09 16:44:16,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +15: [2023-05-09 16:44:16,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +10: [2023-05-09 16:44:16,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +15: [2023-05-09 16:44:16,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +24: [2023-05-09 16:44:16,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +24: [2023-05-09 16:44:16,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +15: [2023-05-09 16:44:16,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 6: [2023-05-09 16:44:16,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +25: [2023-05-09 16:44:16,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +23: [2023-05-09 16:44:16,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +23: [2023-05-09 16:44:16,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +23: [2023-05-09 16:44:16,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 4: [2023-05-09 16:44:16,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +23: [2023-05-09 16:44:16,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +15: [2023-05-09 16:44:16,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +17: [2023-05-09 16:44:16,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +25: [2023-05-09 16:44:16,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +29: [2023-05-09 16:44:16,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +29: [2023-05-09 16:44:16,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +29: [2023-05-09 16:44:16,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +26: [2023-05-09 16:44:16,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +25: [2023-05-09 16:44:16,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +29: [2023-05-09 16:44:16,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +25: [2023-05-09 16:44:16,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +24: [2023-05-09 16:44:16,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +17: [2023-05-09 16:44:16,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +26: [2023-05-09 16:44:16,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 9: [2023-05-09 16:44:16,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +24: [2023-05-09 16:44:16,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 9: [2023-05-09 16:44:16,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +31: [2023-05-09 16:44:16,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +30: [2023-05-09 16:44:16,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +30: [2023-05-09 16:44:16,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +31: [2023-05-09 16:44:16,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +17: [2023-05-09 16:44:16,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +13: [2023-05-09 16:44:16,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +13: [2023-05-09 16:44:16,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +13: [2023-05-09 16:44:16,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +17: [2023-05-09 16:44:16,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 3: [2023-05-09 16:44:16,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... + 1: [2023-05-09 16:44:16,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +20: [2023-05-09 16:44:16,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +20: [2023-05-09 16:44:16,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +20: [2023-05-09 16:44:16,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +20: [2023-05-09 16:44:16,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +13: [2023-05-09 16:44:16,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +26: [2023-05-09 16:44:16,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt... +28: [2023-05-09 16:44:16,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +28: [2023-05-09 16:44:16,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +28: [2023-05-09 16:44:16,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +28: [2023-05-09 16:44:16,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +28: [2023-05-09 16:44:16,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +10: [2023-05-09 16:44:16,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +26: [2023-05-09 16:44:16,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +26: [2023-05-09 16:44:16,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +26: [2023-05-09 16:44:16,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +26: [2023-05-09 16:44:16,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +26: [2023-05-09 16:44:16,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +10: [2023-05-09 16:44:16,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +10: [2023-05-09 16:44:16,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +10: [2023-05-09 16:44:16,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +10: [2023-05-09 16:44:16,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +28: [2023-05-09 16:44:16,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 8: [2023-05-09 16:44:16,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +10: [2023-05-09 16:44:16,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 8: [2023-05-09 16:44:16,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +31: [2023-05-09 16:44:16,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +31: [2023-05-09 16:44:16,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +31: [2023-05-09 16:44:16,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +31: [2023-05-09 16:44:16,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +24: [2023-05-09 16:44:16,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +24: [2023-05-09 16:44:16,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +24: [2023-05-09 16:44:16,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +20: [2023-05-09 16:44:16,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +24: [2023-05-09 16:44:16,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +20: [2023-05-09 16:44:16,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 6: [2023-05-09 16:44:16,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 8: [2023-05-09 16:44:16,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 6: [2023-05-09 16:44:16,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +30: [2023-05-09 16:44:16,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 8: [2023-05-09 16:44:16,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 8: [2023-05-09 16:44:16,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +15: [2023-05-09 16:44:16,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +15: [2023-05-09 16:44:16,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +17: [2023-05-09 16:44:16,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +30: [2023-05-09 16:44:16,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +30: [2023-05-09 16:44:16,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +17: [2023-05-09 16:44:16,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 8: [2023-05-09 16:44:16,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +15: [2023-05-09 16:44:16,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +15: [2023-05-09 16:44:16,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +23: [2023-05-09 16:44:16,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +23: [2023-05-09 16:44:16,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +23: [2023-05-09 16:44:16,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +23: [2023-05-09 16:44:16,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 0: [2023-05-09 16:44:16,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +29: [2023-05-09 16:44:16,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +30: [2023-05-09 16:44:16,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +23: [2023-05-09 16:44:16,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 0: [2023-05-09 16:44:16,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +29: [2023-05-09 16:44:16,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +23: [2023-05-09 16:44:16,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 1: [2023-05-09 16:44:16,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +29: [2023-05-09 16:44:16,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +29: [2023-05-09 16:44:16,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +31: [2023-05-09 16:44:16,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +31: [2023-05-09 16:44:16,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 1: [2023-05-09 16:44:16,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 0: [2023-05-09 16:44:16,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +20: [2023-05-09 16:44:16,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +13: [2023-05-09 16:44:16,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 4: [2023-05-09 16:44:16,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +20: [2023-05-09 16:44:16,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +13: [2023-05-09 16:44:16,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 4: [2023-05-09 16:44:16,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +15: [2023-05-09 16:44:16,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +13: [2023-05-09 16:44:16,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +13: [2023-05-09 16:44:16,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +13: [2023-05-09 16:44:16,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +13: [2023-05-09 16:44:16,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +29: [2023-05-09 16:44:16,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +29: [2023-05-09 16:44:16,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +29: [2023-05-09 16:44:16,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +29: [2023-05-09 16:44:16,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 1: [2023-05-09 16:44:16,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 1: [2023-05-09 16:44:16,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +15: [2023-05-09 16:44:16,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 4: [2023-05-09 16:44:16,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +25: [2023-05-09 16:44:16,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +24: [2023-05-09 16:44:16,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +24: [2023-05-09 16:44:16,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +17: [2023-05-09 16:44:16,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 4: [2023-05-09 16:44:16,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +25: [2023-05-09 16:44:16,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +24: [2023-05-09 16:44:16,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +24: [2023-05-09 16:44:16,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +25: [2023-05-09 16:44:16,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +25: [2023-05-09 16:44:16,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +17: [2023-05-09 16:44:16,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +23: [2023-05-09 16:44:16,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +15: [2023-05-09 16:44:16,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +23: [2023-05-09 16:44:16,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 3: [2023-05-09 16:44:16,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +15: [2023-05-09 16:44:16,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +17: [2023-05-09 16:44:16,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +17: [2023-05-09 16:44:16,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 3: [2023-05-09 16:44:16,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 0: [2023-05-09 16:44:16,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 0: [2023-05-09 16:44:16,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +25: [2023-05-09 16:44:16,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +25: [2023-05-09 16:44:16,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 6: [2023-05-09 16:44:16,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +25: [2023-05-09 16:44:16,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 6: [2023-05-09 16:44:16,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 4: [2023-05-09 16:44:16,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +25: [2023-05-09 16:44:16,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 0: [2023-05-09 16:44:16,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +17: [2023-05-09 16:44:16,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 4: [2023-05-09 16:44:16,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 0: [2023-05-09 16:44:16,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +13: [2023-05-09 16:44:16,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. + 3: [2023-05-09 16:44:16,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt. +17: [2023-05-09 16:44:16,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +13: [2023-05-09 16:44:16,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... + 3: [2023-05-09 16:44:16,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt... +14: [2023-05-09 16:44:17,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +14: [2023-05-09 16:44:17,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +14: [2023-05-09 16:44:17,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +14: [2023-05-09 16:44:17,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +14: [2023-05-09 16:44:17,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +14: [2023-05-09 16:44:17,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +14: [2023-05-09 16:44:17,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +14: [2023-05-09 16:44:17,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 5: [2023-05-09 16:44:17,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 5: [2023-05-09 16:44:17,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 5: [2023-05-09 16:44:17,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 5: [2023-05-09 16:44:17,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +25: [2023-05-09 16:44:17,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +25: [2023-05-09 16:44:17,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +25: [2023-05-09 16:44:17,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +25: [2023-05-09 16:44:17,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +25: [2023-05-09 16:44:17,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +25: [2023-05-09 16:44:17,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +25: [2023-05-09 16:44:17,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +25: [2023-05-09 16:44:17,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +27: [2023-05-09 16:44:17,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +27: [2023-05-09 16:44:17,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +27: [2023-05-09 16:44:17,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +27: [2023-05-09 16:44:17,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 0: [2023-05-09 16:44:17,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +22: [2023-05-09 16:44:17,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +22: [2023-05-09 16:44:17,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +22: [2023-05-09 16:44:17,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +25: [2023-05-09 16:44:17,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +22: [2023-05-09 16:44:17,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +20: [2023-05-09 16:44:17,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +20: [2023-05-09 16:44:17,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +20: [2023-05-09 16:44:17,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +20: [2023-05-09 16:44:17,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +27: [2023-05-09 16:44:17,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +27: [2023-05-09 16:44:17,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +20: [2023-05-09 16:44:17,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +20: [2023-05-09 16:44:17,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +27: [2023-05-09 16:44:17,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +20: [2023-05-09 16:44:17,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +20: [2023-05-09 16:44:17,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +27: [2023-05-09 16:44:17,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +12: [2023-05-09 16:44:17,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +12: [2023-05-09 16:44:17,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 0: [2023-05-09 16:44:17,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +12: [2023-05-09 16:44:17,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +12: [2023-05-09 16:44:17,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +25: [2023-05-09 16:44:17,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +25: [2023-05-09 16:44:17,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +25: [2023-05-09 16:44:17,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +25: [2023-05-09 16:44:17,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +22: [2023-05-09 16:44:17,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +22: [2023-05-09 16:44:17,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +22: [2023-05-09 16:44:17,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +22: [2023-05-09 16:44:17,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +19: [2023-05-09 16:44:17,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +19: [2023-05-09 16:44:17,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +19: [2023-05-09 16:44:17,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +20: [2023-05-09 16:44:17,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +19: [2023-05-09 16:44:17,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +20: [2023-05-09 16:44:17,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +12: [2023-05-09 16:44:17,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +12: [2023-05-09 16:44:17,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +20: [2023-05-09 16:44:17,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +12: [2023-05-09 16:44:17,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +12: [2023-05-09 16:44:17,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +25: [2023-05-09 16:44:17,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +25: [2023-05-09 16:44:17,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +25: [2023-05-09 16:44:17,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +19: [2023-05-09 16:44:17,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +19: [2023-05-09 16:44:17,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +19: [2023-05-09 16:44:17,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +19: [2023-05-09 16:44:17,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +20: [2023-05-09 16:44:17,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +20: [2023-05-09 16:44:17,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +20: [2023-05-09 16:44:17,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +11: [2023-05-09 16:44:17,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +11: [2023-05-09 16:44:17,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +11: [2023-05-09 16:44:17,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 7: [2023-05-09 16:44:17,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +11: [2023-05-09 16:44:17,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 7: [2023-05-09 16:44:17,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 7: [2023-05-09 16:44:17,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 7: [2023-05-09 16:44:17,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +20: [2023-05-09 16:44:17,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +13: [2023-05-09 16:44:17,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +13: [2023-05-09 16:44:17,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +13: [2023-05-09 16:44:17,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +13: [2023-05-09 16:44:17,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +13: [2023-05-09 16:44:17,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +13: [2023-05-09 16:44:17,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +13: [2023-05-09 16:44:17,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +13: [2023-05-09 16:44:17,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +28: [2023-05-09 16:44:17,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +28: [2023-05-09 16:44:17,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +28: [2023-05-09 16:44:17,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +21: [2023-05-09 16:44:17,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +21: [2023-05-09 16:44:17,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +28: [2023-05-09 16:44:17,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +21: [2023-05-09 16:44:17,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +21: [2023-05-09 16:44:17,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +19: [2023-05-09 16:44:17,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +19: [2023-05-09 16:44:17,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +19: [2023-05-09 16:44:17,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +19: [2023-05-09 16:44:17,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +19: [2023-05-09 16:44:17,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +19: [2023-05-09 16:44:17,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 2: [2023-05-09 16:44:17,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 2: [2023-05-09 16:44:17,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 2: [2023-05-09 16:44:17,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +19: [2023-05-09 16:44:17,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +30: [2023-05-09 16:44:17,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 7: [2023-05-09 16:44:17,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +30: [2023-05-09 16:44:17,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +30: [2023-05-09 16:44:17,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +20: [2023-05-09 16:44:17,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +30: [2023-05-09 16:44:17,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 7: [2023-05-09 16:44:17,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +19: [2023-05-09 16:44:17,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +19: [2023-05-09 16:44:17,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +11: [2023-05-09 16:44:17,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +11: [2023-05-09 16:44:17,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +11: [2023-05-09 16:44:17,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +11: [2023-05-09 16:44:17,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +19: [2023-05-09 16:44:17,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +28: [2023-05-09 16:44:17,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +28: [2023-05-09 16:44:17,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +28: [2023-05-09 16:44:17,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +13: [2023-05-09 16:44:17,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +16: [2023-05-09 16:44:17,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +16: [2023-05-09 16:44:17,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +16: [2023-05-09 16:44:17,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 9: [2023-05-09 16:44:17,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +16: [2023-05-09 16:44:17,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 9: [2023-05-09 16:44:17,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +21: [2023-05-09 16:44:17,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +28: [2023-05-09 16:44:17,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +21: [2023-05-09 16:44:17,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +21: [2023-05-09 16:44:17,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +19: [2023-05-09 16:44:17,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +19: [2023-05-09 16:44:17,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +21: [2023-05-09 16:44:17,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 9: [2023-05-09 16:44:17,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +30: [2023-05-09 16:44:17,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +19: [2023-05-09 16:44:17,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +30: [2023-05-09 16:44:17,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +16: [2023-05-09 16:44:17,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +16: [2023-05-09 16:44:17,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +16: [2023-05-09 16:44:17,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +16: [2023-05-09 16:44:17,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +16: [2023-05-09 16:44:17,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +16: [2023-05-09 16:44:17,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +30: [2023-05-09 16:44:17,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +30: [2023-05-09 16:44:17,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +16: [2023-05-09 16:44:17,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +31: [2023-05-09 16:44:17,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +13: [2023-05-09 16:44:17,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +13: [2023-05-09 16:44:17,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +31: [2023-05-09 16:44:17,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +31: [2023-05-09 16:44:17,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +31: [2023-05-09 16:44:17,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +19: [2023-05-09 16:44:17,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +19: [2023-05-09 16:44:17,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +28: [2023-05-09 16:44:17,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +19: [2023-05-09 16:44:17,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +28: [2023-05-09 16:44:17,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +28: [2023-05-09 16:44:17,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +19: [2023-05-09 16:44:17,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +28: [2023-05-09 16:44:17,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +28: [2023-05-09 16:44:17,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +28: [2023-05-09 16:44:17,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +16: [2023-05-09 16:44:17,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +31: [2023-05-09 16:44:17,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +31: [2023-05-09 16:44:17,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +31: [2023-05-09 16:44:17,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +13: [2023-05-09 16:44:17,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +31: [2023-05-09 16:44:17,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 9: [2023-05-09 16:44:17,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +16: [2023-05-09 16:44:17,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +16: [2023-05-09 16:44:17,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 9: [2023-05-09 16:44:17,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +16: [2023-05-09 16:44:17,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +18: [2023-05-09 16:44:17,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +18: [2023-05-09 16:44:17,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +18: [2023-05-09 16:44:17,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +18: [2023-05-09 16:44:17,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +16: [2023-05-09 16:44:17,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +13: [2023-05-09 16:44:17,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +13: [2023-05-09 16:44:17,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +28: [2023-05-09 16:44:17,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +13: [2023-05-09 16:44:17,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +28: [2023-05-09 16:44:17,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +28: [2023-05-09 16:44:17,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +19: [2023-05-09 16:44:17,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 9: [2023-05-09 16:44:17,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +13: [2023-05-09 16:44:17,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +19: [2023-05-09 16:44:17,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +19: [2023-05-09 16:44:17,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +10: [2023-05-09 16:44:17,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +10: [2023-05-09 16:44:17,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +10: [2023-05-09 16:44:17,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +10: [2023-05-09 16:44:17,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +10: [2023-05-09 16:44:17,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +10: [2023-05-09 16:44:17,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +10: [2023-05-09 16:44:17,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +10: [2023-05-09 16:44:17,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +13: [2023-05-09 16:44:17,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +29: [2023-05-09 16:44:17,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +29: [2023-05-09 16:44:17,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +29: [2023-05-09 16:44:17,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +13: [2023-05-09 16:44:17,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +28: [2023-05-09 16:44:17,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +29: [2023-05-09 16:44:17,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +29: [2023-05-09 16:44:17,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +29: [2023-05-09 16:44:17,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +29: [2023-05-09 16:44:17,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +29: [2023-05-09 16:44:17,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 7: [2023-05-09 16:44:17,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 7: [2023-05-09 16:44:17,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +18: [2023-05-09 16:44:17,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +31: [2023-05-09 16:44:17,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +13: [2023-05-09 16:44:17,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +31: [2023-05-09 16:44:17,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +31: [2023-05-09 16:44:17,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +18: [2023-05-09 16:44:17,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +28: [2023-05-09 16:44:17,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +28: [2023-05-09 16:44:17,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +16: [2023-05-09 16:44:17,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +16: [2023-05-09 16:44:17,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +28: [2023-05-09 16:44:17,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +28: [2023-05-09 16:44:17,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +27: [2023-05-09 16:44:17,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +27: [2023-05-09 16:44:17,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +18: [2023-05-09 16:44:17,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +27: [2023-05-09 16:44:17,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +27: [2023-05-09 16:44:17,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +27: [2023-05-09 16:44:17,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +27: [2023-05-09 16:44:17,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +27: [2023-05-09 16:44:17,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +18: [2023-05-09 16:44:17,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +27: [2023-05-09 16:44:17,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +27: [2023-05-09 16:44:17,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +27: [2023-05-09 16:44:17,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +31: [2023-05-09 16:44:17,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +13: [2023-05-09 16:44:17,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +27: [2023-05-09 16:44:17,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +27: [2023-05-09 16:44:17,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +27: [2023-05-09 16:44:17,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +27: [2023-05-09 16:44:17,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +27: [2023-05-09 16:44:17,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +27: [2023-05-09 16:44:17,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +13: [2023-05-09 16:44:17,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +19: [2023-05-09 16:44:17,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +28: [2023-05-09 16:44:17,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +28: [2023-05-09 16:44:17,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +28: [2023-05-09 16:44:17,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +13: [2023-05-09 16:44:17,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +15: [2023-05-09 16:44:17,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +15: [2023-05-09 16:44:17,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +16: [2023-05-09 16:44:17,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +15: [2023-05-09 16:44:17,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +15: [2023-05-09 16:44:17,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +28: [2023-05-09 16:44:17,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +28: [2023-05-09 16:44:17,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +16: [2023-05-09 16:44:17,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +16: [2023-05-09 16:44:17,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +13: [2023-05-09 16:44:17,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +15: [2023-05-09 16:44:17,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +16: [2023-05-09 16:44:17,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +15: [2023-05-09 16:44:17,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +13: [2023-05-09 16:44:17,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +15: [2023-05-09 16:44:17,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +19: [2023-05-09 16:44:17,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +16: [2023-05-09 16:44:17,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +15: [2023-05-09 16:44:17,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +29: [2023-05-09 16:44:17,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +29: [2023-05-09 16:44:17,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 1: [2023-05-09 16:44:17,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 1: [2023-05-09 16:44:17,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +31: [2023-05-09 16:44:17,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +10: [2023-05-09 16:44:17,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +10: [2023-05-09 16:44:17,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +16: [2023-05-09 16:44:17,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +29: [2023-05-09 16:44:17,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +28: [2023-05-09 16:44:17,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +16: [2023-05-09 16:44:17,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +16: [2023-05-09 16:44:17,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +16: [2023-05-09 16:44:17,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +16: [2023-05-09 16:44:17,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +31: [2023-05-09 16:44:17,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +16: [2023-05-09 16:44:17,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +27: [2023-05-09 16:44:17,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +13: [2023-05-09 16:44:17,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +31: [2023-05-09 16:44:17,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +31: [2023-05-09 16:44:17,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +13: [2023-05-09 16:44:17,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +19: [2023-05-09 16:44:17,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +27: [2023-05-09 16:44:17,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +27: [2023-05-09 16:44:17,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +16: [2023-05-09 16:44:17,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +15: [2023-05-09 16:44:17,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +28: [2023-05-09 16:44:17,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +13: [2023-05-09 16:44:17,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +13: [2023-05-09 16:44:17,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +27: [2023-05-09 16:44:17,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +15: [2023-05-09 16:44:17,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +14: [2023-05-09 16:44:17,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +14: [2023-05-09 16:44:17,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +14: [2023-05-09 16:44:17,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +14: [2023-05-09 16:44:17,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +14: [2023-05-09 16:44:17,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +29: [2023-05-09 16:44:17,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +14: [2023-05-09 16:44:17,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +14: [2023-05-09 16:44:17,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +10: [2023-05-09 16:44:17,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +14: [2023-05-09 16:44:17,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +14: [2023-05-09 16:44:17,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +14: [2023-05-09 16:44:17,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +14: [2023-05-09 16:44:17,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +14: [2023-05-09 16:44:17,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +13: [2023-05-09 16:44:17,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +15: [2023-05-09 16:44:17,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +13: [2023-05-09 16:44:17,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +10: [2023-05-09 16:44:17,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +16: [2023-05-09 16:44:17,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +28: [2023-05-09 16:44:17,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +14: [2023-05-09 16:44:17,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +14: [2023-05-09 16:44:17,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +14: [2023-05-09 16:44:17,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +29: [2023-05-09 16:44:17,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +29: [2023-05-09 16:44:17,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +14: [2023-05-09 16:44:17,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +17: [2023-05-09 16:44:17,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 3: [2023-05-09 16:44:17,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 3: [2023-05-09 16:44:17,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +17: [2023-05-09 16:44:17,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +17: [2023-05-09 16:44:17,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +17: [2023-05-09 16:44:17,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +17: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +17: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +12: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +21: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +17: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +12: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +12: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +10: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +12: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +21: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +21: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +21: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +12: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +12: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +12: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +12: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +17: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +12: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +12: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +27: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +12: [2023-05-09 16:44:17,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +21: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +21: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +21: [2023-05-09 16:44:17,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +12: [2023-05-09 16:44:17,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +12: [2023-05-09 16:44:17,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +12: [2023-05-09 16:44:17,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +12: [2023-05-09 16:44:17,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +12: [2023-05-09 16:44:17,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +21: [2023-05-09 16:44:17,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +21: [2023-05-09 16:44:17,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +21: [2023-05-09 16:44:17,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +21: [2023-05-09 16:44:17,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +21: [2023-05-09 16:44:17,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +21: [2023-05-09 16:44:17,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +21: [2023-05-09 16:44:17,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 7: [2023-05-09 16:44:17,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +21: [2023-05-09 16:44:17,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +21: [2023-05-09 16:44:17,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +29: [2023-05-09 16:44:17,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +27: [2023-05-09 16:44:17,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +10: [2023-05-09 16:44:17,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +19: [2023-05-09 16:44:17,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +15: [2023-05-09 16:44:17,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +29: [2023-05-09 16:44:17,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +16: [2023-05-09 16:44:17,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +10: [2023-05-09 16:44:17,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +27: [2023-05-09 16:44:17,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +15: [2023-05-09 16:44:17,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +28: [2023-05-09 16:44:17,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +14: [2023-05-09 16:44:17,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +15: [2023-05-09 16:44:17,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +10: [2023-05-09 16:44:17,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +27: [2023-05-09 16:44:17,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +13: [2023-05-09 16:44:17,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +14: [2023-05-09 16:44:17,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +14: [2023-05-09 16:44:17,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +21: [2023-05-09 16:44:17,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +13: [2023-05-09 16:44:17,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 3: [2023-05-09 16:44:17,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +17: [2023-05-09 16:44:17,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +12: [2023-05-09 16:44:17,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +17: [2023-05-09 16:44:17,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +28: [2023-05-09 16:44:17,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +17: [2023-05-09 16:44:17,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +15: [2023-05-09 16:44:17,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +14: [2023-05-09 16:44:17,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +14: [2023-05-09 16:44:17,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +12: [2023-05-09 16:44:17,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +12: [2023-05-09 16:44:17,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +12: [2023-05-09 16:44:17,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +14: [2023-05-09 16:44:17,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +21: [2023-05-09 16:44:17,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +21: [2023-05-09 16:44:17,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +21: [2023-05-09 16:44:17,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +17: [2023-05-09 16:44:17,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +12: [2023-05-09 16:44:17,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +17: [2023-05-09 16:44:17,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +17: [2023-05-09 16:44:17,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +17: [2023-05-09 16:44:17,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +15: [2023-05-09 16:44:17,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +21: [2023-05-09 16:44:17,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +14: [2023-05-09 16:44:17,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +12: [2023-05-09 16:44:17,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +12: [2023-05-09 16:44:17,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +12: [2023-05-09 16:44:17,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +17: [2023-05-09 16:44:17,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +21: [2023-05-09 16:44:17,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +23: [2023-05-09 16:44:17,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +23: [2023-05-09 16:44:17,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +23: [2023-05-09 16:44:17,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +23: [2023-05-09 16:44:17,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +23: [2023-05-09 16:44:17,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +21: [2023-05-09 16:44:17,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +23: [2023-05-09 16:44:17,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +21: [2023-05-09 16:44:17,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +23: [2023-05-09 16:44:17,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +23: [2023-05-09 16:44:17,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +14: [2023-05-09 16:44:17,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 5: [2023-05-09 16:44:17,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 5: [2023-05-09 16:44:17,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 5: [2023-05-09 16:44:17,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +30: [2023-05-09 16:44:17,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +30: [2023-05-09 16:44:17,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +30: [2023-05-09 16:44:17,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +30: [2023-05-09 16:44:17,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +30: [2023-05-09 16:44:17,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +30: [2023-05-09 16:44:17,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +30: [2023-05-09 16:44:17,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +30: [2023-05-09 16:44:17,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +30: [2023-05-09 16:44:17,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +30: [2023-05-09 16:44:17,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +30: [2023-05-09 16:44:17,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +30: [2023-05-09 16:44:17,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +30: [2023-05-09 16:44:17,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +30: [2023-05-09 16:44:17,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +30: [2023-05-09 16:44:17,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +30: [2023-05-09 16:44:17,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 4: [2023-05-09 16:44:17,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 4: [2023-05-09 16:44:17,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 4: [2023-05-09 16:44:17,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 4: [2023-05-09 16:44:17,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +11: [2023-05-09 16:44:17,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +11: [2023-05-09 16:44:17,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +11: [2023-05-09 16:44:17,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +11: [2023-05-09 16:44:17,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +11: [2023-05-09 16:44:17,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +11: [2023-05-09 16:44:17,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +11: [2023-05-09 16:44:17,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +22: [2023-05-09 16:44:17,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +11: [2023-05-09 16:44:17,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +11: [2023-05-09 16:44:17,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +11: [2023-05-09 16:44:17,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +11: [2023-05-09 16:44:17,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +22: [2023-05-09 16:44:17,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +24: [2023-05-09 16:44:17,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +22: [2023-05-09 16:44:17,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +22: [2023-05-09 16:44:17,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +24: [2023-05-09 16:44:17,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +11: [2023-05-09 16:44:17,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +11: [2023-05-09 16:44:17,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +11: [2023-05-09 16:44:17,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +24: [2023-05-09 16:44:17,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +11: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +22: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +22: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +22: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +22: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +24: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +22: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +24: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +22: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +22: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +22: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +22: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +22: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +22: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +22: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +24: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +24: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 6: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 6: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 6: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +24: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 6: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 6: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 6: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +24: [2023-05-09 16:44:17,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +24: [2023-05-09 16:44:17,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +24: [2023-05-09 16:44:17,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 6: [2023-05-09 16:44:17,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 6: [2023-05-09 16:44:17,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +18: [2023-05-09 16:44:17,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +18: [2023-05-09 16:44:17,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +18: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +18: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +18: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +18: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +18: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +26: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +26: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +18: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +26: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +18: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +18: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +26: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +26: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +26: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 9: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +18: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +18: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +18: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 9: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +26: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 9: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 9: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 9: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 9: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +26: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +24: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 9: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +11: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 9: [2023-05-09 16:44:17,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 3: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 3: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 3: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +18: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 2: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 2: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 3: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 9: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 1: [2023-05-09 16:44:17,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 3: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 3: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +23: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +23: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +18: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +18: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 1: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 1: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +23: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +23: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +30: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +30: [2023-05-09 16:44:17,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 5: [2023-05-09 16:44:17,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 1: [2023-05-09 16:44:17,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +30: [2023-05-09 16:44:17,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +30: [2023-05-09 16:44:17,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +22: [2023-05-09 16:44:17,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +24: [2023-05-09 16:44:17,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +22: [2023-05-09 16:44:17,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +24: [2023-05-09 16:44:17,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 9: [2023-05-09 16:44:17,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +30: [2023-05-09 16:44:17,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +23: [2023-05-09 16:44:17,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +23: [2023-05-09 16:44:17,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +18: [2023-05-09 16:44:17,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +24: [2023-05-09 16:44:17,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +24: [2023-05-09 16:44:17,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 4: [2023-05-09 16:44:17,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +23: [2023-05-09 16:44:17,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +23: [2023-05-09 16:44:17,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 3: [2023-05-09 16:44:17,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +30: [2023-05-09 16:44:17,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +24: [2023-05-09 16:44:17,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 3: [2023-05-09 16:44:17,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +18: [2023-05-09 16:44:17,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +26: [2023-05-09 16:44:17,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +26: [2023-05-09 16:44:17,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +22: [2023-05-09 16:44:17,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +26: [2023-05-09 16:44:17,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +26: [2023-05-09 16:44:17,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +30: [2023-05-09 16:44:17,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +30: [2023-05-09 16:44:17,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +22: [2023-05-09 16:44:17,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 6: [2023-05-09 16:44:17,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 6: [2023-05-09 16:44:17,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 6: [2023-05-09 16:44:17,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +22: [2023-05-09 16:44:17,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 3: [2023-05-09 16:44:17,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 6: [2023-05-09 16:44:17,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +11: [2023-05-09 16:44:17,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +11: [2023-05-09 16:44:17,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +11: [2023-05-09 16:44:17,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +18: [2023-05-09 16:44:17,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 4: [2023-05-09 16:44:17,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 4: [2023-05-09 16:44:17,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +24: [2023-05-09 16:44:17,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +24: [2023-05-09 16:44:17,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +24: [2023-05-09 16:44:17,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +22: [2023-05-09 16:44:17,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +24: [2023-05-09 16:44:17,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 9: [2023-05-09 16:44:17,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +26: [2023-05-09 16:44:17,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +18: [2023-05-09 16:44:17,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 3: [2023-05-09 16:44:17,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +26: [2023-05-09 16:44:17,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +24: [2023-05-09 16:44:17,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +24: [2023-05-09 16:44:17,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 3: [2023-05-09 16:44:17,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +22: [2023-05-09 16:44:17,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 6: [2023-05-09 16:44:17,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +26: [2023-05-09 16:44:17,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 6: [2023-05-09 16:44:17,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +24: [2023-05-09 16:44:17,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +24: [2023-05-09 16:44:17,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 2: [2023-05-09 16:44:17,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 6: [2023-05-09 16:44:17,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 6: [2023-05-09 16:44:17,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +22: [2023-05-09 16:44:17,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 9: [2023-05-09 16:44:17,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +26: [2023-05-09 16:44:17,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +11: [2023-05-09 16:44:17,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +18: [2023-05-09 16:44:17,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +24: [2023-05-09 16:44:17,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 3: [2023-05-09 16:44:17,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +11: [2023-05-09 16:44:17,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 4: [2023-05-09 16:44:17,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +20: [2023-05-09 16:44:17,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +20: [2023-05-09 16:44:17,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +20: [2023-05-09 16:44:17,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +20: [2023-05-09 16:44:17,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 9: [2023-05-09 16:44:17,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +11: [2023-05-09 16:44:17,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +18: [2023-05-09 16:44:17,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +24: [2023-05-09 16:44:17,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +24: [2023-05-09 16:44:17,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +24: [2023-05-09 16:44:17,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +18: [2023-05-09 16:44:17,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +24: [2023-05-09 16:44:17,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +24: [2023-05-09 16:44:17,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 2: [2023-05-09 16:44:17,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +20: [2023-05-09 16:44:17,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +20: [2023-05-09 16:44:17,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +18: [2023-05-09 16:44:17,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +20: [2023-05-09 16:44:17,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +20: [2023-05-09 16:44:17,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +11: [2023-05-09 16:44:17,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +24: [2023-05-09 16:44:17,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +11: [2023-05-09 16:44:17,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +10: [2023-05-09 16:44:17,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +10: [2023-05-09 16:44:17,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +10: [2023-05-09 16:44:17,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +20: [2023-05-09 16:44:17,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +20: [2023-05-09 16:44:17,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +10: [2023-05-09 16:44:17,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +20: [2023-05-09 16:44:17,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +20: [2023-05-09 16:44:17,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +20: [2023-05-09 16:44:17,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +20: [2023-05-09 16:44:17,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +20: [2023-05-09 16:44:17,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +20: [2023-05-09 16:44:17,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 6: [2023-05-09 16:44:17,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 6: [2023-05-09 16:44:17,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 6: [2023-05-09 16:44:17,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 6: [2023-05-09 16:44:17,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +10: [2023-05-09 16:44:17,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +10: [2023-05-09 16:44:17,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +10: [2023-05-09 16:44:17,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 8: [2023-05-09 16:44:17,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 8: [2023-05-09 16:44:17,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +25: [2023-05-09 16:44:17,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +25: [2023-05-09 16:44:17,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +29: [2023-05-09 16:44:17,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +29: [2023-05-09 16:44:17,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +29: [2023-05-09 16:44:17,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +10: [2023-05-09 16:44:17,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +31: [2023-05-09 16:44:17,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +25: [2023-05-09 16:44:17,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +23: [2023-05-09 16:44:17,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +31: [2023-05-09 16:44:17,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +31: [2023-05-09 16:44:17,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +31: [2023-05-09 16:44:17,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +23: [2023-05-09 16:44:17,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +29: [2023-05-09 16:44:17,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +25: [2023-05-09 16:44:17,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 8: [2023-05-09 16:44:17,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +23: [2023-05-09 16:44:17,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 6: [2023-05-09 16:44:17,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +23: [2023-05-09 16:44:17,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 6: [2023-05-09 16:44:17,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 6: [2023-05-09 16:44:17,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 6: [2023-05-09 16:44:17,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +26: [2023-05-09 16:44:17,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +26: [2023-05-09 16:44:17,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +26: [2023-05-09 16:44:17,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 8: [2023-05-09 16:44:17,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +15: [2023-05-09 16:44:17,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +15: [2023-05-09 16:44:17,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 8: [2023-05-09 16:44:17,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 4: [2023-05-09 16:44:17,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 8: [2023-05-09 16:44:17,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +17: [2023-05-09 16:44:17,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +17: [2023-05-09 16:44:17,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +29: [2023-05-09 16:44:17,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +29: [2023-05-09 16:44:17,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +25: [2023-05-09 16:44:17,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +10: [2023-05-09 16:44:17,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +26: [2023-05-09 16:44:17,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +29: [2023-05-09 16:44:17,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +17: [2023-05-09 16:44:17,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +31: [2023-05-09 16:44:17,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +31: [2023-05-09 16:44:17,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +31: [2023-05-09 16:44:17,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +17: [2023-05-09 16:44:17,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +23: [2023-05-09 16:44:17,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +29: [2023-05-09 16:44:17,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +25: [2023-05-09 16:44:17,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 4: [2023-05-09 16:44:17,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 8: [2023-05-09 16:44:17,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +15: [2023-05-09 16:44:17,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +23: [2023-05-09 16:44:17,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +25: [2023-05-09 16:44:17,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +25: [2023-05-09 16:44:17,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 0: [2023-05-09 16:44:17,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +31: [2023-05-09 16:44:17,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +15: [2023-05-09 16:44:17,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +10: [2023-05-09 16:44:17,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +23: [2023-05-09 16:44:17,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +23: [2023-05-09 16:44:17,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 4: [2023-05-09 16:44:17,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. + 4: [2023-05-09 16:44:17,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt. +26: [2023-05-09 16:44:17,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +26: [2023-05-09 16:44:17,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +26: [2023-05-09 16:44:17,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 4: [2023-05-09 16:44:17,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +15: [2023-05-09 16:44:17,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +10: [2023-05-09 16:44:17,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +10: [2023-05-09 16:44:17,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +10: [2023-05-09 16:44:17,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +10: [2023-05-09 16:44:17,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 8: [2023-05-09 16:44:17,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +26: [2023-05-09 16:44:17,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +10: [2023-05-09 16:44:17,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 6: [2023-05-09 16:44:17,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +10: [2023-05-09 16:44:17,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 6: [2023-05-09 16:44:17,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 6: [2023-05-09 16:44:17,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +17: [2023-05-09 16:44:17,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 6: [2023-05-09 16:44:17,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +17: [2023-05-09 16:44:17,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +17: [2023-05-09 16:44:17,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +17: [2023-05-09 16:44:17,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +15: [2023-05-09 16:44:17,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +15: [2023-05-09 16:44:17,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 4: [2023-05-09 16:44:17,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +25: [2023-05-09 16:44:17,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +25: [2023-05-09 16:44:17,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +15: [2023-05-09 16:44:17,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +29: [2023-05-09 16:44:17,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +29: [2023-05-09 16:44:17,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +29: [2023-05-09 16:44:17,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +29: [2023-05-09 16:44:17,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 0: [2023-05-09 16:44:17,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +23: [2023-05-09 16:44:17,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +23: [2023-05-09 16:44:17,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 6: [2023-05-09 16:44:17,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 6: [2023-05-09 16:44:17,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +23: [2023-05-09 16:44:17,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 6: [2023-05-09 16:44:17,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +23: [2023-05-09 16:44:17,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 6: [2023-05-09 16:44:17,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 4: [2023-05-09 16:44:17,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... + 4: [2023-05-09 16:44:17,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt... +29: [2023-05-09 16:44:17,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +29: [2023-05-09 16:44:17,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 8: [2023-05-09 16:44:17,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +23: [2023-05-09 16:44:17,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +23: [2023-05-09 16:44:17,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +29: [2023-05-09 16:44:17,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +29: [2023-05-09 16:44:17,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +26: [2023-05-09 16:44:17,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +26: [2023-05-09 16:44:17,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 8: [2023-05-09 16:44:17,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +31: [2023-05-09 16:44:17,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +31: [2023-05-09 16:44:17,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +31: [2023-05-09 16:44:17,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +31: [2023-05-09 16:44:17,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +31: [2023-05-09 16:44:17,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +15: [2023-05-09 16:44:17,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +31: [2023-05-09 16:44:17,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +15: [2023-05-09 16:44:17,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +26: [2023-05-09 16:44:17,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +26: [2023-05-09 16:44:17,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +26: [2023-05-09 16:44:17,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +26: [2023-05-09 16:44:17,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +15: [2023-05-09 16:44:17,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +15: [2023-05-09 16:44:17,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +17: [2023-05-09 16:44:17,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +15: [2023-05-09 16:44:17,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +15: [2023-05-09 16:44:17,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +25: [2023-05-09 16:44:17,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +25: [2023-05-09 16:44:17,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +17: [2023-05-09 16:44:17,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +17: [2023-05-09 16:44:17,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +17: [2023-05-09 16:44:17,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +31: [2023-05-09 16:44:17,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +31: [2023-05-09 16:44:17,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +23: [2023-05-09 16:44:17,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +17: [2023-05-09 16:44:17,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +17: [2023-05-09 16:44:17,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +17: [2023-05-09 16:44:17,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +17: [2023-05-09 16:44:17,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +26: [2023-05-09 16:44:17,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +26: [2023-05-09 16:44:17,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +23: [2023-05-09 16:44:17,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 4: [2023-05-09 16:44:17,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 4: [2023-05-09 16:44:17,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +25: [2023-05-09 16:44:17,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +25: [2023-05-09 16:44:17,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. +25: [2023-05-09 16:44:17,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +25: [2023-05-09 16:44:17,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 0: [2023-05-09 16:44:17,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +15: [2023-05-09 16:44:17,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 0: [2023-05-09 16:44:17,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 0: [2023-05-09 16:44:17,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +15: [2023-05-09 16:44:17,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 4: [2023-05-09 16:44:17,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... + 4: [2023-05-09 16:44:17,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt... +16: [2023-05-09 16:44:17,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +16: [2023-05-09 16:44:17,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +16: [2023-05-09 16:44:17,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +16: [2023-05-09 16:44:17,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +16: [2023-05-09 16:44:17,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +16: [2023-05-09 16:44:17,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +16: [2023-05-09 16:44:17,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +16: [2023-05-09 16:44:17,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +25: [2023-05-09 16:44:17,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +25: [2023-05-09 16:44:17,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +25: [2023-05-09 16:44:17,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +25: [2023-05-09 16:44:17,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +25: [2023-05-09 16:44:17,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +25: [2023-05-09 16:44:17,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +25: [2023-05-09 16:44:17,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +25: [2023-05-09 16:44:17,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +27: [2023-05-09 16:44:17,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +27: [2023-05-09 16:44:17,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +27: [2023-05-09 16:44:17,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +27: [2023-05-09 16:44:17,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 0: [2023-05-09 16:44:17,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +14: [2023-05-09 16:44:17,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +14: [2023-05-09 16:44:17,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +14: [2023-05-09 16:44:17,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +14: [2023-05-09 16:44:17,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +14: [2023-05-09 16:44:17,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +14: [2023-05-09 16:44:17,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +14: [2023-05-09 16:44:17,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +25: [2023-05-09 16:44:17,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +14: [2023-05-09 16:44:17,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +27: [2023-05-09 16:44:17,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +27: [2023-05-09 16:44:17,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +31: [2023-05-09 16:44:17,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +31: [2023-05-09 16:44:17,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +31: [2023-05-09 16:44:17,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +13: [2023-05-09 16:44:17,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +13: [2023-05-09 16:44:17,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +13: [2023-05-09 16:44:17,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +31: [2023-05-09 16:44:17,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +13: [2023-05-09 16:44:17,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +31: [2023-05-09 16:44:17,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +31: [2023-05-09 16:44:17,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +31: [2023-05-09 16:44:17,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +27: [2023-05-09 16:44:17,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +13: [2023-05-09 16:44:17,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +13: [2023-05-09 16:44:17,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +13: [2023-05-09 16:44:17,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +27: [2023-05-09 16:44:17,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +10: [2023-05-09 16:44:17,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +10: [2023-05-09 16:44:17,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +10: [2023-05-09 16:44:17,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +31: [2023-05-09 16:44:17,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +10: [2023-05-09 16:44:17,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +10: [2023-05-09 16:44:17,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +10: [2023-05-09 16:44:17,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +13: [2023-05-09 16:44:17,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +10: [2023-05-09 16:44:17,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +27: [2023-05-09 16:44:17,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +27: [2023-05-09 16:44:17,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +27: [2023-05-09 16:44:17,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +27: [2023-05-09 16:44:17,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +10: [2023-05-09 16:44:17,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +25: [2023-05-09 16:44:17,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +27: [2023-05-09 16:44:17,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +27: [2023-05-09 16:44:17,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +14: [2023-05-09 16:44:17,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +27: [2023-05-09 16:44:17,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +27: [2023-05-09 16:44:17,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +14: [2023-05-09 16:44:17,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +25: [2023-05-09 16:44:17,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +25: [2023-05-09 16:44:17,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +25: [2023-05-09 16:44:17,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +31: [2023-05-09 16:44:17,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +31: [2023-05-09 16:44:17,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +13: [2023-05-09 16:44:17,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +31: [2023-05-09 16:44:17,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +14: [2023-05-09 16:44:17,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +10: [2023-05-09 16:44:17,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +19: [2023-05-09 16:44:17,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +19: [2023-05-09 16:44:17,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +19: [2023-05-09 16:44:17,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +19: [2023-05-09 16:44:17,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +19: [2023-05-09 16:44:17,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +19: [2023-05-09 16:44:17,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +19: [2023-05-09 16:44:17,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +19: [2023-05-09 16:44:17,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +10: [2023-05-09 16:44:17,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +27: [2023-05-09 16:44:17,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +13: [2023-05-09 16:44:17,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +13: [2023-05-09 16:44:17,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +27: [2023-05-09 16:44:17,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 7: [2023-05-09 16:44:17,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +16: [2023-05-09 16:44:17,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +16: [2023-05-09 16:44:17,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +16: [2023-05-09 16:44:17,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +16: [2023-05-09 16:44:17,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +16: [2023-05-09 16:44:17,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +16: [2023-05-09 16:44:17,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +16: [2023-05-09 16:44:17,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +16: [2023-05-09 16:44:17,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +16: [2023-05-09 16:44:17,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +16: [2023-05-09 16:44:17,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +16: [2023-05-09 16:44:17,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +16: [2023-05-09 16:44:17,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +14: [2023-05-09 16:44:17,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +16: [2023-05-09 16:44:17,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +16: [2023-05-09 16:44:17,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +16: [2023-05-09 16:44:17,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +25: [2023-05-09 16:44:17,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +31: [2023-05-09 16:44:17,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +31: [2023-05-09 16:44:17,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +16: [2023-05-09 16:44:17,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +27: [2023-05-09 16:44:17,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +25: [2023-05-09 16:44:17,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +25: [2023-05-09 16:44:17,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +14: [2023-05-09 16:44:17,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +31: [2023-05-09 16:44:17,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +27: [2023-05-09 16:44:17,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +14: [2023-05-09 16:44:17,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +13: [2023-05-09 16:44:17,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +27: [2023-05-09 16:44:17,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +10: [2023-05-09 16:44:17,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +21: [2023-05-09 16:44:17,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +21: [2023-05-09 16:44:17,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +21: [2023-05-09 16:44:17,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +21: [2023-05-09 16:44:17,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +27: [2023-05-09 16:44:17,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +14: [2023-05-09 16:44:17,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +12: [2023-05-09 16:44:17,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +12: [2023-05-09 16:44:17,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +11: [2023-05-09 16:44:17,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +12: [2023-05-09 16:44:17,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +22: [2023-05-09 16:44:17,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +12: [2023-05-09 16:44:17,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +12: [2023-05-09 16:44:17,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +11: [2023-05-09 16:44:17,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +11: [2023-05-09 16:44:17,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +22: [2023-05-09 16:44:17,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +11: [2023-05-09 16:44:17,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +10: [2023-05-09 16:44:17,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +22: [2023-05-09 16:44:17,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +22: [2023-05-09 16:44:17,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +13: [2023-05-09 16:44:17,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +13: [2023-05-09 16:44:17,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +19: [2023-05-09 16:44:17,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +21: [2023-05-09 16:44:17,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +21: [2023-05-09 16:44:17,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +11: [2023-05-09 16:44:17,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +11: [2023-05-09 16:44:17,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +21: [2023-05-09 16:44:17,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +11: [2023-05-09 16:44:17,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +22: [2023-05-09 16:44:17,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +20: [2023-05-09 16:44:17,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +20: [2023-05-09 16:44:17,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +20: [2023-05-09 16:44:17,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +22: [2023-05-09 16:44:17,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +20: [2023-05-09 16:44:17,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +20: [2023-05-09 16:44:17,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +20: [2023-05-09 16:44:17,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +28: [2023-05-09 16:44:17,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +28: [2023-05-09 16:44:17,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +28: [2023-05-09 16:44:17,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +12: [2023-05-09 16:44:17,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +27: [2023-05-09 16:44:17,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +28: [2023-05-09 16:44:17,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +28: [2023-05-09 16:44:17,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +20: [2023-05-09 16:44:17,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +12: [2023-05-09 16:44:17,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +20: [2023-05-09 16:44:17,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +22: [2023-05-09 16:44:17,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +22: [2023-05-09 16:44:17,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +12: [2023-05-09 16:44:17,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +11: [2023-05-09 16:44:17,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +21: [2023-05-09 16:44:17,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +27: [2023-05-09 16:44:17,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +29: [2023-05-09 16:44:17,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +29: [2023-05-09 16:44:17,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +29: [2023-05-09 16:44:17,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +28: [2023-05-09 16:44:17,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +28: [2023-05-09 16:44:17,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +28: [2023-05-09 16:44:17,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +29: [2023-05-09 16:44:17,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +29: [2023-05-09 16:44:17,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +27: [2023-05-09 16:44:17,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +29: [2023-05-09 16:44:17,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +31: [2023-05-09 16:44:17,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +26: [2023-05-09 16:44:17,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +26: [2023-05-09 16:44:17,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +26: [2023-05-09 16:44:17,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +26: [2023-05-09 16:44:17,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +26: [2023-05-09 16:44:17,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +26: [2023-05-09 16:44:17,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +10: [2023-05-09 16:44:17,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +19: [2023-05-09 16:44:17,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +19: [2023-05-09 16:44:17,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +29: [2023-05-09 16:44:17,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +29: [2023-05-09 16:44:17,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +16: [2023-05-09 16:44:17,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +26: [2023-05-09 16:44:17,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +26: [2023-05-09 16:44:17,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +16: [2023-05-09 16:44:17,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +27: [2023-05-09 16:44:17,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +13: [2023-05-09 16:44:17,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 9: [2023-05-09 16:44:17,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 3: [2023-05-09 16:44:17,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +27: [2023-05-09 16:44:17,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +23: [2023-05-09 16:44:17,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +14: [2023-05-09 16:44:17,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +23: [2023-05-09 16:44:17,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +23: [2023-05-09 16:44:17,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +30: [2023-05-09 16:44:17,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +23: [2023-05-09 16:44:17,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +30: [2023-05-09 16:44:17,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 9: [2023-05-09 16:44:17,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +27: [2023-05-09 16:44:17,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +23: [2023-05-09 16:44:17,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +23: [2023-05-09 16:44:17,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 3: [2023-05-09 16:44:17,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +30: [2023-05-09 16:44:17,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +30: [2023-05-09 16:44:17,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +27: [2023-05-09 16:44:17,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +10: [2023-05-09 16:44:17,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +30: [2023-05-09 16:44:17,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +22: [2023-05-09 16:44:17,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +23: [2023-05-09 16:44:17,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +30: [2023-05-09 16:44:17,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +23: [2023-05-09 16:44:17,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +30: [2023-05-09 16:44:17,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +30: [2023-05-09 16:44:17,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +27: [2023-05-09 16:44:17,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +28: [2023-05-09 16:44:17,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +19: [2023-05-09 16:44:17,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +21: [2023-05-09 16:44:17,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +20: [2023-05-09 16:44:17,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +18: [2023-05-09 16:44:17,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +18: [2023-05-09 16:44:17,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +10: [2023-05-09 16:44:17,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +31: [2023-05-09 16:44:17,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +20: [2023-05-09 16:44:17,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +18: [2023-05-09 16:44:17,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +18: [2023-05-09 16:44:17,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +22: [2023-05-09 16:44:17,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +18: [2023-05-09 16:44:17,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +18: [2023-05-09 16:44:17,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +18: [2023-05-09 16:44:17,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +19: [2023-05-09 16:44:17,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 6: [2023-05-09 16:44:17,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 6: [2023-05-09 16:44:17,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +18: [2023-05-09 16:44:17,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 6: [2023-05-09 16:44:17,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 6: [2023-05-09 16:44:17,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 6: [2023-05-09 16:44:17,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 6: [2023-05-09 16:44:17,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 6: [2023-05-09 16:44:17,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 6: [2023-05-09 16:44:17,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +13: [2023-05-09 16:44:17,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +16: [2023-05-09 16:44:17,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +20: [2023-05-09 16:44:17,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +20: [2023-05-09 16:44:17,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +19: [2023-05-09 16:44:17,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +29: [2023-05-09 16:44:17,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +21: [2023-05-09 16:44:17,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +21: [2023-05-09 16:44:17,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +16: [2023-05-09 16:44:17,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +17: [2023-05-09 16:44:17,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +17: [2023-05-09 16:44:17,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +17: [2023-05-09 16:44:17,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +17: [2023-05-09 16:44:17,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 4: [2023-05-09 16:44:17,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 4: [2023-05-09 16:44:17,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 4: [2023-05-09 16:44:17,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +16: [2023-05-09 16:44:17,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +27: [2023-05-09 16:44:17,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +12: [2023-05-09 16:44:17,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +26: [2023-05-09 16:44:17,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +15: [2023-05-09 16:44:17,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +12: [2023-05-09 16:44:17,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +15: [2023-05-09 16:44:17,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 3: [2023-05-09 16:44:17,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +17: [2023-05-09 16:44:17,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +26: [2023-05-09 16:44:17,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +17: [2023-05-09 16:44:17,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +10: [2023-05-09 16:44:17,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +22: [2023-05-09 16:44:17,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +16: [2023-05-09 16:44:17,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +28: [2023-05-09 16:44:17,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +15: [2023-05-09 16:44:17,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +30: [2023-05-09 16:44:17,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +17: [2023-05-09 16:44:17,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +19: [2023-05-09 16:44:17,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +17: [2023-05-09 16:44:17,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +27: [2023-05-09 16:44:17,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +11: [2023-05-09 16:44:17,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +11: [2023-05-09 16:44:17,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +11: [2023-05-09 16:44:17,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +23: [2023-05-09 16:44:17,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +15: [2023-05-09 16:44:17,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +23: [2023-05-09 16:44:17,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +15: [2023-05-09 16:44:17,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +15: [2023-05-09 16:44:17,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +15: [2023-05-09 16:44:17,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +20: [2023-05-09 16:44:17,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +22: [2023-05-09 16:44:17,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +15: [2023-05-09 16:44:17,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +20: [2023-05-09 16:44:17,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +21: [2023-05-09 16:44:17,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +16: [2023-05-09 16:44:17,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +18: [2023-05-09 16:44:17,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +20: [2023-05-09 16:44:17,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +20: [2023-05-09 16:44:17,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +22: [2023-05-09 16:44:17,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +12: [2023-05-09 16:44:17,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 6: [2023-05-09 16:44:17,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +29: [2023-05-09 16:44:17,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +26: [2023-05-09 16:44:17,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +16: [2023-05-09 16:44:17,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +12: [2023-05-09 16:44:17,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +28: [2023-05-09 16:44:17,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +29: [2023-05-09 16:44:17,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +12: [2023-05-09 16:44:17,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +26: [2023-05-09 16:44:17,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +21: [2023-05-09 16:44:17,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +21: [2023-05-09 16:44:17,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +30: [2023-05-09 16:44:17,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +19: [2023-05-09 16:44:17,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +29: [2023-05-09 16:44:17,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +15: [2023-05-09 16:44:17,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +17: [2023-05-09 16:44:17,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +23: [2023-05-09 16:44:17,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +29: [2023-05-09 16:44:17,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +28: [2023-05-09 16:44:17,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +28: [2023-05-09 16:44:17,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +23: [2023-05-09 16:44:17,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 3: [2023-05-09 16:44:17,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +26: [2023-05-09 16:44:17,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +26: [2023-05-09 16:44:17,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +11: [2023-05-09 16:44:17,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +18: [2023-05-09 16:44:17,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +11: [2023-05-09 16:44:17,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +22: [2023-05-09 16:44:17,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +15: [2023-05-09 16:44:17,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +21: [2023-05-09 16:44:17,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 6: [2023-05-09 16:44:17,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 6: [2023-05-09 16:44:17,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +11: [2023-05-09 16:44:17,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +24: [2023-05-09 16:44:17,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +24: [2023-05-09 16:44:17,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 6: [2023-05-09 16:44:17,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +24: [2023-05-09 16:44:17,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +24: [2023-05-09 16:44:17,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +24: [2023-05-09 16:44:17,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +30: [2023-05-09 16:44:17,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +24: [2023-05-09 16:44:17,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +24: [2023-05-09 16:44:17,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +12: [2023-05-09 16:44:17,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +30: [2023-05-09 16:44:17,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +30: [2023-05-09 16:44:17,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +12: [2023-05-09 16:44:17,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +22: [2023-05-09 16:44:17,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +23: [2023-05-09 16:44:17,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +23: [2023-05-09 16:44:17,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +29: [2023-05-09 16:44:17,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 6: [2023-05-09 16:44:17,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +24: [2023-05-09 16:44:17,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +17: [2023-05-09 16:44:17,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +17: [2023-05-09 16:44:17,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +28: [2023-05-09 16:44:17,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +29: [2023-05-09 16:44:17,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +18: [2023-05-09 16:44:17,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +29: [2023-05-09 16:44:17,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +18: [2023-05-09 16:44:17,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +15: [2023-05-09 16:44:17,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 9: [2023-05-09 16:44:17,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +28: [2023-05-09 16:44:17,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +11: [2023-05-09 16:44:17,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +26: [2023-05-09 16:44:17,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +28: [2023-05-09 16:44:17,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +18: [2023-05-09 16:44:17,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 4: [2023-05-09 16:44:17,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 4: [2023-05-09 16:44:17,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +26: [2023-05-09 16:44:17,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +22: [2023-05-09 16:44:17,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +15: [2023-05-09 16:44:17,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 9: [2023-05-09 16:44:17,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +17: [2023-05-09 16:44:17,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +21: [2023-05-09 16:44:17,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +30: [2023-05-09 16:44:17,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 6: [2023-05-09 16:44:17,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 6: [2023-05-09 16:44:17,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 3: [2023-05-09 16:44:17,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +30: [2023-05-09 16:44:17,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +30: [2023-05-09 16:44:17,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 3: [2023-05-09 16:44:17,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 4: [2023-05-09 16:44:17,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +12: [2023-05-09 16:44:17,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 6: [2023-05-09 16:44:17,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +23: [2023-05-09 16:44:17,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +23: [2023-05-09 16:44:17,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +17: [2023-05-09 16:44:17,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +17: [2023-05-09 16:44:17,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +15: [2023-05-09 16:44:17,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +15: [2023-05-09 16:44:17,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +18: [2023-05-09 16:44:17,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +18: [2023-05-09 16:44:17,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 4: [2023-05-09 16:44:17,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +18: [2023-05-09 16:44:17,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +11: [2023-05-09 16:44:17,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 9: [2023-05-09 16:44:17,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +17: [2023-05-09 16:44:17,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +24: [2023-05-09 16:44:17,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +24: [2023-05-09 16:44:17,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +24: [2023-05-09 16:44:17,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +17: [2023-05-09 16:44:17,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +15: [2023-05-09 16:44:17,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +15: [2023-05-09 16:44:17,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 4: [2023-05-09 16:44:17,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +24: [2023-05-09 16:44:17,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +24: [2023-05-09 16:44:17,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +24: [2023-05-09 16:44:17,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +24: [2023-05-09 16:44:17,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +24: [2023-05-09 16:44:17,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +14: [2023-05-09 16:44:17,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +14: [2023-05-09 16:44:17,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +14: [2023-05-09 16:44:17,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +14: [2023-05-09 16:44:17,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 9: [2023-05-09 16:44:17,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 9: [2023-05-09 16:44:17,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 9: [2023-05-09 16:44:17,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 9: [2023-05-09 16:44:17,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 1: [2023-05-09 16:44:17,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 1: [2023-05-09 16:44:17,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 1: [2023-05-09 16:44:17,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 1: [2023-05-09 16:44:17,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +14: [2023-05-09 16:44:17,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +14: [2023-05-09 16:44:17,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +21: [2023-05-09 16:44:17,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +21: [2023-05-09 16:44:17,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +21: [2023-05-09 16:44:17,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +14: [2023-05-09 16:44:17,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +21: [2023-05-09 16:44:17,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 9: [2023-05-09 16:44:17,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 9: [2023-05-09 16:44:17,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +11: [2023-05-09 16:44:17,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +11: [2023-05-09 16:44:17,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +11: [2023-05-09 16:44:17,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +11: [2023-05-09 16:44:17,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +14: [2023-05-09 16:44:17,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 9: [2023-05-09 16:44:17,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 9: [2023-05-09 16:44:17,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +21: [2023-05-09 16:44:17,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +11: [2023-05-09 16:44:17,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +11: [2023-05-09 16:44:17,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +21: [2023-05-09 16:44:17,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +11: [2023-05-09 16:44:17,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +11: [2023-05-09 16:44:17,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +21: [2023-05-09 16:44:17,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +21: [2023-05-09 16:44:17,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 9: [2023-05-09 16:44:17,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +14: [2023-05-09 16:44:17,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +14: [2023-05-09 16:44:17,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 1: [2023-05-09 16:44:17,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +14: [2023-05-09 16:44:17,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +14: [2023-05-09 16:44:17,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +14: [2023-05-09 16:44:17,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +14: [2023-05-09 16:44:17,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 9: [2023-05-09 16:44:17,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 1: [2023-05-09 16:44:17,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 9: [2023-05-09 16:44:17,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +11: [2023-05-09 16:44:17,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +11: [2023-05-09 16:44:17,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +21: [2023-05-09 16:44:17,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +21: [2023-05-09 16:44:17,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 9: [2023-05-09 16:44:17,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +11: [2023-05-09 16:44:17,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +11: [2023-05-09 16:44:17,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +21: [2023-05-09 16:44:17,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +21: [2023-05-09 16:44:17,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +14: [2023-05-09 16:44:17,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +14: [2023-05-09 16:44:17,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +11: [2023-05-09 16:44:17,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +21: [2023-05-09 16:44:17,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +21: [2023-05-09 16:44:17,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +11: [2023-05-09 16:44:17,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 1: [2023-05-09 16:44:17,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 1: [2023-05-09 16:44:17,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +11: [2023-05-09 16:44:17,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +11: [2023-05-09 16:44:17,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +21: [2023-05-09 16:44:17,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +21: [2023-05-09 16:44:17,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +18: [2023-05-09 16:44:17,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +18: [2023-05-09 16:44:17,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +18: [2023-05-09 16:44:17,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +18: [2023-05-09 16:44:17,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +18: [2023-05-09 16:44:17,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +18: [2023-05-09 16:44:17,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +18: [2023-05-09 16:44:17,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +18: [2023-05-09 16:44:17,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +18: [2023-05-09 16:44:17,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 2: [2023-05-09 16:44:17,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 2: [2023-05-09 16:44:17,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 2: [2023-05-09 16:44:17,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +18: [2023-05-09 16:44:17,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +18: [2023-05-09 16:44:17,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +18: [2023-05-09 16:44:17,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +24: [2023-05-09 16:44:17,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +24: [2023-05-09 16:44:17,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +24: [2023-05-09 16:44:17,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +24: [2023-05-09 16:44:17,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 2: [2023-05-09 16:44:17,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +18: [2023-05-09 16:44:17,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +18: [2023-05-09 16:44:17,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +18: [2023-05-09 16:44:17,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +18: [2023-05-09 16:44:17,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +12: [2023-05-09 16:44:17,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +12: [2023-05-09 16:44:17,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +12: [2023-05-09 16:44:17,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +12: [2023-05-09 16:44:17,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +17: [2023-05-09 16:44:17,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +17: [2023-05-09 16:44:17,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +17: [2023-05-09 16:44:17,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +17: [2023-05-09 16:44:17,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +28: [2023-05-09 16:44:17,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +28: [2023-05-09 16:44:17,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +28: [2023-05-09 16:44:17,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +28: [2023-05-09 16:44:17,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +31: [2023-05-09 16:44:17,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +31: [2023-05-09 16:44:17,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +31: [2023-05-09 16:44:17,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +31: [2023-05-09 16:44:17,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +24: [2023-05-09 16:44:17,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +20: [2023-05-09 16:44:17,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +20: [2023-05-09 16:44:17,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +20: [2023-05-09 16:44:17,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +20: [2023-05-09 16:44:17,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +10: [2023-05-09 16:44:17,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 8: [2023-05-09 16:44:17,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 2: [2023-05-09 16:44:17,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 8: [2023-05-09 16:44:17,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +10: [2023-05-09 16:44:17,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +10: [2023-05-09 16:44:17,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +10: [2023-05-09 16:44:17,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 2: [2023-05-09 16:44:17,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 8: [2023-05-09 16:44:17,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +24: [2023-05-09 16:44:17,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +26: [2023-05-09 16:44:17,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 2: [2023-05-09 16:44:17,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +26: [2023-05-09 16:44:17,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +26: [2023-05-09 16:44:17,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +26: [2023-05-09 16:44:17,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +12: [2023-05-09 16:44:17,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +24: [2023-05-09 16:44:17,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +24: [2023-05-09 16:44:17,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +12: [2023-05-09 16:44:17,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +13: [2023-05-09 16:44:17,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +13: [2023-05-09 16:44:17,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +13: [2023-05-09 16:44:17,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +13: [2023-05-09 16:44:17,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +22: [2023-05-09 16:44:17,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +17: [2023-05-09 16:44:17,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +22: [2023-05-09 16:44:17,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +22: [2023-05-09 16:44:17,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +28: [2023-05-09 16:44:17,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +22: [2023-05-09 16:44:17,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +17: [2023-05-09 16:44:17,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +28: [2023-05-09 16:44:17,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +28: [2023-05-09 16:44:17,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +31: [2023-05-09 16:44:17,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +31: [2023-05-09 16:44:17,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +12: [2023-05-09 16:44:17,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +12: [2023-05-09 16:44:17,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +17: [2023-05-09 16:44:17,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +17: [2023-05-09 16:44:17,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +31: [2023-05-09 16:44:17,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +20: [2023-05-09 16:44:17,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +20: [2023-05-09 16:44:17,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +28: [2023-05-09 16:44:17,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +31: [2023-05-09 16:44:17,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +10: [2023-05-09 16:44:17,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +24: [2023-05-09 16:44:17,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +10: [2023-05-09 16:44:17,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +20: [2023-05-09 16:44:17,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +20: [2023-05-09 16:44:17,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +24: [2023-05-09 16:44:17,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +26: [2023-05-09 16:44:17,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +26: [2023-05-09 16:44:17,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +10: [2023-05-09 16:44:17,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +26: [2023-05-09 16:44:17,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +13: [2023-05-09 16:44:17,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +13: [2023-05-09 16:44:17,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +26: [2023-05-09 16:44:17,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +13: [2023-05-09 16:44:17,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +10: [2023-05-09 16:44:17,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +22: [2023-05-09 16:44:17,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +22: [2023-05-09 16:44:17,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +13: [2023-05-09 16:44:17,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +22: [2023-05-09 16:44:17,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +22: [2023-05-09 16:44:17,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +28: [2023-05-09 16:44:17,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 2: [2023-05-09 16:44:17,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +28: [2023-05-09 16:44:17,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 2: [2023-05-09 16:44:17,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 2: [2023-05-09 16:44:17,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +12: [2023-05-09 16:44:17,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +31: [2023-05-09 16:44:17,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +12: [2023-05-09 16:44:17,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +31: [2023-05-09 16:44:17,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +24: [2023-05-09 16:44:17,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +24: [2023-05-09 16:44:17,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +12: [2023-05-09 16:44:17,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +12: [2023-05-09 16:44:17,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +24: [2023-05-09 16:44:17,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +24: [2023-05-09 16:44:17,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +24: [2023-05-09 16:44:17,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +24: [2023-05-09 16:44:17,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +10: [2023-05-09 16:44:17,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +20: [2023-05-09 16:44:17,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +17: [2023-05-09 16:44:17,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +20: [2023-05-09 16:44:17,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +10: [2023-05-09 16:44:17,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +12: [2023-05-09 16:44:17,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +12: [2023-05-09 16:44:17,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +20: [2023-05-09 16:44:17,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +20: [2023-05-09 16:44:17,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +31: [2023-05-09 16:44:17,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +31: [2023-05-09 16:44:17,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +28: [2023-05-09 16:44:17,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +28: [2023-05-09 16:44:17,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +28: [2023-05-09 16:44:17,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +28: [2023-05-09 16:44:17,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +17: [2023-05-09 16:44:17,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +12: [2023-05-09 16:44:17,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +13: [2023-05-09 16:44:17,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +13: [2023-05-09 16:44:17,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +20: [2023-05-09 16:44:17,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +20: [2023-05-09 16:44:17,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +12: [2023-05-09 16:44:17,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +17: [2023-05-09 16:44:17,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +29: [2023-05-09 16:44:17,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +29: [2023-05-09 16:44:17,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +29: [2023-05-09 16:44:17,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +29: [2023-05-09 16:44:17,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +22: [2023-05-09 16:44:17,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +22: [2023-05-09 16:44:17,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +17: [2023-05-09 16:44:17,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +26: [2023-05-09 16:44:17,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +20: [2023-05-09 16:44:17,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +20: [2023-05-09 16:44:17,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +26: [2023-05-09 16:44:17,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +26: [2023-05-09 16:44:17,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +26: [2023-05-09 16:44:17,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +13: [2023-05-09 16:44:17,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +13: [2023-05-09 16:44:17,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +13: [2023-05-09 16:44:17,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +13: [2023-05-09 16:44:17,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 8: [2023-05-09 16:44:17,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +10: [2023-05-09 16:44:17,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +10: [2023-05-09 16:44:17,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 8: [2023-05-09 16:44:17,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 8: [2023-05-09 16:44:17,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +31: [2023-05-09 16:44:17,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +31: [2023-05-09 16:44:17,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +17: [2023-05-09 16:44:17,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +31: [2023-05-09 16:44:17,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 7: [2023-05-09 16:44:17,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 7: [2023-05-09 16:44:17,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 7: [2023-05-09 16:44:17,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 7: [2023-05-09 16:44:17,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +31: [2023-05-09 16:44:17,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +22: [2023-05-09 16:44:17,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +22: [2023-05-09 16:44:17,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +26: [2023-05-09 16:44:17,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +26: [2023-05-09 16:44:17,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +17: [2023-05-09 16:44:17,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +26: [2023-05-09 16:44:17,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +17: [2023-05-09 16:44:17,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +29: [2023-05-09 16:44:17,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +29: [2023-05-09 16:44:17,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 3: [2023-05-09 16:44:17,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 3: [2023-05-09 16:44:17,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 3: [2023-05-09 16:44:17,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 3: [2023-05-09 16:44:17,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +29: [2023-05-09 16:44:17,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +29: [2023-05-09 16:44:17,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +13: [2023-05-09 16:44:17,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +26: [2023-05-09 16:44:17,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +17: [2023-05-09 16:44:17,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 5: [2023-05-09 16:44:17,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +28: [2023-05-09 16:44:17,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +13: [2023-05-09 16:44:17,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 5: [2023-05-09 16:44:17,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 5: [2023-05-09 16:44:17,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +22: [2023-05-09 16:44:17,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +28: [2023-05-09 16:44:17,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +15: [2023-05-09 16:44:17,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +23: [2023-05-09 16:44:17,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +23: [2023-05-09 16:44:17,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +23: [2023-05-09 16:44:17,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +15: [2023-05-09 16:44:17,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +23: [2023-05-09 16:44:17,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 8: [2023-05-09 16:44:17,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +22: [2023-05-09 16:44:17,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +15: [2023-05-09 16:44:17,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 8: [2023-05-09 16:44:17,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +15: [2023-05-09 16:44:17,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +22: [2023-05-09 16:44:17,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +10: [2023-05-09 16:44:17,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +10: [2023-05-09 16:44:17,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +22: [2023-05-09 16:44:17,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 7: [2023-05-09 16:44:17,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +10: [2023-05-09 16:44:17,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 0: [2023-05-09 16:44:17,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 0: [2023-05-09 16:44:17,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 7: [2023-05-09 16:44:17,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 3: [2023-05-09 16:44:17,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 3: [2023-05-09 16:44:17,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 3: [2023-05-09 16:44:17,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +29: [2023-05-09 16:44:17,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +10: [2023-05-09 16:44:17,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 5: [2023-05-09 16:44:17,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +29: [2023-05-09 16:44:17,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 5: [2023-05-09 16:44:17,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +29: [2023-05-09 16:44:17,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +29: [2023-05-09 16:44:17,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 5: [2023-05-09 16:44:17,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +23: [2023-05-09 16:44:17,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +29: [2023-05-09 16:44:17,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +29: [2023-05-09 16:44:17,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +15: [2023-05-09 16:44:17,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +15: [2023-05-09 16:44:17,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +15: [2023-05-09 16:44:17,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +15: [2023-05-09 16:44:17,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +23: [2023-05-09 16:44:17,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +23: [2023-05-09 16:44:17,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +23: [2023-05-09 16:44:17,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 6: [2023-05-09 16:44:17,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 6: [2023-05-09 16:44:17,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 6: [2023-05-09 16:44:17,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 6: [2023-05-09 16:44:17,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 0: [2023-05-09 16:44:17,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +29: [2023-05-09 16:44:17,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +29: [2023-05-09 16:44:17,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 0: [2023-05-09 16:44:17,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 0: [2023-05-09 16:44:17,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 4: [2023-05-09 16:44:17,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 4: [2023-05-09 16:44:17,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 4: [2023-05-09 16:44:17,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 4: [2023-05-09 16:44:17,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 7: [2023-05-09 16:44:17,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 6: [2023-05-09 16:44:17,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 3: [2023-05-09 16:44:17,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +23: [2023-05-09 16:44:17,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 6: [2023-05-09 16:44:17,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +23: [2023-05-09 16:44:17,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 6: [2023-05-09 16:44:17,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 6: [2023-05-09 16:44:17,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +19: [2023-05-09 16:44:17,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +19: [2023-05-09 16:44:17,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +19: [2023-05-09 16:44:17,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +19: [2023-05-09 16:44:17,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 5: [2023-05-09 16:44:17,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +16: [2023-05-09 16:44:17,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +16: [2023-05-09 16:44:17,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +15: [2023-05-09 16:44:17,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +15: [2023-05-09 16:44:17,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +16: [2023-05-09 16:44:17,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +16: [2023-05-09 16:44:17,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +15: [2023-05-09 16:44:17,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +15: [2023-05-09 16:44:17,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 5: [2023-05-09 16:44:17,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 7: [2023-05-09 16:44:17,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +30: [2023-05-09 16:44:17,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +30: [2023-05-09 16:44:17,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +30: [2023-05-09 16:44:17,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +30: [2023-05-09 16:44:17,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 4: [2023-05-09 16:44:17,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +15: [2023-05-09 16:44:17,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +15: [2023-05-09 16:44:17,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 7: [2023-05-09 16:44:17,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 3: [2023-05-09 16:44:17,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +23: [2023-05-09 16:44:17,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 4: [2023-05-09 16:44:17,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +23: [2023-05-09 16:44:17,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +23: [2023-05-09 16:44:17,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 3: [2023-05-09 16:44:17,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 4: [2023-05-09 16:44:17,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 3: [2023-05-09 16:44:17,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +23: [2023-05-09 16:44:17,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +23: [2023-05-09 16:44:17,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +23: [2023-05-09 16:44:17,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +19: [2023-05-09 16:44:17,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +19: [2023-05-09 16:44:17,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +19: [2023-05-09 16:44:17,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +19: [2023-05-09 16:44:17,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 6: [2023-05-09 16:44:17,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +15: [2023-05-09 16:44:17,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 6: [2023-05-09 16:44:17,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 5: [2023-05-09 16:44:17,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +16: [2023-05-09 16:44:17,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +15: [2023-05-09 16:44:17,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 7: [2023-05-09 16:44:17,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +30: [2023-05-09 16:44:17,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 6: [2023-05-09 16:44:17,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 6: [2023-05-09 16:44:17,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +30: [2023-05-09 16:44:17,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +30: [2023-05-09 16:44:17,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +30: [2023-05-09 16:44:17,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +16: [2023-05-09 16:44:17,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +16: [2023-05-09 16:44:17,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +16: [2023-05-09 16:44:17,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +25: [2023-05-09 16:44:17,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +25: [2023-05-09 16:44:17,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +25: [2023-05-09 16:44:17,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. +25: [2023-05-09 16:44:17,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt. + 6: [2023-05-09 16:44:17,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 6: [2023-05-09 16:44:17,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 6: [2023-05-09 16:44:17,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 6: [2023-05-09 16:44:17,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 0: [2023-05-09 16:44:17,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +19: [2023-05-09 16:44:17,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +19: [2023-05-09 16:44:17,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +19: [2023-05-09 16:44:17,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +19: [2023-05-09 16:44:17,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 0: [2023-05-09 16:44:17,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +30: [2023-05-09 16:44:17,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 0: [2023-05-09 16:44:17,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +30: [2023-05-09 16:44:17,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +25: [2023-05-09 16:44:17,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +19: [2023-05-09 16:44:17,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +19: [2023-05-09 16:44:17,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +25: [2023-05-09 16:44:17,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +19: [2023-05-09 16:44:17,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +19: [2023-05-09 16:44:17,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +27: [2023-05-09 16:44:17,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +27: [2023-05-09 16:44:17,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +27: [2023-05-09 16:44:17,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +27: [2023-05-09 16:44:17,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +25: [2023-05-09 16:44:17,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... +25: [2023-05-09 16:44:17,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt... + 4: [2023-05-09 16:44:17,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +30: [2023-05-09 16:44:17,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +30: [2023-05-09 16:44:17,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 4: [2023-05-09 16:44:17,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. + 4: [2023-05-09 16:44:17,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... + 4: [2023-05-09 16:44:17,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +27: [2023-05-09 16:44:17,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +27: [2023-05-09 16:44:17,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +25: [2023-05-09 16:44:17,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +30: [2023-05-09 16:44:17,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +27: [2023-05-09 16:44:17,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +25: [2023-05-09 16:44:17,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +30: [2023-05-09 16:44:17,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +30: [2023-05-09 16:44:17,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +30: [2023-05-09 16:44:17,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +27: [2023-05-09 16:44:17,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +25: [2023-05-09 16:44:17,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +25: [2023-05-09 16:44:17,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +25: [2023-05-09 16:44:17,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +25: [2023-05-09 16:44:17,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt. +25: [2023-05-09 16:44:17,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +25: [2023-05-09 16:44:17,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt... +14: [2023-05-09 16:44:17,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +14: [2023-05-09 16:44:17,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +14: [2023-05-09 16:44:17,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +14: [2023-05-09 16:44:17,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 9: [2023-05-09 16:44:17,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 9: [2023-05-09 16:44:17,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 9: [2023-05-09 16:44:17,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 9: [2023-05-09 16:44:17,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +14: [2023-05-09 16:44:17,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +14: [2023-05-09 16:44:17,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +14: [2023-05-09 16:44:17,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +14: [2023-05-09 16:44:17,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 9: [2023-05-09 16:44:17,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 9: [2023-05-09 16:44:17,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 9: [2023-05-09 16:44:17,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 9: [2023-05-09 16:44:17,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +25: [2023-05-09 16:44:17,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +25: [2023-05-09 16:44:17,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +25: [2023-05-09 16:44:17,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +25: [2023-05-09 16:44:17,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +25: [2023-05-09 16:44:17,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +25: [2023-05-09 16:44:17,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +25: [2023-05-09 16:44:17,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +25: [2023-05-09 16:44:17,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +21: [2023-05-09 16:44:17,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +21: [2023-05-09 16:44:17,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +21: [2023-05-09 16:44:17,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +21: [2023-05-09 16:44:17,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +11: [2023-05-09 16:44:17,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +11: [2023-05-09 16:44:17,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +11: [2023-05-09 16:44:17,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +11: [2023-05-09 16:44:17,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +25: [2023-05-09 16:44:17,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +21: [2023-05-09 16:44:17,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +21: [2023-05-09 16:44:17,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +21: [2023-05-09 16:44:17,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +21: [2023-05-09 16:44:17,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +11: [2023-05-09 16:44:17,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +11: [2023-05-09 16:44:17,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +11: [2023-05-09 16:44:17,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +11: [2023-05-09 16:44:17,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +25: [2023-05-09 16:44:17,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +25: [2023-05-09 16:44:17,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +25: [2023-05-09 16:44:17,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +25: [2023-05-09 16:44:17,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +25: [2023-05-09 16:44:17,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +25: [2023-05-09 16:44:17,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +25: [2023-05-09 16:44:17,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 2: [2023-05-09 16:44:17,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 2: [2023-05-09 16:44:17,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 2: [2023-05-09 16:44:17,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +18: [2023-05-09 16:44:17,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +18: [2023-05-09 16:44:17,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +18: [2023-05-09 16:44:17,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +18: [2023-05-09 16:44:17,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 2: [2023-05-09 16:44:17,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 2: [2023-05-09 16:44:17,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +18: [2023-05-09 16:44:17,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +18: [2023-05-09 16:44:17,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +18: [2023-05-09 16:44:17,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +18: [2023-05-09 16:44:17,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +27: [2023-05-09 16:44:17,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +27: [2023-05-09 16:44:17,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +27: [2023-05-09 16:44:17,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +27: [2023-05-09 16:44:17,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +27: [2023-05-09 16:44:17,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +27: [2023-05-09 16:44:17,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +27: [2023-05-09 16:44:17,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +27: [2023-05-09 16:44:17,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +27: [2023-05-09 16:44:17,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +27: [2023-05-09 16:44:17,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +27: [2023-05-09 16:44:17,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +27: [2023-05-09 16:44:17,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +27: [2023-05-09 16:44:17,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +27: [2023-05-09 16:44:17,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +27: [2023-05-09 16:44:17,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +27: [2023-05-09 16:44:17,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 1: [2023-05-09 16:44:17,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 1: [2023-05-09 16:44:17,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 1: [2023-05-09 16:44:17,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 7: [2023-05-09 16:44:17,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 7: [2023-05-09 16:44:17,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +28: [2023-05-09 16:44:17,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +28: [2023-05-09 16:44:17,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +28: [2023-05-09 16:44:17,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +28: [2023-05-09 16:44:17,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +28: [2023-05-09 16:44:17,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +28: [2023-05-09 16:44:17,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +28: [2023-05-09 16:44:17,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +28: [2023-05-09 16:44:17,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +18: [2023-05-09 16:44:17,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +18: [2023-05-09 16:44:17,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +18: [2023-05-09 16:44:17,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +18: [2023-05-09 16:44:17,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +18: [2023-05-09 16:44:17,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +18: [2023-05-09 16:44:17,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +18: [2023-05-09 16:44:17,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +18: [2023-05-09 16:44:17,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +18: [2023-05-09 16:44:17,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +18: [2023-05-09 16:44:17,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +18: [2023-05-09 16:44:17,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +18: [2023-05-09 16:44:17,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +18: [2023-05-09 16:44:17,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +18: [2023-05-09 16:44:17,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +18: [2023-05-09 16:44:17,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +18: [2023-05-09 16:44:17,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 1: [2023-05-09 16:44:17,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +27: [2023-05-09 16:44:17,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 1: [2023-05-09 16:44:17,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 1: [2023-05-09 16:44:17,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +27: [2023-05-09 16:44:17,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +10: [2023-05-09 16:44:17,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +10: [2023-05-09 16:44:17,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +10: [2023-05-09 16:44:17,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +10: [2023-05-09 16:44:17,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +28: [2023-05-09 16:44:17,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +27: [2023-05-09 16:44:17,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +10: [2023-05-09 16:44:17,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +10: [2023-05-09 16:44:17,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +28: [2023-05-09 16:44:17,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +28: [2023-05-09 16:44:17,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +10: [2023-05-09 16:44:17,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +10: [2023-05-09 16:44:17,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +28: [2023-05-09 16:44:17,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 5: [2023-05-09 16:44:17,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +27: [2023-05-09 16:44:17,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +18: [2023-05-09 16:44:17,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +18: [2023-05-09 16:44:17,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +27: [2023-05-09 16:44:17,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +28: [2023-05-09 16:44:17,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +28: [2023-05-09 16:44:17,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +18: [2023-05-09 16:44:17,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +28: [2023-05-09 16:44:17,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +28: [2023-05-09 16:44:17,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +28: [2023-05-09 16:44:17,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +10: [2023-05-09 16:44:17,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +29: [2023-05-09 16:44:17,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +27: [2023-05-09 16:44:17,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +18: [2023-05-09 16:44:17,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +29: [2023-05-09 16:44:17,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +29: [2023-05-09 16:44:17,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +29: [2023-05-09 16:44:17,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 5: [2023-05-09 16:44:17,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +27: [2023-05-09 16:44:17,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +27: [2023-05-09 16:44:17,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +28: [2023-05-09 16:44:17,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +28: [2023-05-09 16:44:17,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +28: [2023-05-09 16:44:17,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +18: [2023-05-09 16:44:17,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 5: [2023-05-09 16:44:17,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +18: [2023-05-09 16:44:17,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +10: [2023-05-09 16:44:17,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 8: [2023-05-09 16:44:17,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 7: [2023-05-09 16:44:17,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 7: [2023-05-09 16:44:17,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +28: [2023-05-09 16:44:17,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 8: [2023-05-09 16:44:17,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +18: [2023-05-09 16:44:17,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +18: [2023-05-09 16:44:17,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +10: [2023-05-09 16:44:17,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +28: [2023-05-09 16:44:17,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +28: [2023-05-09 16:44:17,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +10: [2023-05-09 16:44:17,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +10: [2023-05-09 16:44:17,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 5: [2023-05-09 16:44:17,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +28: [2023-05-09 16:44:17,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +29: [2023-05-09 16:44:17,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +29: [2023-05-09 16:44:17,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +29: [2023-05-09 16:44:17,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +28: [2023-05-09 16:44:18,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +28: [2023-05-09 16:44:18,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +29: [2023-05-09 16:44:18,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +10: [2023-05-09 16:44:18,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +10: [2023-05-09 16:44:18,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +24: [2023-05-09 16:44:18,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +24: [2023-05-09 16:44:18,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +24: [2023-05-09 16:44:18,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +24: [2023-05-09 16:44:18,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +10: [2023-05-09 16:44:18,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +28: [2023-05-09 16:44:18,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +28: [2023-05-09 16:44:18,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +28: [2023-05-09 16:44:18,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +28: [2023-05-09 16:44:18,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +28: [2023-05-09 16:44:18,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +28: [2023-05-09 16:44:18,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 8: [2023-05-09 16:44:18,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +26: [2023-05-09 16:44:18,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +26: [2023-05-09 16:44:18,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +26: [2023-05-09 16:44:18,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +26: [2023-05-09 16:44:18,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +26: [2023-05-09 16:44:18,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +26: [2023-05-09 16:44:18,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +26: [2023-05-09 16:44:18,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +26: [2023-05-09 16:44:18,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +24: [2023-05-09 16:44:18,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +29: [2023-05-09 16:44:18,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +29: [2023-05-09 16:44:18,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +29: [2023-05-09 16:44:18,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +29: [2023-05-09 16:44:18,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +29: [2023-05-09 16:44:18,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +29: [2023-05-09 16:44:18,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +29: [2023-05-09 16:44:18,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 9: [2023-05-09 16:44:18,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +29: [2023-05-09 16:44:18,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +29: [2023-05-09 16:44:18,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +29: [2023-05-09 16:44:18,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +29: [2023-05-09 16:44:18,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 9: [2023-05-09 16:44:18,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 9: [2023-05-09 16:44:18,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +24: [2023-05-09 16:44:18,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +24: [2023-05-09 16:44:18,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +24: [2023-05-09 16:44:18,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +29: [2023-05-09 16:44:18,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +29: [2023-05-09 16:44:18,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +29: [2023-05-09 16:44:18,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +29: [2023-05-09 16:44:18,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +26: [2023-05-09 16:44:18,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +29: [2023-05-09 16:44:18,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +26: [2023-05-09 16:44:18,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +26: [2023-05-09 16:44:18,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +31: [2023-05-09 16:44:18,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +31: [2023-05-09 16:44:18,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +31: [2023-05-09 16:44:18,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 8: [2023-05-09 16:44:18,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +31: [2023-05-09 16:44:18,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +29: [2023-05-09 16:44:18,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +26: [2023-05-09 16:44:18,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +26: [2023-05-09 16:44:18,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +29: [2023-05-09 16:44:18,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 8: [2023-05-09 16:44:18,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 8: [2023-05-09 16:44:18,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 8: [2023-05-09 16:44:18,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +26: [2023-05-09 16:44:18,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +23: [2023-05-09 16:44:18,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +23: [2023-05-09 16:44:18,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 9: [2023-05-09 16:44:18,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +26: [2023-05-09 16:44:18,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +29: [2023-05-09 16:44:18,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +23: [2023-05-09 16:44:18,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +29: [2023-05-09 16:44:18,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +31: [2023-05-09 16:44:18,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +23: [2023-05-09 16:44:18,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +31: [2023-05-09 16:44:18,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +31: [2023-05-09 16:44:18,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +31: [2023-05-09 16:44:18,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +29: [2023-05-09 16:44:18,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +26: [2023-05-09 16:44:18,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +29: [2023-05-09 16:44:18,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +23: [2023-05-09 16:44:18,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +12: [2023-05-09 16:44:18,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 6: [2023-05-09 16:44:18,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +12: [2023-05-09 16:44:18,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +29: [2023-05-09 16:44:18,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +12: [2023-05-09 16:44:18,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +12: [2023-05-09 16:44:18,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +23: [2023-05-09 16:44:18,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +23: [2023-05-09 16:44:18,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 1: [2023-05-09 16:44:18,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 9: [2023-05-09 16:44:18,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +23: [2023-05-09 16:44:18,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 9: [2023-05-09 16:44:18,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +29: [2023-05-09 16:44:18,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 6: [2023-05-09 16:44:18,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 3: [2023-05-09 16:44:18,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 3: [2023-05-09 16:44:18,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +31: [2023-05-09 16:44:18,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +31: [2023-05-09 16:44:18,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +19: [2023-05-09 16:44:18,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +31: [2023-05-09 16:44:18,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +17: [2023-05-09 16:44:18,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +17: [2023-05-09 16:44:18,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +17: [2023-05-09 16:44:18,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +17: [2023-05-09 16:44:18,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +17: [2023-05-09 16:44:18,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +19: [2023-05-09 16:44:18,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +31: [2023-05-09 16:44:18,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +19: [2023-05-09 16:44:18,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +31: [2023-05-09 16:44:18,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +17: [2023-05-09 16:44:18,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +19: [2023-05-09 16:44:18,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +19: [2023-05-09 16:44:18,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +19: [2023-05-09 16:44:18,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +24: [2023-05-09 16:44:18,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +23: [2023-05-09 16:44:18,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +23: [2023-05-09 16:44:18,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +23: [2023-05-09 16:44:18,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +23: [2023-05-09 16:44:18,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +23: [2023-05-09 16:44:18,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +23: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +24: [2023-05-09 16:44:18,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +24: [2023-05-09 16:44:18,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +24: [2023-05-09 16:44:18,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +24: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +24: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +24: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +24: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +23: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +24: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +24: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +23: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +31: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +17: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +24: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +24: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +24: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +17: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +24: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +21: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +21: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +12: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +24: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +21: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +12: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +12: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +19: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +21: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +12: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +31: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +21: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +21: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +21: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +21: [2023-05-09 16:44:18,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +21: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +31: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +21: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +21: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +21: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +21: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +16: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +21: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +11: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +16: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +11: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +16: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +16: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +16: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +16: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +16: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +16: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +16: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 2: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +16: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +16: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +16: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +16: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +19: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +16: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +16: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +31: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +31: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +16: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +31: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +11: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +11: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +11: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +11: [2023-05-09 16:44:18,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +11: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +11: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +11: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +11: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +11: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +11: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +11: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +11: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +31: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +31: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +31: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 2: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 2: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 2: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +21: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +21: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +12: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +12: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +12: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +12: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +12: [2023-05-09 16:44:18,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +24: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +12: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +12: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +12: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +22: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +22: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +22: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +22: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +13: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +13: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +13: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +13: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +31: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +13: [2023-05-09 16:44:18,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +13: [2023-05-09 16:44:18,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +13: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +13: [2023-05-09 16:44:18,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +23: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +23: [2023-05-09 16:44:18,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +11: [2023-05-09 16:44:18,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +22: [2023-05-09 16:44:18,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +11: [2023-05-09 16:44:18,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +22: [2023-05-09 16:44:18,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +23: [2023-05-09 16:44:18,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +22: [2023-05-09 16:44:18,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +22: [2023-05-09 16:44:18,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +23: [2023-05-09 16:44:18,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +23: [2023-05-09 16:44:18,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +23: [2023-05-09 16:44:18,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +31: [2023-05-09 16:44:18,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 8: [2023-05-09 16:44:18,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +14: [2023-05-09 16:44:18,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +14: [2023-05-09 16:44:18,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +14: [2023-05-09 16:44:18,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +14: [2023-05-09 16:44:18,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +14: [2023-05-09 16:44:18,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +14: [2023-05-09 16:44:18,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +14: [2023-05-09 16:44:18,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +14: [2023-05-09 16:44:18,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +14: [2023-05-09 16:44:18,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +14: [2023-05-09 16:44:18,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +14: [2023-05-09 16:44:18,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +14: [2023-05-09 16:44:18,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +14: [2023-05-09 16:44:18,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +14: [2023-05-09 16:44:18,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +14: [2023-05-09 16:44:18,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +14: [2023-05-09 16:44:18,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 3: [2023-05-09 16:44:18,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +15: [2023-05-09 16:44:18,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +15: [2023-05-09 16:44:18,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +15: [2023-05-09 16:44:18,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +20: [2023-05-09 16:44:18,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +15: [2023-05-09 16:44:18,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +20: [2023-05-09 16:44:18,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +20: [2023-05-09 16:44:18,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +20: [2023-05-09 16:44:18,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +20: [2023-05-09 16:44:18,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +15: [2023-05-09 16:44:18,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +15: [2023-05-09 16:44:18,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +15: [2023-05-09 16:44:18,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 1: [2023-05-09 16:44:18,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +20: [2023-05-09 16:44:18,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +15: [2023-05-09 16:44:18,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +20: [2023-05-09 16:44:18,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +20: [2023-05-09 16:44:18,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +30: [2023-05-09 16:44:18,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +30: [2023-05-09 16:44:18,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +30: [2023-05-09 16:44:18,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +30: [2023-05-09 16:44:18,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +30: [2023-05-09 16:44:18,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +30: [2023-05-09 16:44:18,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +30: [2023-05-09 16:44:18,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +30: [2023-05-09 16:44:18,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +17: [2023-05-09 16:44:18,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +24: [2023-05-09 16:44:18,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +21: [2023-05-09 16:44:18,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +17: [2023-05-09 16:44:18,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +31: [2023-05-09 16:44:18,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +31: [2023-05-09 16:44:18,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +16: [2023-05-09 16:44:18,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +16: [2023-05-09 16:44:18,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +23: [2023-05-09 16:44:18,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +23: [2023-05-09 16:44:18,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +19: [2023-05-09 16:44:18,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +19: [2023-05-09 16:44:18,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +13: [2023-05-09 16:44:18,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +22: [2023-05-09 16:44:18,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +31: [2023-05-09 16:44:18,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +17: [2023-05-09 16:44:18,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +14: [2023-05-09 16:44:18,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +23: [2023-05-09 16:44:18,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +23: [2023-05-09 16:44:18,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +12: [2023-05-09 16:44:18,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +16: [2023-05-09 16:44:18,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +12: [2023-05-09 16:44:18,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +12: [2023-05-09 16:44:18,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +22: [2023-05-09 16:44:18,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +16: [2023-05-09 16:44:18,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +14: [2023-05-09 16:44:18,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 4: [2023-05-09 16:44:18,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 4: [2023-05-09 16:44:18,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 4: [2023-05-09 16:44:18,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 1: [2023-05-09 16:44:18,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +15: [2023-05-09 16:44:18,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +23: [2023-05-09 16:44:18,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +21: [2023-05-09 16:44:18,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +14: [2023-05-09 16:44:18,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +24: [2023-05-09 16:44:18,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +24: [2023-05-09 16:44:18,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +17: [2023-05-09 16:44:18,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +17: [2023-05-09 16:44:18,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +12: [2023-05-09 16:44:18,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +12: [2023-05-09 16:44:18,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +15: [2023-05-09 16:44:18,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +12: [2023-05-09 16:44:18,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +12: [2023-05-09 16:44:18,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +23: [2023-05-09 16:44:18,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 2: [2023-05-09 16:44:18,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +31: [2023-05-09 16:44:18,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +20: [2023-05-09 16:44:18,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +14: [2023-05-09 16:44:18,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +31: [2023-05-09 16:44:18,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +11: [2023-05-09 16:44:18,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +11: [2023-05-09 16:44:18,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +22: [2023-05-09 16:44:18,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +21: [2023-05-09 16:44:18,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +15: [2023-05-09 16:44:18,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +22: [2023-05-09 16:44:18,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +30: [2023-05-09 16:44:18,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +20: [2023-05-09 16:44:18,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +30: [2023-05-09 16:44:18,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +16: [2023-05-09 16:44:18,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +17: [2023-05-09 16:44:18,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +20: [2023-05-09 16:44:18,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +24: [2023-05-09 16:44:18,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +13: [2023-05-09 16:44:18,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +13: [2023-05-09 16:44:18,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +22: [2023-05-09 16:44:18,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +31: [2023-05-09 16:44:18,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +19: [2023-05-09 16:44:18,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +19: [2023-05-09 16:44:18,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +13: [2023-05-09 16:44:18,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +19: [2023-05-09 16:44:18,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +31: [2023-05-09 16:44:18,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +23: [2023-05-09 16:44:18,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +13: [2023-05-09 16:44:18,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +17: [2023-05-09 16:44:18,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +30: [2023-05-09 16:44:18,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +30: [2023-05-09 16:44:18,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +14: [2023-05-09 16:44:18,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +23: [2023-05-09 16:44:18,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +16: [2023-05-09 16:44:18,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +12: [2023-05-09 16:44:18,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +16: [2023-05-09 16:44:18,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +16: [2023-05-09 16:44:18,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +12: [2023-05-09 16:44:18,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +12: [2023-05-09 16:44:18,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +12: [2023-05-09 16:44:18,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +14: [2023-05-09 16:44:18,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +23: [2023-05-09 16:44:18,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +23: [2023-05-09 16:44:18,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 3: [2023-05-09 16:44:18,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +15: [2023-05-09 16:44:18,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +21: [2023-05-09 16:44:18,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +22: [2023-05-09 16:44:18,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +17: [2023-05-09 16:44:18,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +30: [2023-05-09 16:44:18,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +14: [2023-05-09 16:44:18,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +15: [2023-05-09 16:44:18,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +15: [2023-05-09 16:44:18,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +22: [2023-05-09 16:44:18,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +11: [2023-05-09 16:44:18,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +11: [2023-05-09 16:44:18,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +20: [2023-05-09 16:44:18,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +24: [2023-05-09 16:44:18,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +20: [2023-05-09 16:44:18,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +31: [2023-05-09 16:44:18,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +14: [2023-05-09 16:44:18,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +20: [2023-05-09 16:44:18,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +13: [2023-05-09 16:44:18,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +24: [2023-05-09 16:44:18,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +30: [2023-05-09 16:44:18,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +12: [2023-05-09 16:44:18,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +19: [2023-05-09 16:44:18,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +12: [2023-05-09 16:44:18,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +20: [2023-05-09 16:44:18,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +21: [2023-05-09 16:44:18,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +21: [2023-05-09 16:44:18,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +22: [2023-05-09 16:44:18,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +30: [2023-05-09 16:44:18,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +13: [2023-05-09 16:44:18,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +13: [2023-05-09 16:44:18,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +15: [2023-05-09 16:44:18,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +30: [2023-05-09 16:44:18,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +11: [2023-05-09 16:44:18,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +11: [2023-05-09 16:44:18,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +15: [2023-05-09 16:44:18,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +15: [2023-05-09 16:44:18,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +15: [2023-05-09 16:44:18,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 7: [2023-05-09 16:44:18,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 7: [2023-05-09 16:44:18,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +19: [2023-05-09 16:44:18,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +15: [2023-05-09 16:44:18,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 7: [2023-05-09 16:44:18,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 3: [2023-05-09 16:44:18,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +15: [2023-05-09 16:44:18,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +12: [2023-05-09 16:44:18,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +20: [2023-05-09 16:44:18,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +12: [2023-05-09 16:44:18,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +24: [2023-05-09 16:44:18,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +12: [2023-05-09 16:44:18,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +21: [2023-05-09 16:44:18,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +21: [2023-05-09 16:44:18,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +11: [2023-05-09 16:44:18,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +11: [2023-05-09 16:44:18,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +19: [2023-05-09 16:44:18,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +15: [2023-05-09 16:44:18,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +15: [2023-05-09 16:44:18,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +15: [2023-05-09 16:44:18,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +15: [2023-05-09 16:44:18,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +24: [2023-05-09 16:44:18,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +26: [2023-05-09 16:44:18,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +26: [2023-05-09 16:44:18,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +26: [2023-05-09 16:44:18,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +26: [2023-05-09 16:44:18,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 0: [2023-05-09 16:44:18,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 4: [2023-05-09 16:44:18,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 0: [2023-05-09 16:44:18,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 0: [2023-05-09 16:44:18,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 0: [2023-05-09 16:44:18,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 4: [2023-05-09 16:44:18,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +20: [2023-05-09 16:44:18,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +20: [2023-05-09 16:44:18,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +20: [2023-05-09 16:44:18,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +20: [2023-05-09 16:44:18,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 6: [2023-05-09 16:44:18,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 6: [2023-05-09 16:44:18,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 6: [2023-05-09 16:44:18,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 6: [2023-05-09 16:44:18,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 5: [2023-05-09 16:44:18,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 5: [2023-05-09 16:44:18,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +10: [2023-05-09 16:44:18,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +10: [2023-05-09 16:44:18,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +17: [2023-05-09 16:44:18,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +17: [2023-05-09 16:44:18,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +17: [2023-05-09 16:44:18,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +17: [2023-05-09 16:44:18,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +10: [2023-05-09 16:44:18,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +10: [2023-05-09 16:44:18,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 7: [2023-05-09 16:44:18,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 5: [2023-05-09 16:44:18,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +26: [2023-05-09 16:44:18,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +30: [2023-05-09 16:44:18,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +26: [2023-05-09 16:44:18,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +30: [2023-05-09 16:44:18,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 7: [2023-05-09 16:44:18,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +26: [2023-05-09 16:44:18,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +26: [2023-05-09 16:44:18,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +30: [2023-05-09 16:44:18,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 4: [2023-05-09 16:44:18,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +30: [2023-05-09 16:44:18,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +15: [2023-05-09 16:44:18,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +15: [2023-05-09 16:44:18,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 4: [2023-05-09 16:44:18,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +15: [2023-05-09 16:44:18,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +15: [2023-05-09 16:44:18,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +13: [2023-05-09 16:44:18,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +13: [2023-05-09 16:44:18,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 0: [2023-05-09 16:44:18,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +13: [2023-05-09 16:44:18,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +13: [2023-05-09 16:44:18,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 4: [2023-05-09 16:44:18,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +22: [2023-05-09 16:44:18,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 0: [2023-05-09 16:44:18,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +22: [2023-05-09 16:44:18,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 7: [2023-05-09 16:44:18,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +20: [2023-05-09 16:44:18,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +20: [2023-05-09 16:44:18,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +20: [2023-05-09 16:44:18,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +20: [2023-05-09 16:44:18,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +15: [2023-05-09 16:44:18,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +15: [2023-05-09 16:44:18,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 7: [2023-05-09 16:44:18,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 6: [2023-05-09 16:44:18,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +22: [2023-05-09 16:44:18,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +19: [2023-05-09 16:44:18,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +19: [2023-05-09 16:44:18,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +19: [2023-05-09 16:44:18,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +19: [2023-05-09 16:44:18,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +22: [2023-05-09 16:44:18,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 6: [2023-05-09 16:44:18,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +17: [2023-05-09 16:44:18,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +17: [2023-05-09 16:44:18,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +17: [2023-05-09 16:44:18,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +10: [2023-05-09 16:44:18,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +17: [2023-05-09 16:44:18,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +10: [2023-05-09 16:44:18,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +10: [2023-05-09 16:44:18,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +25: [2023-05-09 16:44:18,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +25: [2023-05-09 16:44:18,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +25: [2023-05-09 16:44:18,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. +30: [2023-05-09 16:44:18,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +10: [2023-05-09 16:44:18,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +25: [2023-05-09 16:44:18,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt. + 5: [2023-05-09 16:44:18,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +13: [2023-05-09 16:44:18,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +13: [2023-05-09 16:44:18,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +15: [2023-05-09 16:44:18,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +30: [2023-05-09 16:44:18,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +30: [2023-05-09 16:44:18,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +13: [2023-05-09 16:44:18,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +13: [2023-05-09 16:44:18,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +30: [2023-05-09 16:44:18,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +15: [2023-05-09 16:44:18,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +22: [2023-05-09 16:44:18,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +26: [2023-05-09 16:44:18,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +26: [2023-05-09 16:44:18,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +22: [2023-05-09 16:44:18,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +22: [2023-05-09 16:44:18,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +19: [2023-05-09 16:44:18,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +22: [2023-05-09 16:44:18,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +19: [2023-05-09 16:44:18,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +19: [2023-05-09 16:44:18,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +19: [2023-05-09 16:44:18,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +25: [2023-05-09 16:44:18,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +25: [2023-05-09 16:44:18,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +25: [2023-05-09 16:44:18,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +26: [2023-05-09 16:44:18,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +25: [2023-05-09 16:44:18,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt... +26: [2023-05-09 16:44:18,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +20: [2023-05-09 16:44:18,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +20: [2023-05-09 16:44:18,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +20: [2023-05-09 16:44:18,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +20: [2023-05-09 16:44:18,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +20: [2023-05-09 16:44:18,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +20: [2023-05-09 16:44:18,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +20: [2023-05-09 16:44:18,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +20: [2023-05-09 16:44:18,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 6: [2023-05-09 16:44:18,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +30: [2023-05-09 16:44:18,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +10: [2023-05-09 16:44:18,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +17: [2023-05-09 16:44:18,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 6: [2023-05-09 16:44:18,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +30: [2023-05-09 16:44:18,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +17: [2023-05-09 16:44:18,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +10: [2023-05-09 16:44:18,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 5: [2023-05-09 16:44:18,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 5: [2023-05-09 16:44:18,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +26: [2023-05-09 16:44:18,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +26: [2023-05-09 16:44:18,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +26: [2023-05-09 16:44:18,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +26: [2023-05-09 16:44:18,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 5: [2023-05-09 16:44:18,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 0: [2023-05-09 16:44:18,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 0: [2023-05-09 16:44:18,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +30: [2023-05-09 16:44:18,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +30: [2023-05-09 16:44:18,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +13: [2023-05-09 16:44:18,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +13: [2023-05-09 16:44:18,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +13: [2023-05-09 16:44:18,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +13: [2023-05-09 16:44:18,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +13: [2023-05-09 16:44:18,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +13: [2023-05-09 16:44:18,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +13: [2023-05-09 16:44:18,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +13: [2023-05-09 16:44:18,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +17: [2023-05-09 16:44:18,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +17: [2023-05-09 16:44:18,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +30: [2023-05-09 16:44:18,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +30: [2023-05-09 16:44:18,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +17: [2023-05-09 16:44:18,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +17: [2023-05-09 16:44:18,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +17: [2023-05-09 16:44:18,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +17: [2023-05-09 16:44:18,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +25: [2023-05-09 16:44:18,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +22: [2023-05-09 16:44:18,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +22: [2023-05-09 16:44:18,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +22: [2023-05-09 16:44:18,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +22: [2023-05-09 16:44:18,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +22: [2023-05-09 16:44:18,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +22: [2023-05-09 16:44:18,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +19: [2023-05-09 16:44:18,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +30: [2023-05-09 16:44:18,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +30: [2023-05-09 16:44:18,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +19: [2023-05-09 16:44:18,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +10: [2023-05-09 16:44:18,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +10: [2023-05-09 16:44:18,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +10: [2023-05-09 16:44:18,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +10: [2023-05-09 16:44:18,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +19: [2023-05-09 16:44:18,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +19: [2023-05-09 16:44:18,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +19: [2023-05-09 16:44:18,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +19: [2023-05-09 16:44:18,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +25: [2023-05-09 16:44:18,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 5: [2023-05-09 16:44:18,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +22: [2023-05-09 16:44:18,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +22: [2023-05-09 16:44:18,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +10: [2023-05-09 16:44:18,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +10: [2023-05-09 16:44:18,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... + 5: [2023-05-09 16:44:18,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +19: [2023-05-09 16:44:18,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +19: [2023-05-09 16:44:18,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +25: [2023-05-09 16:44:18,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +25: [2023-05-09 16:44:18,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +25: [2023-05-09 16:44:18,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +25: [2023-05-09 16:44:18,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +25: [2023-05-09 16:44:18,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt. +25: [2023-05-09 16:44:18,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt... +18: [2023-05-09 16:44:18,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +18: [2023-05-09 16:44:18,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +18: [2023-05-09 16:44:18,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +18: [2023-05-09 16:44:18,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +18: [2023-05-09 16:44:18,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +18: [2023-05-09 16:44:18,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +18: [2023-05-09 16:44:18,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +18: [2023-05-09 16:44:18,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +29: [2023-05-09 16:44:18,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +29: [2023-05-09 16:44:18,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +29: [2023-05-09 16:44:18,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +29: [2023-05-09 16:44:18,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +29: [2023-05-09 16:44:18,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +29: [2023-05-09 16:44:18,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +29: [2023-05-09 16:44:18,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +29: [2023-05-09 16:44:18,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +29: [2023-05-09 16:44:18,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +29: [2023-05-09 16:44:18,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +29: [2023-05-09 16:44:18,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +28: [2023-05-09 16:44:18,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +28: [2023-05-09 16:44:18,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +28: [2023-05-09 16:44:18,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +28: [2023-05-09 16:44:18,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +29: [2023-05-09 16:44:18,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +29: [2023-05-09 16:44:18,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +29: [2023-05-09 16:44:18,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +29: [2023-05-09 16:44:18,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +28: [2023-05-09 16:44:18,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +28: [2023-05-09 16:44:18,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +28: [2023-05-09 16:44:18,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +28: [2023-05-09 16:44:18,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +29: [2023-05-09 16:44:18,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +27: [2023-05-09 16:44:18,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +27: [2023-05-09 16:44:18,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +27: [2023-05-09 16:44:18,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +27: [2023-05-09 16:44:18,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 9: [2023-05-09 16:44:18,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +29: [2023-05-09 16:44:18,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +29: [2023-05-09 16:44:18,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +29: [2023-05-09 16:44:18,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +29: [2023-05-09 16:44:18,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +27: [2023-05-09 16:44:18,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +11: [2023-05-09 16:44:18,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +11: [2023-05-09 16:44:18,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +11: [2023-05-09 16:44:18,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +11: [2023-05-09 16:44:18,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +27: [2023-05-09 16:44:18,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +27: [2023-05-09 16:44:18,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +27: [2023-05-09 16:44:18,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +27: [2023-05-09 16:44:18,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 1: [2023-05-09 16:44:18,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +27: [2023-05-09 16:44:18,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +27: [2023-05-09 16:44:18,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +24: [2023-05-09 16:44:18,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +27: [2023-05-09 16:44:18,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +24: [2023-05-09 16:44:18,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 1: [2023-05-09 16:44:18,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +24: [2023-05-09 16:44:18,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +24: [2023-05-09 16:44:18,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 8: [2023-05-09 16:44:18,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 8: [2023-05-09 16:44:18,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 8: [2023-05-09 16:44:18,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +16: [2023-05-09 16:44:18,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 1: [2023-05-09 16:44:18,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +16: [2023-05-09 16:44:18,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +16: [2023-05-09 16:44:18,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +16: [2023-05-09 16:44:18,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +27: [2023-05-09 16:44:18,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 2: [2023-05-09 16:44:18,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 2: [2023-05-09 16:44:18,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +27: [2023-05-09 16:44:18,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +21: [2023-05-09 16:44:18,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +21: [2023-05-09 16:44:18,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +21: [2023-05-09 16:44:18,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +21: [2023-05-09 16:44:18,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +12: [2023-05-09 16:44:18,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +29: [2023-05-09 16:44:18,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +12: [2023-05-09 16:44:18,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +29: [2023-05-09 16:44:18,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +12: [2023-05-09 16:44:18,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +12: [2023-05-09 16:44:18,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +27: [2023-05-09 16:44:18,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 1: [2023-05-09 16:44:18,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +29: [2023-05-09 16:44:18,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +11: [2023-05-09 16:44:18,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +11: [2023-05-09 16:44:18,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +27: [2023-05-09 16:44:18,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +11: [2023-05-09 16:44:18,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +29: [2023-05-09 16:44:18,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +11: [2023-05-09 16:44:18,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 8: [2023-05-09 16:44:18,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +16: [2023-05-09 16:44:18,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +16: [2023-05-09 16:44:18,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +24: [2023-05-09 16:44:18,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +24: [2023-05-09 16:44:18,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +21: [2023-05-09 16:44:18,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +16: [2023-05-09 16:44:18,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +12: [2023-05-09 16:44:18,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +21: [2023-05-09 16:44:18,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +12: [2023-05-09 16:44:18,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +14: [2023-05-09 16:44:18,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +16: [2023-05-09 16:44:18,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +12: [2023-05-09 16:44:18,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +24: [2023-05-09 16:44:18,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +14: [2023-05-09 16:44:18,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +14: [2023-05-09 16:44:18,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 2: [2023-05-09 16:44:18,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +12: [2023-05-09 16:44:18,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +21: [2023-05-09 16:44:18,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +24: [2023-05-09 16:44:18,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +21: [2023-05-09 16:44:18,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +27: [2023-05-09 16:44:18,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +17: [2023-05-09 16:44:18,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 1: [2023-05-09 16:44:18,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +17: [2023-05-09 16:44:18,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +17: [2023-05-09 16:44:18,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +17: [2023-05-09 16:44:18,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +14: [2023-05-09 16:44:18,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +27: [2023-05-09 16:44:18,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +29: [2023-05-09 16:44:18,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +27: [2023-05-09 16:44:18,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +29: [2023-05-09 16:44:18,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 1: [2023-05-09 16:44:18,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +27: [2023-05-09 16:44:18,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 9: [2023-05-09 16:44:18,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +29: [2023-05-09 16:44:18,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +29: [2023-05-09 16:44:18,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +29: [2023-05-09 16:44:18,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +29: [2023-05-09 16:44:18,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +14: [2023-05-09 16:44:18,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +27: [2023-05-09 16:44:18,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +29: [2023-05-09 16:44:18,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +29: [2023-05-09 16:44:18,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +14: [2023-05-09 16:44:18,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +17: [2023-05-09 16:44:18,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +14: [2023-05-09 16:44:18,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 8: [2023-05-09 16:44:18,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +27: [2023-05-09 16:44:18,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +14: [2023-05-09 16:44:18,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +27: [2023-05-09 16:44:18,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +27: [2023-05-09 16:44:18,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +17: [2023-05-09 16:44:18,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +17: [2023-05-09 16:44:18,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +17: [2023-05-09 16:44:18,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +27: [2023-05-09 16:44:18,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +27: [2023-05-09 16:44:18,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +27: [2023-05-09 16:44:18,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +13: [2023-05-09 16:44:18,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +13: [2023-05-09 16:44:18,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +13: [2023-05-09 16:44:18,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +13: [2023-05-09 16:44:18,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +27: [2023-05-09 16:44:18,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +27: [2023-05-09 16:44:18,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 8: [2023-05-09 16:44:18,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 1: [2023-05-09 16:44:18,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +27: [2023-05-09 16:44:18,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +27: [2023-05-09 16:44:18,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 1: [2023-05-09 16:44:18,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 8: [2023-05-09 16:44:18,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 1: [2023-05-09 16:44:18,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +13: [2023-05-09 16:44:18,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +13: [2023-05-09 16:44:18,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +13: [2023-05-09 16:44:18,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +13: [2023-05-09 16:44:18,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +11: [2023-05-09 16:44:18,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +11: [2023-05-09 16:44:18,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +11: [2023-05-09 16:44:18,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +11: [2023-05-09 16:44:18,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +11: [2023-05-09 16:44:18,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +11: [2023-05-09 16:44:18,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +11: [2023-05-09 16:44:18,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +11: [2023-05-09 16:44:18,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +11: [2023-05-09 16:44:18,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +11: [2023-05-09 16:44:18,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +11: [2023-05-09 16:44:18,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +11: [2023-05-09 16:44:18,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +11: [2023-05-09 16:44:18,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +27: [2023-05-09 16:44:18,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +11: [2023-05-09 16:44:18,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +22: [2023-05-09 16:44:18,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +22: [2023-05-09 16:44:18,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +11: [2023-05-09 16:44:18,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +22: [2023-05-09 16:44:18,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +22: [2023-05-09 16:44:18,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +11: [2023-05-09 16:44:18,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +16: [2023-05-09 16:44:18,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +16: [2023-05-09 16:44:18,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +16: [2023-05-09 16:44:18,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +16: [2023-05-09 16:44:18,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +16: [2023-05-09 16:44:18,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +16: [2023-05-09 16:44:18,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +16: [2023-05-09 16:44:18,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +16: [2023-05-09 16:44:18,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +16: [2023-05-09 16:44:18,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +16: [2023-05-09 16:44:18,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +16: [2023-05-09 16:44:18,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +16: [2023-05-09 16:44:18,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +16: [2023-05-09 16:44:18,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +16: [2023-05-09 16:44:18,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 1: [2023-05-09 16:44:18,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +16: [2023-05-09 16:44:18,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +16: [2023-05-09 16:44:18,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +23: [2023-05-09 16:44:18,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +23: [2023-05-09 16:44:18,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +23: [2023-05-09 16:44:18,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +23: [2023-05-09 16:44:18,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 1: [2023-05-09 16:44:18,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 4: [2023-05-09 16:44:18,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 4: [2023-05-09 16:44:18,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +22: [2023-05-09 16:44:18,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 4: [2023-05-09 16:44:18,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +22: [2023-05-09 16:44:18,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +22: [2023-05-09 16:44:18,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +22: [2023-05-09 16:44:18,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +10: [2023-05-09 16:44:18,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +10: [2023-05-09 16:44:18,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +10: [2023-05-09 16:44:18,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +10: [2023-05-09 16:44:18,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +16: [2023-05-09 16:44:18,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +16: [2023-05-09 16:44:18,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +16: [2023-05-09 16:44:18,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +30: [2023-05-09 16:44:18,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 6: [2023-05-09 16:44:18,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +23: [2023-05-09 16:44:18,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +30: [2023-05-09 16:44:18,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +30: [2023-05-09 16:44:18,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 6: [2023-05-09 16:44:18,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +23: [2023-05-09 16:44:18,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +30: [2023-05-09 16:44:18,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 3: [2023-05-09 16:44:18,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 3: [2023-05-09 16:44:18,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 3: [2023-05-09 16:44:18,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 3: [2023-05-09 16:44:18,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 7: [2023-05-09 16:44:18,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 7: [2023-05-09 16:44:18,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 7: [2023-05-09 16:44:18,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 7: [2023-05-09 16:44:18,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +23: [2023-05-09 16:44:18,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +23: [2023-05-09 16:44:18,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +11: [2023-05-09 16:44:18,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +11: [2023-05-09 16:44:18,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +11: [2023-05-09 16:44:18,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +11: [2023-05-09 16:44:18,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +16: [2023-05-09 16:44:18,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +10: [2023-05-09 16:44:18,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +10: [2023-05-09 16:44:18,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +10: [2023-05-09 16:44:18,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 0: [2023-05-09 16:44:18,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 0: [2023-05-09 16:44:18,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 0: [2023-05-09 16:44:18,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 4: [2023-05-09 16:44:18,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +10: [2023-05-09 16:44:18,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +16: [2023-05-09 16:44:18,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +16: [2023-05-09 16:44:18,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +30: [2023-05-09 16:44:18,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +25: [2023-05-09 16:44:18,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +25: [2023-05-09 16:44:18,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +25: [2023-05-09 16:44:18,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 3: [2023-05-09 16:44:18,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +25: [2023-05-09 16:44:18,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 6: [2023-05-09 16:44:18,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +18: [2023-05-09 16:44:18,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +30: [2023-05-09 16:44:18,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +18: [2023-05-09 16:44:18,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +18: [2023-05-09 16:44:18,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +18: [2023-05-09 16:44:18,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +18: [2023-05-09 16:44:18,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +18: [2023-05-09 16:44:18,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +18: [2023-05-09 16:44:18,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +18: [2023-05-09 16:44:18,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +30: [2023-05-09 16:44:18,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +18: [2023-05-09 16:44:18,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +18: [2023-05-09 16:44:18,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +30: [2023-05-09 16:44:18,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +18: [2023-05-09 16:44:18,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +18: [2023-05-09 16:44:18,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 7: [2023-05-09 16:44:18,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +18: [2023-05-09 16:44:18,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +18: [2023-05-09 16:44:18,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +18: [2023-05-09 16:44:18,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +16: [2023-05-09 16:44:18,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +18: [2023-05-09 16:44:18,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +11: [2023-05-09 16:44:18,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +11: [2023-05-09 16:44:18,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +11: [2023-05-09 16:44:18,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +25: [2023-05-09 16:44:18,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +11: [2023-05-09 16:44:18,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +25: [2023-05-09 16:44:18,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +25: [2023-05-09 16:44:18,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +25: [2023-05-09 16:44:18,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +10: [2023-05-09 16:44:18,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +19: [2023-05-09 16:44:18,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +19: [2023-05-09 16:44:18,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +19: [2023-05-09 16:44:18,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +21: [2023-05-09 16:44:18,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +21: [2023-05-09 16:44:18,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +21: [2023-05-09 16:44:18,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +10: [2023-05-09 16:44:18,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +19: [2023-05-09 16:44:18,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +10: [2023-05-09 16:44:18,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +18: [2023-05-09 16:44:18,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +16: [2023-05-09 16:44:18,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +21: [2023-05-09 16:44:18,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +21: [2023-05-09 16:44:18,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +21: [2023-05-09 16:44:18,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +21: [2023-05-09 16:44:18,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +17: [2023-05-09 16:44:18,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +21: [2023-05-09 16:44:18,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +21: [2023-05-09 16:44:18,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +17: [2023-05-09 16:44:18,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +31: [2023-05-09 16:44:18,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +21: [2023-05-09 16:44:18,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +31: [2023-05-09 16:44:18,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +31: [2023-05-09 16:44:18,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +10: [2023-05-09 16:44:18,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +17: [2023-05-09 16:44:18,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +10: [2023-05-09 16:44:18,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +17: [2023-05-09 16:44:18,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +10: [2023-05-09 16:44:18,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +21: [2023-05-09 16:44:18,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +21: [2023-05-09 16:44:18,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +17: [2023-05-09 16:44:18,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +21: [2023-05-09 16:44:18,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +31: [2023-05-09 16:44:18,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +17: [2023-05-09 16:44:18,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +10: [2023-05-09 16:44:18,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +17: [2023-05-09 16:44:18,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +17: [2023-05-09 16:44:18,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +17: [2023-05-09 16:44:18,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +17: [2023-05-09 16:44:18,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 9: [2023-05-09 16:44:18,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +17: [2023-05-09 16:44:18,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +21: [2023-05-09 16:44:18,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 9: [2023-05-09 16:44:18,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 9: [2023-05-09 16:44:18,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +17: [2023-05-09 16:44:18,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +26: [2023-05-09 16:44:18,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +17: [2023-05-09 16:44:18,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +26: [2023-05-09 16:44:18,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +26: [2023-05-09 16:44:18,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +26: [2023-05-09 16:44:18,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +21: [2023-05-09 16:44:18,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +21: [2023-05-09 16:44:18,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +10: [2023-05-09 16:44:18,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +17: [2023-05-09 16:44:18,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +10: [2023-05-09 16:44:18,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +17: [2023-05-09 16:44:18,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +17: [2023-05-09 16:44:18,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +15: [2023-05-09 16:44:18,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +15: [2023-05-09 16:44:18,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +15: [2023-05-09 16:44:18,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +15: [2023-05-09 16:44:18,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +18: [2023-05-09 16:44:18,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +10: [2023-05-09 16:44:18,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 5: [2023-05-09 16:44:18,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 5: [2023-05-09 16:44:18,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. + 5: [2023-05-09 16:44:18,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +20: [2023-05-09 16:44:18,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +20: [2023-05-09 16:44:18,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +20: [2023-05-09 16:44:18,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +20: [2023-05-09 16:44:18,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt. +19: [2023-05-09 16:44:18,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +19: [2023-05-09 16:44:18,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +31: [2023-05-09 16:44:18,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +19: [2023-05-09 16:44:18,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +19: [2023-05-09 16:44:18,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +10: [2023-05-09 16:44:18,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +18: [2023-05-09 16:44:18,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +21: [2023-05-09 16:44:18,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +31: [2023-05-09 16:44:18,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +31: [2023-05-09 16:44:18,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +31: [2023-05-09 16:44:18,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +10: [2023-05-09 16:44:18,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +26: [2023-05-09 16:44:18,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +26: [2023-05-09 16:44:18,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +26: [2023-05-09 16:44:18,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +26: [2023-05-09 16:44:18,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +18: [2023-05-09 16:44:18,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +18: [2023-05-09 16:44:18,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +17: [2023-05-09 16:44:18,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +15: [2023-05-09 16:44:18,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +15: [2023-05-09 16:44:18,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +15: [2023-05-09 16:44:18,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +15: [2023-05-09 16:44:18,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +17: [2023-05-09 16:44:18,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +20: [2023-05-09 16:44:18,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +20: [2023-05-09 16:44:18,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +10: [2023-05-09 16:44:18,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +10: [2023-05-09 16:44:18,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +20: [2023-05-09 16:44:18,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +26: [2023-05-09 16:44:18,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +26: [2023-05-09 16:44:18,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +26: [2023-05-09 16:44:18,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +26: [2023-05-09 16:44:18,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +20: [2023-05-09 16:44:18,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +26: [2023-05-09 16:44:18,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +26: [2023-05-09 16:44:18,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +26: [2023-05-09 16:44:18,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +18: [2023-05-09 16:44:18,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +21: [2023-05-09 16:44:18,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +21: [2023-05-09 16:44:18,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +18: [2023-05-09 16:44:18,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +21: [2023-05-09 16:44:18,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +21: [2023-05-09 16:44:18,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +26: [2023-05-09 16:44:18,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +17: [2023-05-09 16:44:18,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +10: [2023-05-09 16:44:18,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +10: [2023-05-09 16:44:18,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +17: [2023-05-09 16:44:18,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +10: [2023-05-09 16:44:18,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +17: [2023-05-09 16:44:18,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +17: [2023-05-09 16:44:18,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +10: [2023-05-09 16:44:18,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +18: [2023-05-09 16:44:18,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 9: [2023-05-09 16:44:18,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +21: [2023-05-09 16:44:18,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +25: [2023-05-09 16:44:18,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +25: [2023-05-09 16:44:18,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +25: [2023-05-09 16:44:18,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +25: [2023-05-09 16:44:18,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +25: [2023-05-09 16:44:18,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +25: [2023-05-09 16:44:18,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +25: [2023-05-09 16:44:18,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +25: [2023-05-09 16:44:18,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +25: [2023-05-09 16:44:18,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +25: [2023-05-09 16:44:18,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +25: [2023-05-09 16:44:18,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +25: [2023-05-09 16:44:18,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +25: [2023-05-09 16:44:18,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +21: [2023-05-09 16:44:18,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +10: [2023-05-09 16:44:18,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +26: [2023-05-09 16:44:18,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +26: [2023-05-09 16:44:18,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +26: [2023-05-09 16:44:18,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +26: [2023-05-09 16:44:18,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +26: [2023-05-09 16:44:18,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +25: [2023-05-09 16:44:18,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +26: [2023-05-09 16:44:18,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +25: [2023-05-09 16:44:18,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +26: [2023-05-09 16:44:18,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +25: [2023-05-09 16:44:18,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +10: [2023-05-09 16:44:18,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +21: [2023-05-09 16:44:18,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +26: [2023-05-09 16:44:18,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +26: [2023-05-09 16:44:18,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +17: [2023-05-09 16:44:18,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +17: [2023-05-09 16:44:18,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 6: [2023-05-09 16:44:18,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 6: [2023-05-09 16:44:18,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 6: [2023-05-09 16:44:18,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 6: [2023-05-09 16:44:18,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +10: [2023-05-09 16:44:18,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +10: [2023-05-09 16:44:18,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +25: [2023-05-09 16:44:18,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +26: [2023-05-09 16:44:18,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +26: [2023-05-09 16:44:18,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +10: [2023-05-09 16:44:18,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +26: [2023-05-09 16:44:18,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +26: [2023-05-09 16:44:18,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +26: [2023-05-09 16:44:18,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +26: [2023-05-09 16:44:18,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 6: [2023-05-09 16:44:18,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +25: [2023-05-09 16:44:18,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +10: [2023-05-09 16:44:18,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +25: [2023-05-09 16:44:18,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +26: [2023-05-09 16:44:18,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +25: [2023-05-09 16:44:18,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 7: [2023-05-09 16:44:18,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 7: [2023-05-09 16:44:18,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +25: [2023-05-09 16:44:18,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 7: [2023-05-09 16:44:18,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +25: [2023-05-09 16:44:18,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +25: [2023-05-09 16:44:18,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +28: [2023-05-09 16:44:18,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +28: [2023-05-09 16:44:18,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +25: [2023-05-09 16:44:18,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +28: [2023-05-09 16:44:18,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 7: [2023-05-09 16:44:18,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +28: [2023-05-09 16:44:18,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +28: [2023-05-09 16:44:18,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +28: [2023-05-09 16:44:18,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +28: [2023-05-09 16:44:18,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +28: [2023-05-09 16:44:18,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +28: [2023-05-09 16:44:18,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +28: [2023-05-09 16:44:18,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 7: [2023-05-09 16:44:18,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +28: [2023-05-09 16:44:18,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +28: [2023-05-09 16:44:18,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +28: [2023-05-09 16:44:18,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +28: [2023-05-09 16:44:18,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +28: [2023-05-09 16:44:18,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +28: [2023-05-09 16:44:18,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +28: [2023-05-09 16:44:18,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +28: [2023-05-09 16:44:18,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +28: [2023-05-09 16:44:18,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +28: [2023-05-09 16:44:18,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +28: [2023-05-09 16:44:18,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +28: [2023-05-09 16:44:18,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +30: [2023-05-09 16:44:18,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +30: [2023-05-09 16:44:18,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +30: [2023-05-09 16:44:18,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +30: [2023-05-09 16:44:18,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +30: [2023-05-09 16:44:18,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +30: [2023-05-09 16:44:18,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +30: [2023-05-09 16:44:18,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +30: [2023-05-09 16:44:18,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +30: [2023-05-09 16:44:18,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +30: [2023-05-09 16:44:18,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +30: [2023-05-09 16:44:18,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +30: [2023-05-09 16:44:18,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +30: [2023-05-09 16:44:18,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +30: [2023-05-09 16:44:18,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +30: [2023-05-09 16:44:18,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +30: [2023-05-09 16:44:18,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +28: [2023-05-09 16:44:18,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +28: [2023-05-09 16:44:18,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +30: [2023-05-09 16:44:18,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +30: [2023-05-09 16:44:18,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +22: [2023-05-09 16:44:18,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +22: [2023-05-09 16:44:18,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +22: [2023-05-09 16:44:18,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +22: [2023-05-09 16:44:18,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +22: [2023-05-09 16:44:18,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +22: [2023-05-09 16:44:18,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +22: [2023-05-09 16:44:18,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +22: [2023-05-09 16:44:18,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +22: [2023-05-09 16:44:18,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +22: [2023-05-09 16:44:18,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +22: [2023-05-09 16:44:18,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +22: [2023-05-09 16:44:18,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +30: [2023-05-09 16:44:18,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +22: [2023-05-09 16:44:18,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +22: [2023-05-09 16:44:18,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +30: [2023-05-09 16:44:18,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +22: [2023-05-09 16:44:18,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +22: [2023-05-09 16:44:18,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +27: [2023-05-09 16:44:18,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +27: [2023-05-09 16:44:18,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +27: [2023-05-09 16:44:18,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +27: [2023-05-09 16:44:18,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +27: [2023-05-09 16:44:18,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +27: [2023-05-09 16:44:18,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +27: [2023-05-09 16:44:18,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +30: [2023-05-09 16:44:18,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +27: [2023-05-09 16:44:18,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +30: [2023-05-09 16:44:18,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +22: [2023-05-09 16:44:18,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +30: [2023-05-09 16:44:18,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +22: [2023-05-09 16:44:18,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +27: [2023-05-09 16:44:18,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +12: [2023-05-09 16:44:18,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +12: [2023-05-09 16:44:18,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +12: [2023-05-09 16:44:18,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +12: [2023-05-09 16:44:18,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +12: [2023-05-09 16:44:18,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +12: [2023-05-09 16:44:18,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +12: [2023-05-09 16:44:18,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +12: [2023-05-09 16:44:18,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +12: [2023-05-09 16:44:18,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +12: [2023-05-09 16:44:18,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +12: [2023-05-09 16:44:18,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +12: [2023-05-09 16:44:18,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +12: [2023-05-09 16:44:18,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +30: [2023-05-09 16:44:18,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +12: [2023-05-09 16:44:18,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +27: [2023-05-09 16:44:18,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +27: [2023-05-09 16:44:18,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +12: [2023-05-09 16:44:18,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +12: [2023-05-09 16:44:18,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +16: [2023-05-09 16:44:18,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +16: [2023-05-09 16:44:18,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +16: [2023-05-09 16:44:18,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +16: [2023-05-09 16:44:18,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +16: [2023-05-09 16:44:18,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +22: [2023-05-09 16:44:18,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +16: [2023-05-09 16:44:18,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +16: [2023-05-09 16:44:18,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +16: [2023-05-09 16:44:18,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +22: [2023-05-09 16:44:18,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +27: [2023-05-09 16:44:18,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +22: [2023-05-09 16:44:18,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +27: [2023-05-09 16:44:18,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +27: [2023-05-09 16:44:18,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +27: [2023-05-09 16:44:18,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +16: [2023-05-09 16:44:18,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +12: [2023-05-09 16:44:18,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +16: [2023-05-09 16:44:18,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +16: [2023-05-09 16:44:18,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +16: [2023-05-09 16:44:18,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +22: [2023-05-09 16:44:18,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +22: [2023-05-09 16:44:18,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +27: [2023-05-09 16:44:18,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +12: [2023-05-09 16:44:18,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +12: [2023-05-09 16:44:18,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +16: [2023-05-09 16:44:18,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +16: [2023-05-09 16:44:18,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +16: [2023-05-09 16:44:18,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +16: [2023-05-09 16:44:18,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +22: [2023-05-09 16:44:18,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +12: [2023-05-09 16:44:18,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +12: [2023-05-09 16:44:18,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +12: [2023-05-09 16:44:18,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +29: [2023-05-09 16:44:18,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +29: [2023-05-09 16:44:18,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +29: [2023-05-09 16:44:18,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +29: [2023-05-09 16:44:18,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +29: [2023-05-09 16:44:18,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +29: [2023-05-09 16:44:18,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 3: [2023-05-09 16:44:18,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 3: [2023-05-09 16:44:18,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +29: [2023-05-09 16:44:18,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +29: [2023-05-09 16:44:18,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +12: [2023-05-09 16:44:18,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +12: [2023-05-09 16:44:18,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 1: [2023-05-09 16:44:18,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +29: [2023-05-09 16:44:18,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +29: [2023-05-09 16:44:18,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +21: [2023-05-09 16:44:18,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +21: [2023-05-09 16:44:18,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +21: [2023-05-09 16:44:18,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +21: [2023-05-09 16:44:18,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +21: [2023-05-09 16:44:18,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +21: [2023-05-09 16:44:18,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 4: [2023-05-09 16:44:18,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 0: [2023-05-09 16:44:18,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +21: [2023-05-09 16:44:18,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +21: [2023-05-09 16:44:18,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 0: [2023-05-09 16:44:18,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 0: [2023-05-09 16:44:18,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 9: [2023-05-09 16:44:18,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +11: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +11: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +11: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +11: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +11: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +18: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +11: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 5: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +18: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +18: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +18: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +18: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +18: [2023-05-09 16:44:18,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +18: [2023-05-09 16:44:18,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 0: [2023-05-09 16:44:18,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 0: [2023-05-09 16:44:18,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +18: [2023-05-09 16:44:18,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +11: [2023-05-09 16:44:18,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +11: [2023-05-09 16:44:18,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 5: [2023-05-09 16:44:18,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 1: [2023-05-09 16:44:18,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +29: [2023-05-09 16:44:18,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +29: [2023-05-09 16:44:18,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +29: [2023-05-09 16:44:18,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +29: [2023-05-09 16:44:18,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +18: [2023-05-09 16:44:18,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +18: [2023-05-09 16:44:18,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +18: [2023-05-09 16:44:18,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +21: [2023-05-09 16:44:18,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +31: [2023-05-09 16:44:18,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +31: [2023-05-09 16:44:18,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +18: [2023-05-09 16:44:18,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +31: [2023-05-09 16:44:18,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +31: [2023-05-09 16:44:18,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +31: [2023-05-09 16:44:18,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +31: [2023-05-09 16:44:18,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +31: [2023-05-09 16:44:18,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +31: [2023-05-09 16:44:18,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 0: [2023-05-09 16:44:18,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +31: [2023-05-09 16:44:18,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +31: [2023-05-09 16:44:18,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +31: [2023-05-09 16:44:18,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +31: [2023-05-09 16:44:18,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +31: [2023-05-09 16:44:18,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +31: [2023-05-09 16:44:18,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +31: [2023-05-09 16:44:18,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +31: [2023-05-09 16:44:18,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 1: [2023-05-09 16:44:18,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +29: [2023-05-09 16:44:18,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +29: [2023-05-09 16:44:18,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +21: [2023-05-09 16:44:18,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +18: [2023-05-09 16:44:18,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +15: [2023-05-09 16:44:18,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +15: [2023-05-09 16:44:18,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +15: [2023-05-09 16:44:18,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +15: [2023-05-09 16:44:18,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +15: [2023-05-09 16:44:18,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +15: [2023-05-09 16:44:18,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +15: [2023-05-09 16:44:18,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +15: [2023-05-09 16:44:18,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +15: [2023-05-09 16:44:18,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +15: [2023-05-09 16:44:18,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +15: [2023-05-09 16:44:18,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +18: [2023-05-09 16:44:18,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +15: [2023-05-09 16:44:18,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +21: [2023-05-09 16:44:18,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +11: [2023-05-09 16:44:18,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +11: [2023-05-09 16:44:18,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +15: [2023-05-09 16:44:18,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +18: [2023-05-09 16:44:18,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +18: [2023-05-09 16:44:18,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +15: [2023-05-09 16:44:18,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +18: [2023-05-09 16:44:18,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +15: [2023-05-09 16:44:18,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +15: [2023-05-09 16:44:18,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +31: [2023-05-09 16:44:18,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +18: [2023-05-09 16:44:18,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +31: [2023-05-09 16:44:18,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +31: [2023-05-09 16:44:18,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +21: [2023-05-09 16:44:18,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +10: [2023-05-09 16:44:18,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +10: [2023-05-09 16:44:18,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 4: [2023-05-09 16:44:18,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +23: [2023-05-09 16:44:18,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +18: [2023-05-09 16:44:18,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +23: [2023-05-09 16:44:18,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +23: [2023-05-09 16:44:18,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +18: [2023-05-09 16:44:18,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +23: [2023-05-09 16:44:18,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +23: [2023-05-09 16:44:18,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +23: [2023-05-09 16:44:18,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +23: [2023-05-09 16:44:18,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +23: [2023-05-09 16:44:18,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +10: [2023-05-09 16:44:18,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +23: [2023-05-09 16:44:18,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +31: [2023-05-09 16:44:18,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +23: [2023-05-09 16:44:18,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +23: [2023-05-09 16:44:18,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +23: [2023-05-09 16:44:18,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +10: [2023-05-09 16:44:18,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 5: [2023-05-09 16:44:18,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +15: [2023-05-09 16:44:18,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +23: [2023-05-09 16:44:18,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +23: [2023-05-09 16:44:18,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +23: [2023-05-09 16:44:18,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +23: [2023-05-09 16:44:18,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +11: [2023-05-09 16:44:18,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +11: [2023-05-09 16:44:18,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +18: [2023-05-09 16:44:18,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +21: [2023-05-09 16:44:18,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +21: [2023-05-09 16:44:18,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +18: [2023-05-09 16:44:18,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +31: [2023-05-09 16:44:18,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +31: [2023-05-09 16:44:18,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +24: [2023-05-09 16:44:18,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +24: [2023-05-09 16:44:18,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +24: [2023-05-09 16:44:18,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +24: [2023-05-09 16:44:18,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +24: [2023-05-09 16:44:18,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +24: [2023-05-09 16:44:18,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +24: [2023-05-09 16:44:18,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +18: [2023-05-09 16:44:18,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +24: [2023-05-09 16:44:18,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +24: [2023-05-09 16:44:18,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +11: [2023-05-09 16:44:18,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +11: [2023-05-09 16:44:18,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +24: [2023-05-09 16:44:18,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +31: [2023-05-09 16:44:18,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +24: [2023-05-09 16:44:18,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +31: [2023-05-09 16:44:18,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +24: [2023-05-09 16:44:18,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +24: [2023-05-09 16:44:18,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +15: [2023-05-09 16:44:18,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +24: [2023-05-09 16:44:18,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +24: [2023-05-09 16:44:18,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +24: [2023-05-09 16:44:18,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +18: [2023-05-09 16:44:18,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +10: [2023-05-09 16:44:18,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +10: [2023-05-09 16:44:18,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +10: [2023-05-09 16:44:18,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +18: [2023-05-09 16:44:18,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +10: [2023-05-09 16:44:18,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +18: [2023-05-09 16:44:18,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +23: [2023-05-09 16:44:18,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +23: [2023-05-09 16:44:18,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +19: [2023-05-09 16:44:18,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +19: [2023-05-09 16:44:18,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +19: [2023-05-09 16:44:18,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +21: [2023-05-09 16:44:18,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +18: [2023-05-09 16:44:18,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +19: [2023-05-09 16:44:18,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +21: [2023-05-09 16:44:18,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +18: [2023-05-09 16:44:18,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +27: [2023-05-09 16:44:18,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +27: [2023-05-09 16:44:18,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +19: [2023-05-09 16:44:18,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +27: [2023-05-09 16:44:18,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +19: [2023-05-09 16:44:18,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +19: [2023-05-09 16:44:18,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +19: [2023-05-09 16:44:18,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +19: [2023-05-09 16:44:18,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +27: [2023-05-09 16:44:18,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +19: [2023-05-09 16:44:18,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +19: [2023-05-09 16:44:18,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +19: [2023-05-09 16:44:18,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 0: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +20: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +20: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +20: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +19: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +19: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +20: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +14: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +14: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +20: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +19: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +14: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 2: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 2: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +20: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 2: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +14: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +14: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +14: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +14: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +14: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 2: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +14: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +14: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +20: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +20: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +20: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +20: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +20: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +20: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +20: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +20: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +14: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +15: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +20: [2023-05-09 16:44:18,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +14: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +19: [2023-05-09 16:44:18,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +11: [2023-05-09 16:44:18,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +11: [2023-05-09 16:44:18,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +20: [2023-05-09 16:44:18,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +14: [2023-05-09 16:44:18,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +15: [2023-05-09 16:44:18,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +14: [2023-05-09 16:44:18,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +14: [2023-05-09 16:44:18,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +14: [2023-05-09 16:44:18,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +18: [2023-05-09 16:44:18,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +18: [2023-05-09 16:44:18,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +18: [2023-05-09 16:44:18,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +18: [2023-05-09 16:44:18,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +23: [2023-05-09 16:44:18,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +23: [2023-05-09 16:44:18,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +27: [2023-05-09 16:44:18,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +27: [2023-05-09 16:44:18,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +27: [2023-05-09 16:44:18,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +27: [2023-05-09 16:44:18,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +19: [2023-05-09 16:44:18,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +15: [2023-05-09 16:44:18,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +23: [2023-05-09 16:44:18,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +14: [2023-05-09 16:44:18,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +23: [2023-05-09 16:44:18,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +20: [2023-05-09 16:44:18,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +20: [2023-05-09 16:44:18,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +24: [2023-05-09 16:44:18,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +15: [2023-05-09 16:44:18,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +15: [2023-05-09 16:44:18,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +20: [2023-05-09 16:44:18,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +24: [2023-05-09 16:44:18,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +24: [2023-05-09 16:44:18,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +24: [2023-05-09 16:44:18,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +27: [2023-05-09 16:44:18,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +15: [2023-05-09 16:44:18,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +27: [2023-05-09 16:44:18,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +14: [2023-05-09 16:44:18,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +19: [2023-05-09 16:44:18,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +19: [2023-05-09 16:44:18,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +23: [2023-05-09 16:44:18,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +23: [2023-05-09 16:44:18,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +19: [2023-05-09 16:44:18,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +20: [2023-05-09 16:44:18,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +20: [2023-05-09 16:44:18,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +27: [2023-05-09 16:44:18,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +27: [2023-05-09 16:44:18,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +27: [2023-05-09 16:44:18,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +27: [2023-05-09 16:44:18,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 2: [2023-05-09 16:44:18,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +20: [2023-05-09 16:44:18,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +13: [2023-05-09 16:44:18,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +13: [2023-05-09 16:44:18,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +13: [2023-05-09 16:44:18,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +13: [2023-05-09 16:44:18,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +13: [2023-05-09 16:44:18,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +13: [2023-05-09 16:44:18,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +13: [2023-05-09 16:44:18,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +13: [2023-05-09 16:44:18,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +13: [2023-05-09 16:44:18,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +13: [2023-05-09 16:44:18,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +24: [2023-05-09 16:44:18,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +13: [2023-05-09 16:44:18,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +13: [2023-05-09 16:44:18,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +27: [2023-05-09 16:44:18,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +14: [2023-05-09 16:44:18,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +27: [2023-05-09 16:44:18,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +13: [2023-05-09 16:44:18,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... +13: [2023-05-09 16:44:18,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt... + 2: [2023-05-09 16:44:18,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +24: [2023-05-09 16:44:18,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +24: [2023-05-09 16:44:18,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +13: [2023-05-09 16:44:18,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +29: [2023-05-09 16:44:18,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +29: [2023-05-09 16:44:18,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +19: [2023-05-09 16:44:18,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +29: [2023-05-09 16:44:18,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +29: [2023-05-09 16:44:18,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +24: [2023-05-09 16:44:18,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +13: [2023-05-09 16:44:18,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt... +19: [2023-05-09 16:44:18,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +20: [2023-05-09 16:44:18,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +14: [2023-05-09 16:44:18,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +14: [2023-05-09 16:44:18,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +13: [2023-05-09 16:44:18,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +14: [2023-05-09 16:44:18,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +29: [2023-05-09 16:44:18,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +19: [2023-05-09 16:44:18,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +29: [2023-05-09 16:44:18,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +29: [2023-05-09 16:44:18,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +29: [2023-05-09 16:44:18,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +20: [2023-05-09 16:44:18,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +14: [2023-05-09 16:44:18,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +14: [2023-05-09 16:44:18,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +17: [2023-05-09 16:44:18,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +13: [2023-05-09 16:44:18,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +13: [2023-05-09 16:44:18,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +17: [2023-05-09 16:44:18,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +17: [2023-05-09 16:44:18,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +17: [2023-05-09 16:44:18,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +17: [2023-05-09 16:44:18,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +17: [2023-05-09 16:44:18,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +17: [2023-05-09 16:44:18,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +17: [2023-05-09 16:44:18,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +13: [2023-05-09 16:44:18,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +19: [2023-05-09 16:44:18,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +29: [2023-05-09 16:44:18,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +29: [2023-05-09 16:44:18,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +29: [2023-05-09 16:44:18,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +29: [2023-05-09 16:44:18,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +29: [2023-05-09 16:44:18,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +29: [2023-05-09 16:44:18,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +13: [2023-05-09 16:44:18,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +29: [2023-05-09 16:44:18,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +17: [2023-05-09 16:44:18,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +29: [2023-05-09 16:44:18,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +17: [2023-05-09 16:44:18,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +13: [2023-05-09 16:44:18,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +17: [2023-05-09 16:44:18,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +17: [2023-05-09 16:44:18,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +26: [2023-05-09 16:44:18,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +26: [2023-05-09 16:44:18,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +26: [2023-05-09 16:44:18,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +26: [2023-05-09 16:44:18,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +26: [2023-05-09 16:44:18,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +26: [2023-05-09 16:44:18,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +26: [2023-05-09 16:44:18,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +17: [2023-05-09 16:44:18,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +17: [2023-05-09 16:44:18,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +26: [2023-05-09 16:44:18,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +17: [2023-05-09 16:44:18,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +17: [2023-05-09 16:44:18,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +13: [2023-05-09 16:44:18,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt. +17: [2023-05-09 16:44:18,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +17: [2023-05-09 16:44:18,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +10: [2023-05-09 16:44:18,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +10: [2023-05-09 16:44:18,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +10: [2023-05-09 16:44:18,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +10: [2023-05-09 16:44:18,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +10: [2023-05-09 16:44:18,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +10: [2023-05-09 16:44:18,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +10: [2023-05-09 16:44:18,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +10: [2023-05-09 16:44:18,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +10: [2023-05-09 16:44:18,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +10: [2023-05-09 16:44:18,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +10: [2023-05-09 16:44:18,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +10: [2023-05-09 16:44:18,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +10: [2023-05-09 16:44:18,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +10: [2023-05-09 16:44:18,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +10: [2023-05-09 16:44:18,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +25: [2023-05-09 16:44:18,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +25: [2023-05-09 16:44:18,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +25: [2023-05-09 16:44:18,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +25: [2023-05-09 16:44:18,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +25: [2023-05-09 16:44:18,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +25: [2023-05-09 16:44:18,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +25: [2023-05-09 16:44:18,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +25: [2023-05-09 16:44:18,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +21: [2023-05-09 16:44:18,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +21: [2023-05-09 16:44:18,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +10: [2023-05-09 16:44:18,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +21: [2023-05-09 16:44:18,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +21: [2023-05-09 16:44:18,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +17: [2023-05-09 16:44:18,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +13: [2023-05-09 16:44:18,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +17: [2023-05-09 16:44:18,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +17: [2023-05-09 16:44:18,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +17: [2023-05-09 16:44:18,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +17: [2023-05-09 16:44:18,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +17: [2023-05-09 16:44:18,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +26: [2023-05-09 16:44:18,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +26: [2023-05-09 16:44:18,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +26: [2023-05-09 16:44:18,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +10: [2023-05-09 16:44:18,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +25: [2023-05-09 16:44:18,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +21: [2023-05-09 16:44:18,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +21: [2023-05-09 16:44:18,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +21: [2023-05-09 16:44:18,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +22: [2023-05-09 16:44:18,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +22: [2023-05-09 16:44:18,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +22: [2023-05-09 16:44:18,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +22: [2023-05-09 16:44:18,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +22: [2023-05-09 16:44:18,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +22: [2023-05-09 16:44:18,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +21: [2023-05-09 16:44:18,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +22: [2023-05-09 16:44:18,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +22: [2023-05-09 16:44:18,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +26: [2023-05-09 16:44:18,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +26: [2023-05-09 16:44:18,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +17: [2023-05-09 16:44:18,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +26: [2023-05-09 16:44:18,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +10: [2023-05-09 16:44:18,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +17: [2023-05-09 16:44:18,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +10: [2023-05-09 16:44:18,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +26: [2023-05-09 16:44:18,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +25: [2023-05-09 16:44:18,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +10: [2023-05-09 16:44:18,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +25: [2023-05-09 16:44:18,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +25: [2023-05-09 16:44:18,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +25: [2023-05-09 16:44:18,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 1: [2023-05-09 16:44:18,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 1: [2023-05-09 16:44:18,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +28: [2023-05-09 16:44:18,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 1: [2023-05-09 16:44:18,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +28: [2023-05-09 16:44:18,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +17: [2023-05-09 16:44:18,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +17: [2023-05-09 16:44:18,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +17: [2023-05-09 16:44:18,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +17: [2023-05-09 16:44:18,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +17: [2023-05-09 16:44:18,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +17: [2023-05-09 16:44:18,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +22: [2023-05-09 16:44:18,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +28: [2023-05-09 16:44:18,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +10: [2023-05-09 16:44:18,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +22: [2023-05-09 16:44:18,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +28: [2023-05-09 16:44:18,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +26: [2023-05-09 16:44:18,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +21: [2023-05-09 16:44:18,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +21: [2023-05-09 16:44:18,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +26: [2023-05-09 16:44:18,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +26: [2023-05-09 16:44:18,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +26: [2023-05-09 16:44:18,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +22: [2023-05-09 16:44:18,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +21: [2023-05-09 16:44:18,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +21: [2023-05-09 16:44:18,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 8: [2023-05-09 16:44:18,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +21: [2023-05-09 16:44:18,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +21: [2023-05-09 16:44:18,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +22: [2023-05-09 16:44:18,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +10: [2023-05-09 16:44:18,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +26: [2023-05-09 16:44:18,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +21: [2023-05-09 16:44:18,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +21: [2023-05-09 16:44:18,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +25: [2023-05-09 16:44:18,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +11: [2023-05-09 16:44:18,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +11: [2023-05-09 16:44:18,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +11: [2023-05-09 16:44:18,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +11: [2023-05-09 16:44:18,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 3: [2023-05-09 16:44:18,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 3: [2023-05-09 16:44:18,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 3: [2023-05-09 16:44:18,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 8: [2023-05-09 16:44:18,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 3: [2023-05-09 16:44:18,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +25: [2023-05-09 16:44:18,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +25: [2023-05-09 16:44:18,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +28: [2023-05-09 16:44:18,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +28: [2023-05-09 16:44:18,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 1: [2023-05-09 16:44:18,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +22: [2023-05-09 16:44:18,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +10: [2023-05-09 16:44:18,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +28: [2023-05-09 16:44:18,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +12: [2023-05-09 16:44:18,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 1: [2023-05-09 16:44:18,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +12: [2023-05-09 16:44:18,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 1: [2023-05-09 16:44:18,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +12: [2023-05-09 16:44:18,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +12: [2023-05-09 16:44:18,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +22: [2023-05-09 16:44:18,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +28: [2023-05-09 16:44:18,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +30: [2023-05-09 16:44:18,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +30: [2023-05-09 16:44:18,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +30: [2023-05-09 16:44:18,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +30: [2023-05-09 16:44:18,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +25: [2023-05-09 16:44:18,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +25: [2023-05-09 16:44:18,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +25: [2023-05-09 16:44:18,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 9: [2023-05-09 16:44:18,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 9: [2023-05-09 16:44:18,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +25: [2023-05-09 16:44:18,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +26: [2023-05-09 16:44:18,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +22: [2023-05-09 16:44:18,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 6: [2023-05-09 16:44:18,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 7: [2023-05-09 16:44:18,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 7: [2023-05-09 16:44:18,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 7: [2023-05-09 16:44:18,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 6: [2023-05-09 16:44:18,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 0: [2023-05-09 16:44:18,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 0: [2023-05-09 16:44:18,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 7: [2023-05-09 16:44:18,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +26: [2023-05-09 16:44:18,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 4: [2023-05-09 16:44:18,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 4: [2023-05-09 16:44:18,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +11: [2023-05-09 16:44:18,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +26: [2023-05-09 16:44:18,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 5: [2023-05-09 16:44:18,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 0: [2023-05-09 16:44:18,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 0: [2023-05-09 16:44:18,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 4: [2023-05-09 16:44:18,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +16: [2023-05-09 16:44:18,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 9: [2023-05-09 16:44:18,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 5: [2023-05-09 16:44:18,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +16: [2023-05-09 16:44:18,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +11: [2023-05-09 16:44:18,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +26: [2023-05-09 16:44:18,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +16: [2023-05-09 16:44:18,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 8: [2023-05-09 16:44:18,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +11: [2023-05-09 16:44:18,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +16: [2023-05-09 16:44:18,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 5: [2023-05-09 16:44:18,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 8: [2023-05-09 16:44:18,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +22: [2023-05-09 16:44:18,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +15: [2023-05-09 16:44:18,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +15: [2023-05-09 16:44:18,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +22: [2023-05-09 16:44:18,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +22: [2023-05-09 16:44:18,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +15: [2023-05-09 16:44:18,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +15: [2023-05-09 16:44:18,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +10: [2023-05-09 16:44:18,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +22: [2023-05-09 16:44:18,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 3: [2023-05-09 16:44:18,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +22: [2023-05-09 16:44:18,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 8: [2023-05-09 16:44:18,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +11: [2023-05-09 16:44:18,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +23: [2023-05-09 16:44:18,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +31: [2023-05-09 16:44:18,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +23: [2023-05-09 16:44:18,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +31: [2023-05-09 16:44:18,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +23: [2023-05-09 16:44:18,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +31: [2023-05-09 16:44:18,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +31: [2023-05-09 16:44:18,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +23: [2023-05-09 16:44:18,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +30: [2023-05-09 16:44:18,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +30: [2023-05-09 16:44:18,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +30: [2023-05-09 16:44:18,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +25: [2023-05-09 16:44:18,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +12: [2023-05-09 16:44:18,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +12: [2023-05-09 16:44:18,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +12: [2023-05-09 16:44:18,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +12: [2023-05-09 16:44:18,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +30: [2023-05-09 16:44:18,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +25: [2023-05-09 16:44:18,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +25: [2023-05-09 16:44:18,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +25: [2023-05-09 16:44:18,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 9: [2023-05-09 16:44:18,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +16: [2023-05-09 16:44:18,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +16: [2023-05-09 16:44:18,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +26: [2023-05-09 16:44:18,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +11: [2023-05-09 16:44:18,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +26: [2023-05-09 16:44:18,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +15: [2023-05-09 16:44:18,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +16: [2023-05-09 16:44:18,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +11: [2023-05-09 16:44:18,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +16: [2023-05-09 16:44:18,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +31: [2023-05-09 16:44:18,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +15: [2023-05-09 16:44:18,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 1: [2023-05-09 16:44:18,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 4: [2023-05-09 16:44:18,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +15: [2023-05-09 16:44:18,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +23: [2023-05-09 16:44:18,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +31: [2023-05-09 16:44:18,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +22: [2023-05-09 16:44:18,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +22: [2023-05-09 16:44:18,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +23: [2023-05-09 16:44:18,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +31: [2023-05-09 16:44:18,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +31: [2023-05-09 16:44:18,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +23: [2023-05-09 16:44:18,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +22: [2023-05-09 16:44:18,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +15: [2023-05-09 16:44:18,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +22: [2023-05-09 16:44:18,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +23: [2023-05-09 16:44:18,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +11: [2023-05-09 16:44:18,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +11: [2023-05-09 16:44:18,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 8: [2023-05-09 16:44:18,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +26: [2023-05-09 16:44:18,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +26: [2023-05-09 16:44:18,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 6: [2023-05-09 16:44:18,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 9: [2023-05-09 16:44:18,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +25: [2023-05-09 16:44:18,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +25: [2023-05-09 16:44:18,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 8: [2023-05-09 16:44:18,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 8: [2023-05-09 16:44:18,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +11: [2023-05-09 16:44:18,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +11: [2023-05-09 16:44:18,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 8: [2023-05-09 16:44:18,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 9: [2023-05-09 16:44:18,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +16: [2023-05-09 16:44:18,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +11: [2023-05-09 16:44:18,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +16: [2023-05-09 16:44:18,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +19: [2023-05-09 16:44:18,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +19: [2023-05-09 16:44:18,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +19: [2023-05-09 16:44:18,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +19: [2023-05-09 16:44:18,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +12: [2023-05-09 16:44:18,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +12: [2023-05-09 16:44:18,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +13: [2023-05-09 16:44:18,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +13: [2023-05-09 16:44:18,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +12: [2023-05-09 16:44:18,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +12: [2023-05-09 16:44:18,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +12: [2023-05-09 16:44:18,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +12: [2023-05-09 16:44:18,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +12: [2023-05-09 16:44:18,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +13: [2023-05-09 16:44:18,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 8: [2023-05-09 16:44:18,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +13: [2023-05-09 16:44:18,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +12: [2023-05-09 16:44:18,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +12: [2023-05-09 16:44:18,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +16: [2023-05-09 16:44:18,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +12: [2023-05-09 16:44:18,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +16: [2023-05-09 16:44:18,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +12: [2023-05-09 16:44:18,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +12: [2023-05-09 16:44:18,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +12: [2023-05-09 16:44:18,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 2: [2023-05-09 16:44:18,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +11: [2023-05-09 16:44:18,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 2: [2023-05-09 16:44:18,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 2: [2023-05-09 16:44:18,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 6: [2023-05-09 16:44:18,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +12: [2023-05-09 16:44:18,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +14: [2023-05-09 16:44:18,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 6: [2023-05-09 16:44:18,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 6: [2023-05-09 16:44:18,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +14: [2023-05-09 16:44:18,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +14: [2023-05-09 16:44:18,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +14: [2023-05-09 16:44:18,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +20: [2023-05-09 16:44:18,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +20: [2023-05-09 16:44:18,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +20: [2023-05-09 16:44:18,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +20: [2023-05-09 16:44:18,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 2: [2023-05-09 16:44:18,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +28: [2023-05-09 16:44:18,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +28: [2023-05-09 16:44:18,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +28: [2023-05-09 16:44:18,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +28: [2023-05-09 16:44:18,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +28: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +28: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +28: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +24: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +28: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +24: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. +28: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +28: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +28: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +28: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +28: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +28: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +28: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 7: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +28: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +26: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +26: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +24: [2023-05-09 16:44:18,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 7: [2023-05-09 16:44:18,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +24: [2023-05-09 16:44:18,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt. + 7: [2023-05-09 16:44:18,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 7: [2023-05-09 16:44:18,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 7: [2023-05-09 16:44:18,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +30: [2023-05-09 16:44:18,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +30: [2023-05-09 16:44:18,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +30: [2023-05-09 16:44:18,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +30: [2023-05-09 16:44:18,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +30: [2023-05-09 16:44:18,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +30: [2023-05-09 16:44:18,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +26: [2023-05-09 16:44:18,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +26: [2023-05-09 16:44:18,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 7: [2023-05-09 16:44:18,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +30: [2023-05-09 16:44:18,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +30: [2023-05-09 16:44:18,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +30: [2023-05-09 16:44:18,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +30: [2023-05-09 16:44:18,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +30: [2023-05-09 16:44:18,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +30: [2023-05-09 16:44:18,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +30: [2023-05-09 16:44:18,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +30: [2023-05-09 16:44:18,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 6: [2023-05-09 16:44:18,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +22: [2023-05-09 16:44:18,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +22: [2023-05-09 16:44:18,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +22: [2023-05-09 16:44:18,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +22: [2023-05-09 16:44:18,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +25: [2023-05-09 16:44:18,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +25: [2023-05-09 16:44:18,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +25: [2023-05-09 16:44:18,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +25: [2023-05-09 16:44:18,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +22: [2023-05-09 16:44:18,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +22: [2023-05-09 16:44:18,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +22: [2023-05-09 16:44:18,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +22: [2023-05-09 16:44:18,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +30: [2023-05-09 16:44:18,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +12: [2023-05-09 16:44:18,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +16: [2023-05-09 16:44:18,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +16: [2023-05-09 16:44:18,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 9: [2023-05-09 16:44:18,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +30: [2023-05-09 16:44:18,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +12: [2023-05-09 16:44:18,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 8: [2023-05-09 16:44:18,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +19: [2023-05-09 16:44:18,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +19: [2023-05-09 16:44:18,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +13: [2023-05-09 16:44:18,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +13: [2023-05-09 16:44:18,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +13: [2023-05-09 16:44:18,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +13: [2023-05-09 16:44:18,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +16: [2023-05-09 16:44:18,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +20: [2023-05-09 16:44:18,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +14: [2023-05-09 16:44:18,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +14: [2023-05-09 16:44:18,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +19: [2023-05-09 16:44:18,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +19: [2023-05-09 16:44:18,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +16: [2023-05-09 16:44:18,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +24: [2023-05-09 16:44:18,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +25: [2023-05-09 16:44:18,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +12: [2023-05-09 16:44:18,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +25: [2023-05-09 16:44:18,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +20: [2023-05-09 16:44:18,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +20: [2023-05-09 16:44:18,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +20: [2023-05-09 16:44:18,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +24: [2023-05-09 16:44:18,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +24: [2023-05-09 16:44:18,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +24: [2023-05-09 16:44:18,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +28: [2023-05-09 16:44:18,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +14: [2023-05-09 16:44:18,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +14: [2023-05-09 16:44:18,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +30: [2023-05-09 16:44:18,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 6: [2023-05-09 16:44:18,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 6: [2023-05-09 16:44:18,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +28: [2023-05-09 16:44:18,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +28: [2023-05-09 16:44:18,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 8: [2023-05-09 16:44:18,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +12: [2023-05-09 16:44:18,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +12: [2023-05-09 16:44:18,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +30: [2023-05-09 16:44:18,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +30: [2023-05-09 16:44:18,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +12: [2023-05-09 16:44:18,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +30: [2023-05-09 16:44:18,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +28: [2023-05-09 16:44:18,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 4: [2023-05-09 16:44:18,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 4: [2023-05-09 16:44:18,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 4: [2023-05-09 16:44:18,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +12: [2023-05-09 16:44:18,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +28: [2023-05-09 16:44:18,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +16: [2023-05-09 16:44:18,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +16: [2023-05-09 16:44:18,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +16: [2023-05-09 16:44:18,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +16: [2023-05-09 16:44:18,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +30: [2023-05-09 16:44:18,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +16: [2023-05-09 16:44:18,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +16: [2023-05-09 16:44:18,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +16: [2023-05-09 16:44:18,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +30: [2023-05-09 16:44:18,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +16: [2023-05-09 16:44:18,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +12: [2023-05-09 16:44:18,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +12: [2023-05-09 16:44:18,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +30: [2023-05-09 16:44:18,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +28: [2023-05-09 16:44:18,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +28: [2023-05-09 16:44:18,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +30: [2023-05-09 16:44:18,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +12: [2023-05-09 16:44:18,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +28: [2023-05-09 16:44:18,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 8: [2023-05-09 16:44:18,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +16: [2023-05-09 16:44:18,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +16: [2023-05-09 16:44:18,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +16: [2023-05-09 16:44:18,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +23: [2023-05-09 16:44:18,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +23: [2023-05-09 16:44:18,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +23: [2023-05-09 16:44:18,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +23: [2023-05-09 16:44:18,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +23: [2023-05-09 16:44:18,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +23: [2023-05-09 16:44:18,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +23: [2023-05-09 16:44:18,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +23: [2023-05-09 16:44:18,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +23: [2023-05-09 16:44:18,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +23: [2023-05-09 16:44:18,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +23: [2023-05-09 16:44:18,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +23: [2023-05-09 16:44:18,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +23: [2023-05-09 16:44:18,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 7: [2023-05-09 16:44:18,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +23: [2023-05-09 16:44:18,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 7: [2023-05-09 16:44:18,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +23: [2023-05-09 16:44:18,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +23: [2023-05-09 16:44:18,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +16: [2023-05-09 16:44:18,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +16: [2023-05-09 16:44:18,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +16: [2023-05-09 16:44:18,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +15: [2023-05-09 16:44:18,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +15: [2023-05-09 16:44:18,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +15: [2023-05-09 16:44:18,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +15: [2023-05-09 16:44:18,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +15: [2023-05-09 16:44:18,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +15: [2023-05-09 16:44:18,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +15: [2023-05-09 16:44:18,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +15: [2023-05-09 16:44:18,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +15: [2023-05-09 16:44:18,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 4: [2023-05-09 16:44:18,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +15: [2023-05-09 16:44:18,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +15: [2023-05-09 16:44:18,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +15: [2023-05-09 16:44:18,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +15: [2023-05-09 16:44:18,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +15: [2023-05-09 16:44:18,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +15: [2023-05-09 16:44:18,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 3: [2023-05-09 16:44:18,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 3: [2023-05-09 16:44:18,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 3: [2023-05-09 16:44:18,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +15: [2023-05-09 16:44:18,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 3: [2023-05-09 16:44:18,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 0: [2023-05-09 16:44:18,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +14: [2023-05-09 16:44:18,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +14: [2023-05-09 16:44:18,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +14: [2023-05-09 16:44:18,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +14: [2023-05-09 16:44:18,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +14: [2023-05-09 16:44:18,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +14: [2023-05-09 16:44:18,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 4: [2023-05-09 16:44:18,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +14: [2023-05-09 16:44:18,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +14: [2023-05-09 16:44:18,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +14: [2023-05-09 16:44:18,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +14: [2023-05-09 16:44:18,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +14: [2023-05-09 16:44:18,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +14: [2023-05-09 16:44:18,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 0: [2023-05-09 16:44:18,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 0: [2023-05-09 16:44:18,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +14: [2023-05-09 16:44:18,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +14: [2023-05-09 16:44:18,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +31: [2023-05-09 16:44:18,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +31: [2023-05-09 16:44:18,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +31: [2023-05-09 16:44:18,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +31: [2023-05-09 16:44:18,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +31: [2023-05-09 16:44:18,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +14: [2023-05-09 16:44:18,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +23: [2023-05-09 16:44:18,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +31: [2023-05-09 16:44:18,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +14: [2023-05-09 16:44:18,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 0: [2023-05-09 16:44:18,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +31: [2023-05-09 16:44:18,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +31: [2023-05-09 16:44:18,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +31: [2023-05-09 16:44:18,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +31: [2023-05-09 16:44:18,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +31: [2023-05-09 16:44:18,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +31: [2023-05-09 16:44:18,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +31: [2023-05-09 16:44:18,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +23: [2023-05-09 16:44:18,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +24: [2023-05-09 16:44:18,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +16: [2023-05-09 16:44:18,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +24: [2023-05-09 16:44:18,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +24: [2023-05-09 16:44:18,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +24: [2023-05-09 16:44:18,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +24: [2023-05-09 16:44:18,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +31: [2023-05-09 16:44:18,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +24: [2023-05-09 16:44:18,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +24: [2023-05-09 16:44:18,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +24: [2023-05-09 16:44:18,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +24: [2023-05-09 16:44:18,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +24: [2023-05-09 16:44:18,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +24: [2023-05-09 16:44:18,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +24: [2023-05-09 16:44:18,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +24: [2023-05-09 16:44:18,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +24: [2023-05-09 16:44:18,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +31: [2023-05-09 16:44:18,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +31: [2023-05-09 16:44:18,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +23: [2023-05-09 16:44:18,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +24: [2023-05-09 16:44:18,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +24: [2023-05-09 16:44:18,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +19: [2023-05-09 16:44:18,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +19: [2023-05-09 16:44:18,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +19: [2023-05-09 16:44:18,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +19: [2023-05-09 16:44:18,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +19: [2023-05-09 16:44:18,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +19: [2023-05-09 16:44:18,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +19: [2023-05-09 16:44:18,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +19: [2023-05-09 16:44:18,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +19: [2023-05-09 16:44:18,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +19: [2023-05-09 16:44:18,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +19: [2023-05-09 16:44:18,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +19: [2023-05-09 16:44:18,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +19: [2023-05-09 16:44:18,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +19: [2023-05-09 16:44:18,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 4: [2023-05-09 16:44:18,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +19: [2023-05-09 16:44:18,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +19: [2023-05-09 16:44:18,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +13: [2023-05-09 16:44:18,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +13: [2023-05-09 16:44:18,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +13: [2023-05-09 16:44:18,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +13: [2023-05-09 16:44:18,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +13: [2023-05-09 16:44:18,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +13: [2023-05-09 16:44:18,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +15: [2023-05-09 16:44:18,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +13: [2023-05-09 16:44:18,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +13: [2023-05-09 16:44:18,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +13: [2023-05-09 16:44:18,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +13: [2023-05-09 16:44:18,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +13: [2023-05-09 16:44:18,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +13: [2023-05-09 16:44:18,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +13: [2023-05-09 16:44:18,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +13: [2023-05-09 16:44:18,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 4: [2023-05-09 16:44:18,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +13: [2023-05-09 16:44:18,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +13: [2023-05-09 16:44:18,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +23: [2023-05-09 16:44:18,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +16: [2023-05-09 16:44:18,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +15: [2023-05-09 16:44:18,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +15: [2023-05-09 16:44:18,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +23: [2023-05-09 16:44:18,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +14: [2023-05-09 16:44:18,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +14: [2023-05-09 16:44:18,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 5: [2023-05-09 16:44:18,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 5: [2023-05-09 16:44:18,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 0: [2023-05-09 16:44:18,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +27: [2023-05-09 16:44:18,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +27: [2023-05-09 16:44:18,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +27: [2023-05-09 16:44:18,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +27: [2023-05-09 16:44:18,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +27: [2023-05-09 16:44:18,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +27: [2023-05-09 16:44:18,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +24: [2023-05-09 16:44:18,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +31: [2023-05-09 16:44:18,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +23: [2023-05-09 16:44:18,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +14: [2023-05-09 16:44:18,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +23: [2023-05-09 16:44:18,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +14: [2023-05-09 16:44:18,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +15: [2023-05-09 16:44:18,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +27: [2023-05-09 16:44:18,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +15: [2023-05-09 16:44:18,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +27: [2023-05-09 16:44:18,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +31: [2023-05-09 16:44:18,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 0: [2023-05-09 16:44:18,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 2: [2023-05-09 16:44:18,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 3: [2023-05-09 16:44:18,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +23: [2023-05-09 16:44:18,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... + 2: [2023-05-09 16:44:18,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +24: [2023-05-09 16:44:18,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +14: [2023-05-09 16:44:18,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +19: [2023-05-09 16:44:18,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +24: [2023-05-09 16:44:18,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +31: [2023-05-09 16:44:18,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +15: [2023-05-09 16:44:18,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +15: [2023-05-09 16:44:18,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +14: [2023-05-09 16:44:18,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +13: [2023-05-09 16:44:18,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +13: [2023-05-09 16:44:18,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 3: [2023-05-09 16:44:18,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +31: [2023-05-09 16:44:18,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +15: [2023-05-09 16:44:18,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +19: [2023-05-09 16:44:18,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +19: [2023-05-09 16:44:18,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +19: [2023-05-09 16:44:18,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +14: [2023-05-09 16:44:18,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +14: [2023-05-09 16:44:18,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +31: [2023-05-09 16:44:18,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +27: [2023-05-09 16:44:18,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +27: [2023-05-09 16:44:18,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +31: [2023-05-09 16:44:18,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 5: [2023-05-09 16:44:18,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +24: [2023-05-09 16:44:18,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 0: [2023-05-09 16:44:18,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +24: [2023-05-09 16:44:18,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +20: [2023-05-09 16:44:18,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +20: [2023-05-09 16:44:18,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +20: [2023-05-09 16:44:18,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 0: [2023-05-09 16:44:18,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +20: [2023-05-09 16:44:18,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +20: [2023-05-09 16:44:18,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +20: [2023-05-09 16:44:18,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +20: [2023-05-09 16:44:18,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +20: [2023-05-09 16:44:18,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +20: [2023-05-09 16:44:18,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +20: [2023-05-09 16:44:18,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +20: [2023-05-09 16:44:18,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... +20: [2023-05-09 16:44:18,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +20: [2023-05-09 16:44:18,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +20: [2023-05-09 16:44:18,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +20: [2023-05-09 16:44:18,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt... +19: [2023-05-09 16:44:18,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +20: [2023-05-09 16:44:18,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +13: [2023-05-09 16:44:18,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +31: [2023-05-09 16:44:18,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +13: [2023-05-09 16:44:18,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +19: [2023-05-09 16:44:18,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +31: [2023-05-09 16:44:18,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +24: [2023-05-09 16:44:18,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +27: [2023-05-09 16:44:18,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +27: [2023-05-09 16:44:18,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +24: [2023-05-09 16:44:18,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +19: [2023-05-09 16:44:18,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +19: [2023-05-09 16:44:18,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 5: [2023-05-09 16:44:18,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +13: [2023-05-09 16:44:18,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +13: [2023-05-09 16:44:18,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +20: [2023-05-09 16:44:18,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 2: [2023-05-09 16:44:18,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +20: [2023-05-09 16:44:18,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +27: [2023-05-09 16:44:18,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +20: [2023-05-09 16:44:18,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +27: [2023-05-09 16:44:18,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +24: [2023-05-09 16:44:18,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. +13: [2023-05-09 16:44:18,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +13: [2023-05-09 16:44:18,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +20: [2023-05-09 16:44:18,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +20: [2023-05-09 16:44:18,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +27: [2023-05-09 16:44:18,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +27: [2023-05-09 16:44:18,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +21: [2023-05-09 16:44:18,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +21: [2023-05-09 16:44:18,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +21: [2023-05-09 16:44:18,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +21: [2023-05-09 16:44:18,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +21: [2023-05-09 16:44:18,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +21: [2023-05-09 16:44:18,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +21: [2023-05-09 16:44:18,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +21: [2023-05-09 16:44:18,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 2: [2023-05-09 16:44:18,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +20: [2023-05-09 16:44:18,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +20: [2023-05-09 16:44:18,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +29: [2023-05-09 16:44:18,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +29: [2023-05-09 16:44:18,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +29: [2023-05-09 16:44:18,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +29: [2023-05-09 16:44:18,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +29: [2023-05-09 16:44:18,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +29: [2023-05-09 16:44:18,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +29: [2023-05-09 16:44:18,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 9: [2023-05-09 16:44:18,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 1: [2023-05-09 16:44:18,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 9: [2023-05-09 16:44:18,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +29: [2023-05-09 16:44:18,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 1: [2023-05-09 16:44:18,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +21: [2023-05-09 16:44:18,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +20: [2023-05-09 16:44:18,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +29: [2023-05-09 16:44:19,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +29: [2023-05-09 16:44:19,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +21: [2023-05-09 16:44:19,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +21: [2023-05-09 16:44:19,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +21: [2023-05-09 16:44:19,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +29: [2023-05-09 16:44:19,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +29: [2023-05-09 16:44:19,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +29: [2023-05-09 16:44:19,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +29: [2023-05-09 16:44:19,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +21: [2023-05-09 16:44:19,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +29: [2023-05-09 16:44:19,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +11: [2023-05-09 16:44:19,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +11: [2023-05-09 16:44:19,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +11: [2023-05-09 16:44:19,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +11: [2023-05-09 16:44:19,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +11: [2023-05-09 16:44:19,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +11: [2023-05-09 16:44:19,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +11: [2023-05-09 16:44:19,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +11: [2023-05-09 16:44:19,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +18: [2023-05-09 16:44:19,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +18: [2023-05-09 16:44:19,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +18: [2023-05-09 16:44:19,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +18: [2023-05-09 16:44:19,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +21: [2023-05-09 16:44:19,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +21: [2023-05-09 16:44:19,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +29: [2023-05-09 16:44:19,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +21: [2023-05-09 16:44:19,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +29: [2023-05-09 16:44:19,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +29: [2023-05-09 16:44:19,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +29: [2023-05-09 16:44:19,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +10: [2023-05-09 16:44:19,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +10: [2023-05-09 16:44:19,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +10: [2023-05-09 16:44:19,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +10: [2023-05-09 16:44:19,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +29: [2023-05-09 16:44:19,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +29: [2023-05-09 16:44:19,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +29: [2023-05-09 16:44:19,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +18: [2023-05-09 16:44:19,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +27: [2023-05-09 16:44:19,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +27: [2023-05-09 16:44:19,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +27: [2023-05-09 16:44:19,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +27: [2023-05-09 16:44:19,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +18: [2023-05-09 16:44:19,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +18: [2023-05-09 16:44:19,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +18: [2023-05-09 16:44:19,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +10: [2023-05-09 16:44:19,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +29: [2023-05-09 16:44:19,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +10: [2023-05-09 16:44:19,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +29: [2023-05-09 16:44:19,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +10: [2023-05-09 16:44:19,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +29: [2023-05-09 16:44:19,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +10: [2023-05-09 16:44:19,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +29: [2023-05-09 16:44:19,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +29: [2023-05-09 16:44:19,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +27: [2023-05-09 16:44:19,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +27: [2023-05-09 16:44:19,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +11: [2023-05-09 16:44:19,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +11: [2023-05-09 16:44:19,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +11: [2023-05-09 16:44:19,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +27: [2023-05-09 16:44:19,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +11: [2023-05-09 16:44:19,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +29: [2023-05-09 16:44:19,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +29: [2023-05-09 16:44:19,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +29: [2023-05-09 16:44:19,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +29: [2023-05-09 16:44:19,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +27: [2023-05-09 16:44:19,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +29: [2023-05-09 16:44:19,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +26: [2023-05-09 16:44:19,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +26: [2023-05-09 16:44:19,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +17: [2023-05-09 16:44:19,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +17: [2023-05-09 16:44:19,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +26: [2023-05-09 16:44:19,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +17: [2023-05-09 16:44:19,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +17: [2023-05-09 16:44:19,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +26: [2023-05-09 16:44:19,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +11: [2023-05-09 16:44:19,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +11: [2023-05-09 16:44:19,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +11: [2023-05-09 16:44:19,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +27: [2023-05-09 16:44:19,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +27: [2023-05-09 16:44:19,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +11: [2023-05-09 16:44:19,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +18: [2023-05-09 16:44:19,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +18: [2023-05-09 16:44:19,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +18: [2023-05-09 16:44:19,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +18: [2023-05-09 16:44:19,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +18: [2023-05-09 16:44:19,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +18: [2023-05-09 16:44:19,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +18: [2023-05-09 16:44:19,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +18: [2023-05-09 16:44:19,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +18: [2023-05-09 16:44:19,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +18: [2023-05-09 16:44:19,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +18: [2023-05-09 16:44:19,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +18: [2023-05-09 16:44:19,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +18: [2023-05-09 16:44:19,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +18: [2023-05-09 16:44:19,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +17: [2023-05-09 16:44:19,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +18: [2023-05-09 16:44:19,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +18: [2023-05-09 16:44:19,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +26: [2023-05-09 16:44:19,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +26: [2023-05-09 16:44:19,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +26: [2023-05-09 16:44:19,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +26: [2023-05-09 16:44:19,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +27: [2023-05-09 16:44:19,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +17: [2023-05-09 16:44:19,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +17: [2023-05-09 16:44:19,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +17: [2023-05-09 16:44:19,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +27: [2023-05-09 16:44:19,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +27: [2023-05-09 16:44:19,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +27: [2023-05-09 16:44:19,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +27: [2023-05-09 16:44:19,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +27: [2023-05-09 16:44:19,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +17: [2023-05-09 16:44:19,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +17: [2023-05-09 16:44:19,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +17: [2023-05-09 16:44:19,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +17: [2023-05-09 16:44:19,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +17: [2023-05-09 16:44:19,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +17: [2023-05-09 16:44:19,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +18: [2023-05-09 16:44:19,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +17: [2023-05-09 16:44:19,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +17: [2023-05-09 16:44:19,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +17: [2023-05-09 16:44:19,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +17: [2023-05-09 16:44:19,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +18: [2023-05-09 16:44:19,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +21: [2023-05-09 16:44:19,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +21: [2023-05-09 16:44:19,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +21: [2023-05-09 16:44:19,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +21: [2023-05-09 16:44:19,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +18: [2023-05-09 16:44:19,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +17: [2023-05-09 16:44:19,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +18: [2023-05-09 16:44:19,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +18: [2023-05-09 16:44:19,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +21: [2023-05-09 16:44:19,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +21: [2023-05-09 16:44:19,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +21: [2023-05-09 16:44:19,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +21: [2023-05-09 16:44:19,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +25: [2023-05-09 16:44:19,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +25: [2023-05-09 16:44:19,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +17: [2023-05-09 16:44:19,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +25: [2023-05-09 16:44:19,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +25: [2023-05-09 16:44:19,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +25: [2023-05-09 16:44:19,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +18: [2023-05-09 16:44:19,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +25: [2023-05-09 16:44:19,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +18: [2023-05-09 16:44:19,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +17: [2023-05-09 16:44:19,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +25: [2023-05-09 16:44:19,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +25: [2023-05-09 16:44:19,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +17: [2023-05-09 16:44:19,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +17: [2023-05-09 16:44:19,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +17: [2023-05-09 16:44:19,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +17: [2023-05-09 16:44:19,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +18: [2023-05-09 16:44:19,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +28: [2023-05-09 16:44:19,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +17: [2023-05-09 16:44:19,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +28: [2023-05-09 16:44:19,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +28: [2023-05-09 16:44:19,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +28: [2023-05-09 16:44:19,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +28: [2023-05-09 16:44:19,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +28: [2023-05-09 16:44:19,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +21: [2023-05-09 16:44:19,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +17: [2023-05-09 16:44:19,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +25: [2023-05-09 16:44:19,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +21: [2023-05-09 16:44:19,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +28: [2023-05-09 16:44:19,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +28: [2023-05-09 16:44:19,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +26: [2023-05-09 16:44:19,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +26: [2023-05-09 16:44:19,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +26: [2023-05-09 16:44:19,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +26: [2023-05-09 16:44:19,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +26: [2023-05-09 16:44:19,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +26: [2023-05-09 16:44:19,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +26: [2023-05-09 16:44:19,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +26: [2023-05-09 16:44:19,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +26: [2023-05-09 16:44:19,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +17: [2023-05-09 16:44:19,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +26: [2023-05-09 16:44:19,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +26: [2023-05-09 16:44:19,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +26: [2023-05-09 16:44:19,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +26: [2023-05-09 16:44:19,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +26: [2023-05-09 16:44:19,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +26: [2023-05-09 16:44:19,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +17: [2023-05-09 16:44:19,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +26: [2023-05-09 16:44:19,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +17: [2023-05-09 16:44:19,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +17: [2023-05-09 16:44:19,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +17: [2023-05-09 16:44:19,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +21: [2023-05-09 16:44:19,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +21: [2023-05-09 16:44:19,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +25: [2023-05-09 16:44:19,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +25: [2023-05-09 16:44:19,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +28: [2023-05-09 16:44:19,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +21: [2023-05-09 16:44:19,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +28: [2023-05-09 16:44:19,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +21: [2023-05-09 16:44:19,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +21: [2023-05-09 16:44:19,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 6: [2023-05-09 16:44:19,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +21: [2023-05-09 16:44:19,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 1: [2023-05-09 16:44:19,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 1: [2023-05-09 16:44:19,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 1: [2023-05-09 16:44:19,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 1: [2023-05-09 16:44:19,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +22: [2023-05-09 16:44:19,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +22: [2023-05-09 16:44:19,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +22: [2023-05-09 16:44:19,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +22: [2023-05-09 16:44:19,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +22: [2023-05-09 16:44:19,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +22: [2023-05-09 16:44:19,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +22: [2023-05-09 16:44:19,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +25: [2023-05-09 16:44:19,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +26: [2023-05-09 16:44:19,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +26: [2023-05-09 16:44:19,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +26: [2023-05-09 16:44:19,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +22: [2023-05-09 16:44:19,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +30: [2023-05-09 16:44:19,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +30: [2023-05-09 16:44:19,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +30: [2023-05-09 16:44:19,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +12: [2023-05-09 16:44:19,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +30: [2023-05-09 16:44:19,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +28: [2023-05-09 16:44:19,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +30: [2023-05-09 16:44:19,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +10: [2023-05-09 16:44:19,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +10: [2023-05-09 16:44:19,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +12: [2023-05-09 16:44:19,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +12: [2023-05-09 16:44:19,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +30: [2023-05-09 16:44:19,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +10: [2023-05-09 16:44:19,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +12: [2023-05-09 16:44:19,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +10: [2023-05-09 16:44:19,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +12: [2023-05-09 16:44:19,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +10: [2023-05-09 16:44:19,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +10: [2023-05-09 16:44:19,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +10: [2023-05-09 16:44:19,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +10: [2023-05-09 16:44:19,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +12: [2023-05-09 16:44:19,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +10: [2023-05-09 16:44:19,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +10: [2023-05-09 16:44:19,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +10: [2023-05-09 16:44:19,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +10: [2023-05-09 16:44:19,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +12: [2023-05-09 16:44:19,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +30: [2023-05-09 16:44:19,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +10: [2023-05-09 16:44:19,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +12: [2023-05-09 16:44:19,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +10: [2023-05-09 16:44:19,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +25: [2023-05-09 16:44:19,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +25: [2023-05-09 16:44:19,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +10: [2023-05-09 16:44:19,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +30: [2023-05-09 16:44:19,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +10: [2023-05-09 16:44:19,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +28: [2023-05-09 16:44:19,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +26: [2023-05-09 16:44:19,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +26: [2023-05-09 16:44:19,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +22: [2023-05-09 16:44:19,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +22: [2023-05-09 16:44:19,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +26: [2023-05-09 16:44:19,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +26: [2023-05-09 16:44:19,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +22: [2023-05-09 16:44:19,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +25: [2023-05-09 16:44:19,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +25: [2023-05-09 16:44:19,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +30: [2023-05-09 16:44:19,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +28: [2023-05-09 16:44:19,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +28: [2023-05-09 16:44:19,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +30: [2023-05-09 16:44:19,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +26: [2023-05-09 16:44:19,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +22: [2023-05-09 16:44:19,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +15: [2023-05-09 16:44:19,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +15: [2023-05-09 16:44:19,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +22: [2023-05-09 16:44:19,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +10: [2023-05-09 16:44:19,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +12: [2023-05-09 16:44:19,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +12: [2023-05-09 16:44:19,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +10: [2023-05-09 16:44:19,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +15: [2023-05-09 16:44:19,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +15: [2023-05-09 16:44:19,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +28: [2023-05-09 16:44:19,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +22: [2023-05-09 16:44:19,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +28: [2023-05-09 16:44:19,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +30: [2023-05-09 16:44:19,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +15: [2023-05-09 16:44:19,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +15: [2023-05-09 16:44:19,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +24: [2023-05-09 16:44:19,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +24: [2023-05-09 16:44:19,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +24: [2023-05-09 16:44:19,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +24: [2023-05-09 16:44:19,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +24: [2023-05-09 16:44:19,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +24: [2023-05-09 16:44:19,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +24: [2023-05-09 16:44:19,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +24: [2023-05-09 16:44:19,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +30: [2023-05-09 16:44:19,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +12: [2023-05-09 16:44:19,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +15: [2023-05-09 16:44:19,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +15: [2023-05-09 16:44:19,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +30: [2023-05-09 16:44:19,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +16: [2023-05-09 16:44:19,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +16: [2023-05-09 16:44:19,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +10: [2023-05-09 16:44:19,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +12: [2023-05-09 16:44:19,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +16: [2023-05-09 16:44:19,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +16: [2023-05-09 16:44:19,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +16: [2023-05-09 16:44:19,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +16: [2023-05-09 16:44:19,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +12: [2023-05-09 16:44:19,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +16: [2023-05-09 16:44:19,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +16: [2023-05-09 16:44:19,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +15: [2023-05-09 16:44:19,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +10: [2023-05-09 16:44:19,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +22: [2023-05-09 16:44:19,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +10: [2023-05-09 16:44:19,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +12: [2023-05-09 16:44:19,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 1: [2023-05-09 16:44:19,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 1: [2023-05-09 16:44:19,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +12: [2023-05-09 16:44:19,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +30: [2023-05-09 16:44:19,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +24: [2023-05-09 16:44:19,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +30: [2023-05-09 16:44:19,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +10: [2023-05-09 16:44:19,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +10: [2023-05-09 16:44:19,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +22: [2023-05-09 16:44:19,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +16: [2023-05-09 16:44:19,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +15: [2023-05-09 16:44:19,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +16: [2023-05-09 16:44:19,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +16: [2023-05-09 16:44:19,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +16: [2023-05-09 16:44:19,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +12: [2023-05-09 16:44:19,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +15: [2023-05-09 16:44:19,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +24: [2023-05-09 16:44:19,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +24: [2023-05-09 16:44:19,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +24: [2023-05-09 16:44:19,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +24: [2023-05-09 16:44:19,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +30: [2023-05-09 16:44:19,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +10: [2023-05-09 16:44:19,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +23: [2023-05-09 16:44:19,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +23: [2023-05-09 16:44:19,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +23: [2023-05-09 16:44:19,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +23: [2023-05-09 16:44:19,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +23: [2023-05-09 16:44:19,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +23: [2023-05-09 16:44:19,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +23: [2023-05-09 16:44:19,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +23: [2023-05-09 16:44:19,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +15: [2023-05-09 16:44:19,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +16: [2023-05-09 16:44:19,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +16: [2023-05-09 16:44:19,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +15: [2023-05-09 16:44:19,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +16: [2023-05-09 16:44:19,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +16: [2023-05-09 16:44:19,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +24: [2023-05-09 16:44:19,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +24: [2023-05-09 16:44:19,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +24: [2023-05-09 16:44:19,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +15: [2023-05-09 16:44:19,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +15: [2023-05-09 16:44:19,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +23: [2023-05-09 16:44:19,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +23: [2023-05-09 16:44:19,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +23: [2023-05-09 16:44:19,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +23: [2023-05-09 16:44:19,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +29: [2023-05-09 16:44:19,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +29: [2023-05-09 16:44:19,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +29: [2023-05-09 16:44:19,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +29: [2023-05-09 16:44:19,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +20: [2023-05-09 16:44:19,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +20: [2023-05-09 16:44:19,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +20: [2023-05-09 16:44:19,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +20: [2023-05-09 16:44:19,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +20: [2023-05-09 16:44:19,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +20: [2023-05-09 16:44:19,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +20: [2023-05-09 16:44:19,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +20: [2023-05-09 16:44:19,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +29: [2023-05-09 16:44:19,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +29: [2023-05-09 16:44:19,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +15: [2023-05-09 16:44:19,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 6: [2023-05-09 16:44:19,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 6: [2023-05-09 16:44:19,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +29: [2023-05-09 16:44:19,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +29: [2023-05-09 16:44:19,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +23: [2023-05-09 16:44:19,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +23: [2023-05-09 16:44:19,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +23: [2023-05-09 16:44:19,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +23: [2023-05-09 16:44:19,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +13: [2023-05-09 16:44:19,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +13: [2023-05-09 16:44:19,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +13: [2023-05-09 16:44:19,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +13: [2023-05-09 16:44:19,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +13: [2023-05-09 16:44:19,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +13: [2023-05-09 16:44:19,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +13: [2023-05-09 16:44:19,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +14: [2023-05-09 16:44:19,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +20: [2023-05-09 16:44:19,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +14: [2023-05-09 16:44:19,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +20: [2023-05-09 16:44:19,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +13: [2023-05-09 16:44:19,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +14: [2023-05-09 16:44:19,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +14: [2023-05-09 16:44:19,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +14: [2023-05-09 16:44:19,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +14: [2023-05-09 16:44:19,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +20: [2023-05-09 16:44:19,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +20: [2023-05-09 16:44:19,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +12: [2023-05-09 16:44:19,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 5: [2023-05-09 16:44:19,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +14: [2023-05-09 16:44:19,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +12: [2023-05-09 16:44:19,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +12: [2023-05-09 16:44:19,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 5: [2023-05-09 16:44:19,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +12: [2023-05-09 16:44:19,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 5: [2023-05-09 16:44:19,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +14: [2023-05-09 16:44:19,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +31: [2023-05-09 16:44:19,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +31: [2023-05-09 16:44:19,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +31: [2023-05-09 16:44:19,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +31: [2023-05-09 16:44:19,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +29: [2023-05-09 16:44:19,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +29: [2023-05-09 16:44:19,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +31: [2023-05-09 16:44:19,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +31: [2023-05-09 16:44:19,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +31: [2023-05-09 16:44:19,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +11: [2023-05-09 16:44:19,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +11: [2023-05-09 16:44:19,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +11: [2023-05-09 16:44:19,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +11: [2023-05-09 16:44:19,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +11: [2023-05-09 16:44:19,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +11: [2023-05-09 16:44:19,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +31: [2023-05-09 16:44:19,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +11: [2023-05-09 16:44:19,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +11: [2023-05-09 16:44:19,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +29: [2023-05-09 16:44:19,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +20: [2023-05-09 16:44:19,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +20: [2023-05-09 16:44:19,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +13: [2023-05-09 16:44:19,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +20: [2023-05-09 16:44:19,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +12: [2023-05-09 16:44:19,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +12: [2023-05-09 16:44:19,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +12: [2023-05-09 16:44:19,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +12: [2023-05-09 16:44:19,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +20: [2023-05-09 16:44:19,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 6: [2023-05-09 16:44:19,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 6: [2023-05-09 16:44:19,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 1: [2023-05-09 16:44:19,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +29: [2023-05-09 16:44:19,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +29: [2023-05-09 16:44:19,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +13: [2023-05-09 16:44:19,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +13: [2023-05-09 16:44:19,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +31: [2023-05-09 16:44:19,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +19: [2023-05-09 16:44:19,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +19: [2023-05-09 16:44:19,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +19: [2023-05-09 16:44:19,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +19: [2023-05-09 16:44:19,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +19: [2023-05-09 16:44:19,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +29: [2023-05-09 16:44:19,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +29: [2023-05-09 16:44:19,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +19: [2023-05-09 16:44:19,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +19: [2023-05-09 16:44:19,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +19: [2023-05-09 16:44:19,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +31: [2023-05-09 16:44:19,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +31: [2023-05-09 16:44:19,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +13: [2023-05-09 16:44:19,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +14: [2023-05-09 16:44:19,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +12: [2023-05-09 16:44:19,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +12: [2023-05-09 16:44:19,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 5: [2023-05-09 16:44:19,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +31: [2023-05-09 16:44:19,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +14: [2023-05-09 16:44:19,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +12: [2023-05-09 16:44:19,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +12: [2023-05-09 16:44:19,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +29: [2023-05-09 16:44:19,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +14: [2023-05-09 16:44:19,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +11: [2023-05-09 16:44:19,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +11: [2023-05-09 16:44:19,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +11: [2023-05-09 16:44:19,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +12: [2023-05-09 16:44:19,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +12: [2023-05-09 16:44:19,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +12: [2023-05-09 16:44:19,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +12: [2023-05-09 16:44:19,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +13: [2023-05-09 16:44:19,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +13: [2023-05-09 16:44:19,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +31: [2023-05-09 16:44:19,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +31: [2023-05-09 16:44:19,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +31: [2023-05-09 16:44:19,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +19: [2023-05-09 16:44:19,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +14: [2023-05-09 16:44:19,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +13: [2023-05-09 16:44:19,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +14: [2023-05-09 16:44:19,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +19: [2023-05-09 16:44:19,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +19: [2023-05-09 16:44:19,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +14: [2023-05-09 16:44:19,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +14: [2023-05-09 16:44:19,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +11: [2023-05-09 16:44:19,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +17: [2023-05-09 16:44:19,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +27: [2023-05-09 16:44:19,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +27: [2023-05-09 16:44:19,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +27: [2023-05-09 16:44:19,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +27: [2023-05-09 16:44:19,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +27: [2023-05-09 16:44:19,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +18: [2023-05-09 16:44:19,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +27: [2023-05-09 16:44:19,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +31: [2023-05-09 16:44:19,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +17: [2023-05-09 16:44:19,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +17: [2023-05-09 16:44:19,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +18: [2023-05-09 16:44:19,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +18: [2023-05-09 16:44:19,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +17: [2023-05-09 16:44:19,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +27: [2023-05-09 16:44:19,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +17: [2023-05-09 16:44:19,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +18: [2023-05-09 16:44:19,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +18: [2023-05-09 16:44:19,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +27: [2023-05-09 16:44:19,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +17: [2023-05-09 16:44:19,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +17: [2023-05-09 16:44:19,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +18: [2023-05-09 16:44:19,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +11: [2023-05-09 16:44:19,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +18: [2023-05-09 16:44:19,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +17: [2023-05-09 16:44:19,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +11: [2023-05-09 16:44:19,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +18: [2023-05-09 16:44:19,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +19: [2023-05-09 16:44:19,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +13: [2023-05-09 16:44:19,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +19: [2023-05-09 16:44:19,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +19: [2023-05-09 16:44:19,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +11: [2023-05-09 16:44:19,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +14: [2023-05-09 16:44:19,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +14: [2023-05-09 16:44:19,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +14: [2023-05-09 16:44:19,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +14: [2023-05-09 16:44:19,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +29: [2023-05-09 16:44:19,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +14: [2023-05-09 16:44:19,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +29: [2023-05-09 16:44:19,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +29: [2023-05-09 16:44:19,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +29: [2023-05-09 16:44:19,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +17: [2023-05-09 16:44:19,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +19: [2023-05-09 16:44:19,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +18: [2023-05-09 16:44:19,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +27: [2023-05-09 16:44:19,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +27: [2023-05-09 16:44:19,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +27: [2023-05-09 16:44:19,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +17: [2023-05-09 16:44:19,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +17: [2023-05-09 16:44:19,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +18: [2023-05-09 16:44:19,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +11: [2023-05-09 16:44:19,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +14: [2023-05-09 16:44:19,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +14: [2023-05-09 16:44:19,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +14: [2023-05-09 16:44:19,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +29: [2023-05-09 16:44:19,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +29: [2023-05-09 16:44:19,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +27: [2023-05-09 16:44:19,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +29: [2023-05-09 16:44:19,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +19: [2023-05-09 16:44:19,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +29: [2023-05-09 16:44:19,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +18: [2023-05-09 16:44:19,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +14: [2023-05-09 16:44:19,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +17: [2023-05-09 16:44:19,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +27: [2023-05-09 16:44:19,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +27: [2023-05-09 16:44:19,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +27: [2023-05-09 16:44:19,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +17: [2023-05-09 16:44:19,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +17: [2023-05-09 16:44:19,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +18: [2023-05-09 16:44:19,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +18: [2023-05-09 16:44:19,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +17: [2023-05-09 16:44:19,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +18: [2023-05-09 16:44:19,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +18: [2023-05-09 16:44:19,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +18: [2023-05-09 16:44:19,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +18: [2023-05-09 16:44:19,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +27: [2023-05-09 16:44:19,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +20: [2023-05-09 16:44:19,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +20: [2023-05-09 16:44:19,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +20: [2023-05-09 16:44:19,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +20: [2023-05-09 16:44:19,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +18: [2023-05-09 16:44:19,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +11: [2023-05-09 16:44:19,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +11: [2023-05-09 16:44:19,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +11: [2023-05-09 16:44:19,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +11: [2023-05-09 16:44:19,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +29: [2023-05-09 16:44:19,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +29: [2023-05-09 16:44:19,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +29: [2023-05-09 16:44:19,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +29: [2023-05-09 16:44:19,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +29: [2023-05-09 16:44:19,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +29: [2023-05-09 16:44:19,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +17: [2023-05-09 16:44:19,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 3: [2023-05-09 16:44:19,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 9: [2023-05-09 16:44:19,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 9: [2023-05-09 16:44:19,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 9: [2023-05-09 16:44:19,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 9: [2023-05-09 16:44:19,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +14: [2023-05-09 16:44:19,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 8: [2023-05-09 16:44:19,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 8: [2023-05-09 16:44:19,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 8: [2023-05-09 16:44:19,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +19: [2023-05-09 16:44:19,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +19: [2023-05-09 16:44:19,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +19: [2023-05-09 16:44:19,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +19: [2023-05-09 16:44:19,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +20: [2023-05-09 16:44:19,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +20: [2023-05-09 16:44:19,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +18: [2023-05-09 16:44:19,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +14: [2023-05-09 16:44:19,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +18: [2023-05-09 16:44:19,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +20: [2023-05-09 16:44:19,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +20: [2023-05-09 16:44:19,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +31: [2023-05-09 16:44:19,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 2: [2023-05-09 16:44:19,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +31: [2023-05-09 16:44:19,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +31: [2023-05-09 16:44:19,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 2: [2023-05-09 16:44:19,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 2: [2023-05-09 16:44:19,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 2: [2023-05-09 16:44:19,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +31: [2023-05-09 16:44:19,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +14: [2023-05-09 16:44:19,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +11: [2023-05-09 16:44:19,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +14: [2023-05-09 16:44:19,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +15: [2023-05-09 16:44:19,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +15: [2023-05-09 16:44:19,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +15: [2023-05-09 16:44:19,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +23: [2023-05-09 16:44:19,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +23: [2023-05-09 16:44:19,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +29: [2023-05-09 16:44:19,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +29: [2023-05-09 16:44:19,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +11: [2023-05-09 16:44:19,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +23: [2023-05-09 16:44:19,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +23: [2023-05-09 16:44:19,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +27: [2023-05-09 16:44:19,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +18: [2023-05-09 16:44:19,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +27: [2023-05-09 16:44:19,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +18: [2023-05-09 16:44:19,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +18: [2023-05-09 16:44:19,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +18: [2023-05-09 16:44:19,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +27: [2023-05-09 16:44:19,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +11: [2023-05-09 16:44:19,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +21: [2023-05-09 16:44:19,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +21: [2023-05-09 16:44:19,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +21: [2023-05-09 16:44:19,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +21: [2023-05-09 16:44:19,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +21: [2023-05-09 16:44:19,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +21: [2023-05-09 16:44:19,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +21: [2023-05-09 16:44:19,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +27: [2023-05-09 16:44:19,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +28: [2023-05-09 16:44:19,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 0: [2023-05-09 16:44:19,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 0: [2023-05-09 16:44:19,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +15: [2023-05-09 16:44:19,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +28: [2023-05-09 16:44:19,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +11: [2023-05-09 16:44:19,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +30: [2023-05-09 16:44:19,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +30: [2023-05-09 16:44:19,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +30: [2023-05-09 16:44:19,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +28: [2023-05-09 16:44:19,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +28: [2023-05-09 16:44:19,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +21: [2023-05-09 16:44:19,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +22: [2023-05-09 16:44:19,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +22: [2023-05-09 16:44:19,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +22: [2023-05-09 16:44:19,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +14: [2023-05-09 16:44:19,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +22: [2023-05-09 16:44:19,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +14: [2023-05-09 16:44:19,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 9: [2023-05-09 16:44:19,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 4: [2023-05-09 16:44:19,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 4: [2023-05-09 16:44:19,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 0: [2023-05-09 16:44:19,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 0: [2023-05-09 16:44:19,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 3: [2023-05-09 16:44:19,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 7: [2023-05-09 16:44:19,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +25: [2023-05-09 16:44:19,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 9: [2023-05-09 16:44:19,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +25: [2023-05-09 16:44:19,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +25: [2023-05-09 16:44:19,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +24: [2023-05-09 16:44:19,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +25: [2023-05-09 16:44:19,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +24: [2023-05-09 16:44:19,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +24: [2023-05-09 16:44:19,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 5: [2023-05-09 16:44:19,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 5: [2023-05-09 16:44:19,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 5: [2023-05-09 16:44:19,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 5: [2023-05-09 16:44:19,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +13: [2023-05-09 16:44:19,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +13: [2023-05-09 16:44:19,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +13: [2023-05-09 16:44:19,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 3: [2023-05-09 16:44:19,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +19: [2023-05-09 16:44:19,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +16: [2023-05-09 16:44:19,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +30: [2023-05-09 16:44:19,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +19: [2023-05-09 16:44:19,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +16: [2023-05-09 16:44:19,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 7: [2023-05-09 16:44:19,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +19: [2023-05-09 16:44:19,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +16: [2023-05-09 16:44:19,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +31: [2023-05-09 16:44:19,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +19: [2023-05-09 16:44:19,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +16: [2023-05-09 16:44:19,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 2: [2023-05-09 16:44:19,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +13: [2023-05-09 16:44:19,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. +31: [2023-05-09 16:44:19,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +24: [2023-05-09 16:44:19,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt. + 8: [2023-05-09 16:44:19,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +27: [2023-05-09 16:44:19,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +31: [2023-05-09 16:44:19,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +31: [2023-05-09 16:44:19,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +11: [2023-05-09 16:44:19,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +27: [2023-05-09 16:44:19,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +11: [2023-05-09 16:44:19,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +27: [2023-05-09 16:44:19,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +18: [2023-05-09 16:44:19,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +23: [2023-05-09 16:44:19,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +23: [2023-05-09 16:44:19,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +23: [2023-05-09 16:44:19,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +27: [2023-05-09 16:44:19,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +28: [2023-05-09 16:44:19,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +14: [2023-05-09 16:44:19,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +18: [2023-05-09 16:44:19,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +30: [2023-05-09 16:44:19,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +28: [2023-05-09 16:44:19,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +30: [2023-05-09 16:44:19,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +22: [2023-05-09 16:44:19,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +15: [2023-05-09 16:44:19,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +14: [2023-05-09 16:44:19,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +30: [2023-05-09 16:44:19,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +15: [2023-05-09 16:44:19,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +23: [2023-05-09 16:44:19,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +21: [2023-05-09 16:44:19,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +25: [2023-05-09 16:44:19,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +18: [2023-05-09 16:44:19,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +18: [2023-05-09 16:44:19,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +20: [2023-05-09 16:44:19,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +20: [2023-05-09 16:44:19,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +11: [2023-05-09 16:44:19,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +25: [2023-05-09 16:44:19,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +11: [2023-05-09 16:44:19,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +20: [2023-05-09 16:44:19,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +20: [2023-05-09 16:44:19,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +28: [2023-05-09 16:44:19,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +28: [2023-05-09 16:44:19,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +22: [2023-05-09 16:44:19,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +22: [2023-05-09 16:44:19,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +15: [2023-05-09 16:44:19,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +15: [2023-05-09 16:44:19,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +25: [2023-05-09 16:44:19,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +22: [2023-05-09 16:44:19,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 4: [2023-05-09 16:44:19,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +30: [2023-05-09 16:44:19,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +24: [2023-05-09 16:44:19,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +16: [2023-05-09 16:44:19,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +24: [2023-05-09 16:44:19,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +20: [2023-05-09 16:44:19,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +24: [2023-05-09 16:44:19,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 5: [2023-05-09 16:44:19,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +16: [2023-05-09 16:44:19,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +20: [2023-05-09 16:44:19,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +25: [2023-05-09 16:44:19,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +24: [2023-05-09 16:44:19,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +25: [2023-05-09 16:44:19,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +25: [2023-05-09 16:44:19,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +13: [2023-05-09 16:44:19,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +13: [2023-05-09 16:44:19,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +20: [2023-05-09 16:44:19,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 4: [2023-05-09 16:44:19,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +13: [2023-05-09 16:44:19,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +13: [2023-05-09 16:44:19,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +20: [2023-05-09 16:44:19,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +25: [2023-05-09 16:44:19,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +25: [2023-05-09 16:44:19,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +25: [2023-05-09 16:44:19,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +25: [2023-05-09 16:44:19,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +25: [2023-05-09 16:44:19,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +25: [2023-05-09 16:44:19,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +16: [2023-05-09 16:44:19,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... +21: [2023-05-09 16:44:19,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +21: [2023-05-09 16:44:19,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +16: [2023-05-09 16:44:19,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 3: [2023-05-09 16:44:19,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +19: [2023-05-09 16:44:19,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +19: [2023-05-09 16:44:19,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +23: [2023-05-09 16:44:19,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +11: [2023-05-09 16:44:19,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +19: [2023-05-09 16:44:19,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +19: [2023-05-09 16:44:19,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +23: [2023-05-09 16:44:19,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 2: [2023-05-09 16:44:19,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +18: [2023-05-09 16:44:19,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 3: [2023-05-09 16:44:19,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 9: [2023-05-09 16:44:19,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +27: [2023-05-09 16:44:19,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +27: [2023-05-09 16:44:19,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +11: [2023-05-09 16:44:19,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +21: [2023-05-09 16:44:19,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +28: [2023-05-09 16:44:19,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +28: [2023-05-09 16:44:19,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +28: [2023-05-09 16:44:19,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +28: [2023-05-09 16:44:19,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +30: [2023-05-09 16:44:19,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +28: [2023-05-09 16:44:19,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +28: [2023-05-09 16:44:19,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +30: [2023-05-09 16:44:19,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +23: [2023-05-09 16:44:19,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +23: [2023-05-09 16:44:19,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +26: [2023-05-09 16:44:19,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +26: [2023-05-09 16:44:19,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +18: [2023-05-09 16:44:19,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +26: [2023-05-09 16:44:19,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +26: [2023-05-09 16:44:19,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +26: [2023-05-09 16:44:19,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +28: [2023-05-09 16:44:19,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +18: [2023-05-09 16:44:19,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +28: [2023-05-09 16:44:19,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 2: [2023-05-09 16:44:19,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +25: [2023-05-09 16:44:19,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +26: [2023-05-09 16:44:19,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +26: [2023-05-09 16:44:19,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +22: [2023-05-09 16:44:19,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +26: [2023-05-09 16:44:19,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +25: [2023-05-09 16:44:19,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +27: [2023-05-09 16:44:19,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +27: [2023-05-09 16:44:19,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +27: [2023-05-09 16:44:19,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +27: [2023-05-09 16:44:19,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +22: [2023-05-09 16:44:19,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +27: [2023-05-09 16:44:19,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +27: [2023-05-09 16:44:19,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +18: [2023-05-09 16:44:19,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +19: [2023-05-09 16:44:19,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +19: [2023-05-09 16:44:19,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +19: [2023-05-09 16:44:19,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +19: [2023-05-09 16:44:19,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +31: [2023-05-09 16:44:19,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +31: [2023-05-09 16:44:19,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 7: [2023-05-09 16:44:19,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +15: [2023-05-09 16:44:19,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +24: [2023-05-09 16:44:19,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +30: [2023-05-09 16:44:19,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +30: [2023-05-09 16:44:19,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +15: [2023-05-09 16:44:19,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +13: [2023-05-09 16:44:19,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +25: [2023-05-09 16:44:19,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +30: [2023-05-09 16:44:19,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +30: [2023-05-09 16:44:19,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +23: [2023-05-09 16:44:19,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +13: [2023-05-09 16:44:19,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +23: [2023-05-09 16:44:19,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 5: [2023-05-09 16:44:19,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +13: [2023-05-09 16:44:19,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +16: [2023-05-09 16:44:19,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +16: [2023-05-09 16:44:19,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +13: [2023-05-09 16:44:19,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 5: [2023-05-09 16:44:19,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +24: [2023-05-09 16:44:19,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +24: [2023-05-09 16:44:19,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +16: [2023-05-09 16:44:19,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +16: [2023-05-09 16:44:19,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +11: [2023-05-09 16:44:19,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +24: [2023-05-09 16:44:19,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +24: [2023-05-09 16:44:19,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +24: [2023-05-09 16:44:19,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +21: [2023-05-09 16:44:19,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +21: [2023-05-09 16:44:19,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +28: [2023-05-09 16:44:19,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +31: [2023-05-09 16:44:19,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +11: [2023-05-09 16:44:19,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +31: [2023-05-09 16:44:19,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +17: [2023-05-09 16:44:19,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +17: [2023-05-09 16:44:19,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +17: [2023-05-09 16:44:19,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +31: [2023-05-09 16:44:19,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +31: [2023-05-09 16:44:19,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +17: [2023-05-09 16:44:19,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +28: [2023-05-09 16:44:19,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +15: [2023-05-09 16:44:19,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +15: [2023-05-09 16:44:19,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +24: [2023-05-09 16:44:19,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +24: [2023-05-09 16:44:19,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +15: [2023-05-09 16:44:19,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +15: [2023-05-09 16:44:19,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 4: [2023-05-09 16:44:19,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +30: [2023-05-09 16:44:19,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +21: [2023-05-09 16:44:19,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +30: [2023-05-09 16:44:19,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 0: [2023-05-09 16:44:19,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 0: [2023-05-09 16:44:19,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 0: [2023-05-09 16:44:19,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +31: [2023-05-09 16:44:19,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +23: [2023-05-09 16:44:19,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +22: [2023-05-09 16:44:19,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +22: [2023-05-09 16:44:19,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +22: [2023-05-09 16:44:19,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +22: [2023-05-09 16:44:19,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +31: [2023-05-09 16:44:19,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +22: [2023-05-09 16:44:19,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +28: [2023-05-09 16:44:19,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +28: [2023-05-09 16:44:19,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +28: [2023-05-09 16:44:19,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +22: [2023-05-09 16:44:19,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +22: [2023-05-09 16:44:19,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +26: [2023-05-09 16:44:19,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +23: [2023-05-09 16:44:19,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +22: [2023-05-09 16:44:19,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +22: [2023-05-09 16:44:19,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +28: [2023-05-09 16:44:19,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +15: [2023-05-09 16:44:19,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +25: [2023-05-09 16:44:19,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +26: [2023-05-09 16:44:19,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +26: [2023-05-09 16:44:19,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +26: [2023-05-09 16:44:19,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +26: [2023-05-09 16:44:19,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +26: [2023-05-09 16:44:19,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +28: [2023-05-09 16:44:19,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 5: [2023-05-09 16:44:19,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +24: [2023-05-09 16:44:19,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +24: [2023-05-09 16:44:19,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +28: [2023-05-09 16:44:19,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +28: [2023-05-09 16:44:19,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +28: [2023-05-09 16:44:19,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +24: [2023-05-09 16:44:19,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +24: [2023-05-09 16:44:19,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +24: [2023-05-09 16:44:19,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +22: [2023-05-09 16:44:19,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +24: [2023-05-09 16:44:19,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +24: [2023-05-09 16:44:19,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +24: [2023-05-09 16:44:19,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +15: [2023-05-09 16:44:19,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +15: [2023-05-09 16:44:19,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +15: [2023-05-09 16:44:19,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +16: [2023-05-09 16:44:19,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +15: [2023-05-09 16:44:19,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +15: [2023-05-09 16:44:19,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 4: [2023-05-09 16:44:19,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +15: [2023-05-09 16:44:19,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +15: [2023-05-09 16:44:19,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +15: [2023-05-09 16:44:19,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 4: [2023-05-09 16:44:19,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +17: [2023-05-09 16:44:19,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +17: [2023-05-09 16:44:19,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +13: [2023-05-09 16:44:19,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +17: [2023-05-09 16:44:19,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +16: [2023-05-09 16:44:19,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +22: [2023-05-09 16:44:19,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +17: [2023-05-09 16:44:19,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +15: [2023-05-09 16:44:19,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +21: [2023-05-09 16:44:19,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +13: [2023-05-09 16:44:19,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +13: [2023-05-09 16:44:19,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 7: [2023-05-09 16:44:19,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +22: [2023-05-09 16:44:19,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +22: [2023-05-09 16:44:19,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 7: [2023-05-09 16:44:19,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 5: [2023-05-09 16:44:19,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +16: [2023-05-09 16:44:19,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +13: [2023-05-09 16:44:19,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +22: [2023-05-09 16:44:19,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +16: [2023-05-09 16:44:19,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 0: [2023-05-09 16:44:19,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +26: [2023-05-09 16:44:19,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +30: [2023-05-09 16:44:19,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +30: [2023-05-09 16:44:19,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +26: [2023-05-09 16:44:19,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +26: [2023-05-09 16:44:19,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +26: [2023-05-09 16:44:19,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +28: [2023-05-09 16:44:19,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +22: [2023-05-09 16:44:19,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +22: [2023-05-09 16:44:19,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +26: [2023-05-09 16:44:19,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +30: [2023-05-09 16:44:19,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +10: [2023-05-09 16:44:19,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +10: [2023-05-09 16:44:19,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +10: [2023-05-09 16:44:19,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +10: [2023-05-09 16:44:19,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +10: [2023-05-09 16:44:19,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +10: [2023-05-09 16:44:19,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +10: [2023-05-09 16:44:19,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +26: [2023-05-09 16:44:19,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +30: [2023-05-09 16:44:19,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +30: [2023-05-09 16:44:19,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +30: [2023-05-09 16:44:19,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +28: [2023-05-09 16:44:19,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +26: [2023-05-09 16:44:19,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +30: [2023-05-09 16:44:19,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +28: [2023-05-09 16:44:19,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +28: [2023-05-09 16:44:19,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +10: [2023-05-09 16:44:19,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +28: [2023-05-09 16:44:19,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +30: [2023-05-09 16:44:19,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +26: [2023-05-09 16:44:19,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +25: [2023-05-09 16:44:19,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +25: [2023-05-09 16:44:19,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +15: [2023-05-09 16:44:19,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +28: [2023-05-09 16:44:19,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... + 3: [2023-05-09 16:44:19,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +24: [2023-05-09 16:44:19,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +25: [2023-05-09 16:44:19,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +25: [2023-05-09 16:44:19,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +15: [2023-05-09 16:44:19,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +17: [2023-05-09 16:44:19,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +25: [2023-05-09 16:44:19,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +25: [2023-05-09 16:44:19,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +17: [2023-05-09 16:44:19,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +15: [2023-05-09 16:44:19,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +22: [2023-05-09 16:44:19,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +22: [2023-05-09 16:44:19,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +26: [2023-05-09 16:44:19,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +22: [2023-05-09 16:44:19,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +24: [2023-05-09 16:44:19,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +24: [2023-05-09 16:44:19,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +24: [2023-05-09 16:44:19,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +22: [2023-05-09 16:44:19,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +17: [2023-05-09 16:44:19,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +17: [2023-05-09 16:44:19,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +17: [2023-05-09 16:44:19,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +17: [2023-05-09 16:44:19,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +24: [2023-05-09 16:44:19,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +12: [2023-05-09 16:44:19,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +12: [2023-05-09 16:44:19,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +10: [2023-05-09 16:44:19,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +12: [2023-05-09 16:44:19,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +12: [2023-05-09 16:44:19,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +12: [2023-05-09 16:44:19,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +12: [2023-05-09 16:44:19,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +12: [2023-05-09 16:44:19,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +12: [2023-05-09 16:44:19,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +26: [2023-05-09 16:44:19,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +15: [2023-05-09 16:44:19,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +25: [2023-05-09 16:44:19,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt. +25: [2023-05-09 16:44:19,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt... +26: [2023-05-09 16:44:19,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +30: [2023-05-09 16:44:19,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +22: [2023-05-09 16:44:19,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +25: [2023-05-09 16:44:19,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +22: [2023-05-09 16:44:19,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +25: [2023-05-09 16:44:19,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +25: [2023-05-09 16:44:19,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +17: [2023-05-09 16:44:19,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +15: [2023-05-09 16:44:19,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +17: [2023-05-09 16:44:19,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +30: [2023-05-09 16:44:19,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +15: [2023-05-09 16:44:19,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +10: [2023-05-09 16:44:19,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +15: [2023-05-09 16:44:19,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +10: [2023-05-09 16:44:19,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +26: [2023-05-09 16:44:19,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 6: [2023-05-09 16:44:19,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +26: [2023-05-09 16:44:19,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +26: [2023-05-09 16:44:19,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +24: [2023-05-09 16:44:19,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +24: [2023-05-09 16:44:19,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +10: [2023-05-09 16:44:19,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +24: [2023-05-09 16:44:19,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +30: [2023-05-09 16:44:19,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +25: [2023-05-09 16:44:19,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +12: [2023-05-09 16:44:19,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +30: [2023-05-09 16:44:19,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +15: [2023-05-09 16:44:19,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +30: [2023-05-09 16:44:19,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +23: [2023-05-09 16:44:19,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +23: [2023-05-09 16:44:19,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +23: [2023-05-09 16:44:19,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +23: [2023-05-09 16:44:19,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +23: [2023-05-09 16:44:19,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +23: [2023-05-09 16:44:19,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +23: [2023-05-09 16:44:19,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +30: [2023-05-09 16:44:19,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +23: [2023-05-09 16:44:19,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +26: [2023-05-09 16:44:19,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +26: [2023-05-09 16:44:19,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +10: [2023-05-09 16:44:19,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +26: [2023-05-09 16:44:19,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +26: [2023-05-09 16:44:19,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 4: [2023-05-09 16:44:19,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +10: [2023-05-09 16:44:19,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +12: [2023-05-09 16:44:19,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +12: [2023-05-09 16:44:19,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +30: [2023-05-09 16:44:19,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +10: [2023-05-09 16:44:19,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +12: [2023-05-09 16:44:19,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +12: [2023-05-09 16:44:19,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +30: [2023-05-09 16:44:19,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +23: [2023-05-09 16:44:19,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +23: [2023-05-09 16:44:19,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +23: [2023-05-09 16:44:19,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +10: [2023-05-09 16:44:19,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +12: [2023-05-09 16:44:19,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +12: [2023-05-09 16:44:19,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +12: [2023-05-09 16:44:19,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +23: [2023-05-09 16:44:19,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +23: [2023-05-09 16:44:19,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +23: [2023-05-09 16:44:19,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +23: [2023-05-09 16:44:19,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 1: [2023-05-09 16:44:19,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 1: [2023-05-09 16:44:19,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 1: [2023-05-09 16:44:19,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +23: [2023-05-09 16:44:19,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +10: [2023-05-09 16:44:19,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +10: [2023-05-09 16:44:19,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 6: [2023-05-09 16:44:19,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 6: [2023-05-09 16:44:19,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 1: [2023-05-09 16:44:19,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 6: [2023-05-09 16:44:19,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 1: [2023-05-09 16:44:19,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +10: [2023-05-09 16:44:19,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 1: [2023-05-09 16:44:19,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +12: [2023-05-09 16:44:19,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +20: [2023-05-09 16:44:19,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +20: [2023-05-09 16:44:19,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +20: [2023-05-09 16:44:19,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +12: [2023-05-09 16:44:19,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +12: [2023-05-09 16:44:19,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +10: [2023-05-09 16:44:19,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +12: [2023-05-09 16:44:19,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +20: [2023-05-09 16:44:19,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +20: [2023-05-09 16:44:19,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +20: [2023-05-09 16:44:19,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +20: [2023-05-09 16:44:19,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +20: [2023-05-09 16:44:19,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +13: [2023-05-09 16:44:19,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +13: [2023-05-09 16:44:19,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +13: [2023-05-09 16:44:19,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +13: [2023-05-09 16:44:19,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +10: [2023-05-09 16:44:19,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +10: [2023-05-09 16:44:19,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +14: [2023-05-09 16:44:19,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +14: [2023-05-09 16:44:19,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +14: [2023-05-09 16:44:19,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +14: [2023-05-09 16:44:19,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +14: [2023-05-09 16:44:19,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +14: [2023-05-09 16:44:19,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +14: [2023-05-09 16:44:19,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +14: [2023-05-09 16:44:19,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +13: [2023-05-09 16:44:19,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +13: [2023-05-09 16:44:19,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +13: [2023-05-09 16:44:19,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +10: [2023-05-09 16:44:19,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +12: [2023-05-09 16:44:19,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +10: [2023-05-09 16:44:19,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +12: [2023-05-09 16:44:19,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +12: [2023-05-09 16:44:19,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +20: [2023-05-09 16:44:19,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +12: [2023-05-09 16:44:19,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 2: [2023-05-09 16:44:19,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +13: [2023-05-09 16:44:19,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +20: [2023-05-09 16:44:19,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 1: [2023-05-09 16:44:19,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 1: [2023-05-09 16:44:19,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +14: [2023-05-09 16:44:19,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +14: [2023-05-09 16:44:19,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +12: [2023-05-09 16:44:19,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +12: [2023-05-09 16:44:19,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +20: [2023-05-09 16:44:19,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +10: [2023-05-09 16:44:19,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +20: [2023-05-09 16:44:19,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +10: [2023-05-09 16:44:19,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 6: [2023-05-09 16:44:19,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +12: [2023-05-09 16:44:19,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +12: [2023-05-09 16:44:19,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +13: [2023-05-09 16:44:19,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +13: [2023-05-09 16:44:19,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 6: [2023-05-09 16:44:19,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 6: [2023-05-09 16:44:19,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +14: [2023-05-09 16:44:19,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +19: [2023-05-09 16:44:19,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +19: [2023-05-09 16:44:19,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +19: [2023-05-09 16:44:19,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +19: [2023-05-09 16:44:19,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +19: [2023-05-09 16:44:19,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +19: [2023-05-09 16:44:19,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +14: [2023-05-09 16:44:19,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +19: [2023-05-09 16:44:19,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +19: [2023-05-09 16:44:19,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +20: [2023-05-09 16:44:19,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +13: [2023-05-09 16:44:19,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +31: [2023-05-09 16:44:19,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +31: [2023-05-09 16:44:19,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +31: [2023-05-09 16:44:19,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +31: [2023-05-09 16:44:19,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +31: [2023-05-09 16:44:19,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +31: [2023-05-09 16:44:19,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +10: [2023-05-09 16:44:19,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +10: [2023-05-09 16:44:19,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +10: [2023-05-09 16:44:19,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +10: [2023-05-09 16:44:19,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +10: [2023-05-09 16:44:19,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +10: [2023-05-09 16:44:19,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +20: [2023-05-09 16:44:19,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +31: [2023-05-09 16:44:19,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +31: [2023-05-09 16:44:19,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +14: [2023-05-09 16:44:19,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +14: [2023-05-09 16:44:19,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +13: [2023-05-09 16:44:19,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +12: [2023-05-09 16:44:19,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +14: [2023-05-09 16:44:19,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +12: [2023-05-09 16:44:19,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +14: [2023-05-09 16:44:19,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +20: [2023-05-09 16:44:19,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +13: [2023-05-09 16:44:19,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +12: [2023-05-09 16:44:19,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +12: [2023-05-09 16:44:19,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +16: [2023-05-09 16:44:19,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +16: [2023-05-09 16:44:19,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +16: [2023-05-09 16:44:19,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +16: [2023-05-09 16:44:19,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +16: [2023-05-09 16:44:19,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +16: [2023-05-09 16:44:19,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +16: [2023-05-09 16:44:19,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +19: [2023-05-09 16:44:19,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +13: [2023-05-09 16:44:19,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +20: [2023-05-09 16:44:19,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +16: [2023-05-09 16:44:19,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +31: [2023-05-09 16:44:19,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +31: [2023-05-09 16:44:19,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +19: [2023-05-09 16:44:19,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +13: [2023-05-09 16:44:19,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +19: [2023-05-09 16:44:19,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +19: [2023-05-09 16:44:19,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +16: [2023-05-09 16:44:19,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +31: [2023-05-09 16:44:19,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +16: [2023-05-09 16:44:19,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +31: [2023-05-09 16:44:19,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +19: [2023-05-09 16:44:19,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +16: [2023-05-09 16:44:19,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +13: [2023-05-09 16:44:19,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +19: [2023-05-09 16:44:19,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +19: [2023-05-09 16:44:19,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +31: [2023-05-09 16:44:19,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +31: [2023-05-09 16:44:19,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +16: [2023-05-09 16:44:19,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +19: [2023-05-09 16:44:19,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +16: [2023-05-09 16:44:19,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +16: [2023-05-09 16:44:19,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +31: [2023-05-09 16:44:19,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +16: [2023-05-09 16:44:19,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +31: [2023-05-09 16:44:19,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +16: [2023-05-09 16:44:19,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +14: [2023-05-09 16:44:19,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +14: [2023-05-09 16:44:19,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +14: [2023-05-09 16:44:19,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +14: [2023-05-09 16:44:19,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +21: [2023-05-09 16:44:19,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +21: [2023-05-09 16:44:19,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +21: [2023-05-09 16:44:19,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +21: [2023-05-09 16:44:19,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +14: [2023-05-09 16:44:19,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +29: [2023-05-09 16:44:19,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +29: [2023-05-09 16:44:19,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +29: [2023-05-09 16:44:19,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +29: [2023-05-09 16:44:19,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +29: [2023-05-09 16:44:19,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +29: [2023-05-09 16:44:19,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +29: [2023-05-09 16:44:19,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +21: [2023-05-09 16:44:19,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +21: [2023-05-09 16:44:19,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +14: [2023-05-09 16:44:19,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +29: [2023-05-09 16:44:19,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +14: [2023-05-09 16:44:19,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +11: [2023-05-09 16:44:19,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +11: [2023-05-09 16:44:19,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +11: [2023-05-09 16:44:19,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +11: [2023-05-09 16:44:19,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +11: [2023-05-09 16:44:19,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +11: [2023-05-09 16:44:19,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +11: [2023-05-09 16:44:19,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +14: [2023-05-09 16:44:19,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +21: [2023-05-09 16:44:19,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +11: [2023-05-09 16:44:19,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +21: [2023-05-09 16:44:19,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +14: [2023-05-09 16:44:19,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 9: [2023-05-09 16:44:19,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 9: [2023-05-09 16:44:19,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +14: [2023-05-09 16:44:19,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +29: [2023-05-09 16:44:19,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +29: [2023-05-09 16:44:19,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +29: [2023-05-09 16:44:19,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +18: [2023-05-09 16:44:19,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 8: [2023-05-09 16:44:19,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +18: [2023-05-09 16:44:19,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 8: [2023-05-09 16:44:19,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 8: [2023-05-09 16:44:19,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +21: [2023-05-09 16:44:19,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +18: [2023-05-09 16:44:19,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +21: [2023-05-09 16:44:19,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +18: [2023-05-09 16:44:19,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 9: [2023-05-09 16:44:19,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +14: [2023-05-09 16:44:19,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +14: [2023-05-09 16:44:19,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +14: [2023-05-09 16:44:19,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +14: [2023-05-09 16:44:19,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 9: [2023-05-09 16:44:19,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +29: [2023-05-09 16:44:19,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +21: [2023-05-09 16:44:19,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +21: [2023-05-09 16:44:19,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +11: [2023-05-09 16:44:19,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +11: [2023-05-09 16:44:19,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +11: [2023-05-09 16:44:19,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +14: [2023-05-09 16:44:19,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +14: [2023-05-09 16:44:19,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +18: [2023-05-09 16:44:19,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +18: [2023-05-09 16:44:19,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +29: [2023-05-09 16:44:19,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +21: [2023-05-09 16:44:19,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +21: [2023-05-09 16:44:19,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +29: [2023-05-09 16:44:19,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +18: [2023-05-09 16:44:19,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +21: [2023-05-09 16:44:19,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +21: [2023-05-09 16:44:19,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 8: [2023-05-09 16:44:19,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +18: [2023-05-09 16:44:19,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +29: [2023-05-09 16:44:19,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +11: [2023-05-09 16:44:19,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +11: [2023-05-09 16:44:19,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +11: [2023-05-09 16:44:19,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 9: [2023-05-09 16:44:19,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +27: [2023-05-09 16:44:19,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +27: [2023-05-09 16:44:19,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +27: [2023-05-09 16:44:19,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +29: [2023-05-09 16:44:19,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +27: [2023-05-09 16:44:19,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 9: [2023-05-09 16:44:19,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +11: [2023-05-09 16:44:19,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 9: [2023-05-09 16:44:19,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 8: [2023-05-09 16:44:19,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +27: [2023-05-09 16:44:19,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 1: [2023-05-09 16:44:19,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +27: [2023-05-09 16:44:19,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +27: [2023-05-09 16:44:19,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +27: [2023-05-09 16:44:19,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +11: [2023-05-09 16:44:19,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +26: [2023-05-09 16:44:19,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +26: [2023-05-09 16:44:19,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 9: [2023-05-09 16:44:19,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +26: [2023-05-09 16:44:19,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +26: [2023-05-09 16:44:19,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +21: [2023-05-09 16:44:19,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +21: [2023-05-09 16:44:19,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +21: [2023-05-09 16:44:19,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +21: [2023-05-09 16:44:19,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +21: [2023-05-09 16:44:19,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +21: [2023-05-09 16:44:19,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +21: [2023-05-09 16:44:19,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +21: [2023-05-09 16:44:19,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +29: [2023-05-09 16:44:19,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +29: [2023-05-09 16:44:19,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +29: [2023-05-09 16:44:19,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +29: [2023-05-09 16:44:19,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +26: [2023-05-09 16:44:19,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 1: [2023-05-09 16:44:19,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 1: [2023-05-09 16:44:19,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 1: [2023-05-09 16:44:19,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +26: [2023-05-09 16:44:19,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +26: [2023-05-09 16:44:19,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +26: [2023-05-09 16:44:19,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +29: [2023-05-09 16:44:19,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +29: [2023-05-09 16:44:19,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +29: [2023-05-09 16:44:19,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +21: [2023-05-09 16:44:19,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +29: [2023-05-09 16:44:19,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +24: [2023-05-09 16:44:19,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +24: [2023-05-09 16:44:19,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +24: [2023-05-09 16:44:19,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +19: [2023-05-09 16:44:19,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +19: [2023-05-09 16:44:19,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +19: [2023-05-09 16:44:19,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +24: [2023-05-09 16:44:19,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +19: [2023-05-09 16:44:19,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +20: [2023-05-09 16:44:19,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +20: [2023-05-09 16:44:19,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +20: [2023-05-09 16:44:19,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +20: [2023-05-09 16:44:19,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 1: [2023-05-09 16:44:19,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +21: [2023-05-09 16:44:19,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +21: [2023-05-09 16:44:19,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +21: [2023-05-09 16:44:19,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 0: [2023-05-09 16:44:19,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 0: [2023-05-09 16:44:19,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +21: [2023-05-09 16:44:19,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +29: [2023-05-09 16:44:19,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +20: [2023-05-09 16:44:19,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +24: [2023-05-09 16:44:19,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +19: [2023-05-09 16:44:19,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +20: [2023-05-09 16:44:19,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +20: [2023-05-09 16:44:19,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +20: [2023-05-09 16:44:19,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +24: [2023-05-09 16:44:19,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +19: [2023-05-09 16:44:19,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +19: [2023-05-09 16:44:19,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +24: [2023-05-09 16:44:19,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +29: [2023-05-09 16:44:19,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +24: [2023-05-09 16:44:19,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +21: [2023-05-09 16:44:19,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +19: [2023-05-09 16:44:19,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +21: [2023-05-09 16:44:19,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +29: [2023-05-09 16:44:19,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +29: [2023-05-09 16:44:19,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +29: [2023-05-09 16:44:19,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +29: [2023-05-09 16:44:19,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 1: [2023-05-09 16:44:19,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +29: [2023-05-09 16:44:19,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +29: [2023-05-09 16:44:19,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +21: [2023-05-09 16:44:19,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +17: [2023-05-09 16:44:19,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +17: [2023-05-09 16:44:19,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +17: [2023-05-09 16:44:19,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +17: [2023-05-09 16:44:19,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +17: [2023-05-09 16:44:19,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +17: [2023-05-09 16:44:19,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +17: [2023-05-09 16:44:19,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +17: [2023-05-09 16:44:19,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +24: [2023-05-09 16:44:19,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +27: [2023-05-09 16:44:19,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +24: [2023-05-09 16:44:19,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +27: [2023-05-09 16:44:19,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +27: [2023-05-09 16:44:19,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +27: [2023-05-09 16:44:19,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +27: [2023-05-09 16:44:19,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +12: [2023-05-09 16:44:19,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +12: [2023-05-09 16:44:19,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +12: [2023-05-09 16:44:19,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +27: [2023-05-09 16:44:19,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +27: [2023-05-09 16:44:19,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +27: [2023-05-09 16:44:19,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +27: [2023-05-09 16:44:19,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +27: [2023-05-09 16:44:19,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +27: [2023-05-09 16:44:19,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +27: [2023-05-09 16:44:19,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +12: [2023-05-09 16:44:19,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +27: [2023-05-09 16:44:19,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +20: [2023-05-09 16:44:19,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +17: [2023-05-09 16:44:19,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +17: [2023-05-09 16:44:19,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +17: [2023-05-09 16:44:19,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +20: [2023-05-09 16:44:19,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +17: [2023-05-09 16:44:19,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +27: [2023-05-09 16:44:19,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +27: [2023-05-09 16:44:19,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +19: [2023-05-09 16:44:19,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +20: [2023-05-09 16:44:19,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +19: [2023-05-09 16:44:19,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +20: [2023-05-09 16:44:19,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +20: [2023-05-09 16:44:19,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +20: [2023-05-09 16:44:19,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +27: [2023-05-09 16:44:19,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +24: [2023-05-09 16:44:19,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +24: [2023-05-09 16:44:19,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +24: [2023-05-09 16:44:19,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +24: [2023-05-09 16:44:19,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 1: [2023-05-09 16:44:19,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +19: [2023-05-09 16:44:19,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +19: [2023-05-09 16:44:19,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 1: [2023-05-09 16:44:19,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +24: [2023-05-09 16:44:19,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +24: [2023-05-09 16:44:19,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +20: [2023-05-09 16:44:19,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +20: [2023-05-09 16:44:19,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 1: [2023-05-09 16:44:19,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 1: [2023-05-09 16:44:19,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +12: [2023-05-09 16:44:19,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +12: [2023-05-09 16:44:19,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +12: [2023-05-09 16:44:19,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +12: [2023-05-09 16:44:19,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +17: [2023-05-09 16:44:19,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +19: [2023-05-09 16:44:19,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +19: [2023-05-09 16:44:19,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +17: [2023-05-09 16:44:19,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +17: [2023-05-09 16:44:19,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +19: [2023-05-09 16:44:19,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +27: [2023-05-09 16:44:19,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +19: [2023-05-09 16:44:19,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +27: [2023-05-09 16:44:19,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +27: [2023-05-09 16:44:19,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +17: [2023-05-09 16:44:19,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +10: [2023-05-09 16:44:19,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +10: [2023-05-09 16:44:19,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +17: [2023-05-09 16:44:19,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +10: [2023-05-09 16:44:19,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +10: [2023-05-09 16:44:19,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +27: [2023-05-09 16:44:19,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +17: [2023-05-09 16:44:19,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 0: [2023-05-09 16:44:19,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 0: [2023-05-09 16:44:19,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +17: [2023-05-09 16:44:19,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +27: [2023-05-09 16:44:19,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +27: [2023-05-09 16:44:19,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +17: [2023-05-09 16:44:19,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +18: [2023-05-09 16:44:19,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +18: [2023-05-09 16:44:19,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +18: [2023-05-09 16:44:19,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +18: [2023-05-09 16:44:19,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +18: [2023-05-09 16:44:19,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +27: [2023-05-09 16:44:19,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +18: [2023-05-09 16:44:19,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +17: [2023-05-09 16:44:19,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +18: [2023-05-09 16:44:19,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +18: [2023-05-09 16:44:19,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +18: [2023-05-09 16:44:19,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +18: [2023-05-09 16:44:19,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +18: [2023-05-09 16:44:19,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +18: [2023-05-09 16:44:19,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +17: [2023-05-09 16:44:19,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +18: [2023-05-09 16:44:19,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +18: [2023-05-09 16:44:19,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +18: [2023-05-09 16:44:19,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +10: [2023-05-09 16:44:19,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +17: [2023-05-09 16:44:19,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +10: [2023-05-09 16:44:19,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +18: [2023-05-09 16:44:19,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +27: [2023-05-09 16:44:19,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +10: [2023-05-09 16:44:19,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +10: [2023-05-09 16:44:19,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +17: [2023-05-09 16:44:19,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 6: [2023-05-09 16:44:19,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 6: [2023-05-09 16:44:19,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +17: [2023-05-09 16:44:19,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 7: [2023-05-09 16:44:19,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 7: [2023-05-09 16:44:19,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 7: [2023-05-09 16:44:19,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 7: [2023-05-09 16:44:19,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +17: [2023-05-09 16:44:19,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +17: [2023-05-09 16:44:19,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +17: [2023-05-09 16:44:19,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +18: [2023-05-09 16:44:19,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +18: [2023-05-09 16:44:19,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +18: [2023-05-09 16:44:19,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +28: [2023-05-09 16:44:19,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +28: [2023-05-09 16:44:19,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +28: [2023-05-09 16:44:19,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +28: [2023-05-09 16:44:19,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +28: [2023-05-09 16:44:19,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +15: [2023-05-09 16:44:19,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +15: [2023-05-09 16:44:19,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +28: [2023-05-09 16:44:19,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +28: [2023-05-09 16:44:19,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +28: [2023-05-09 16:44:19,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +15: [2023-05-09 16:44:19,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +30: [2023-05-09 16:44:19,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +30: [2023-05-09 16:44:19,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +15: [2023-05-09 16:44:19,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +15: [2023-05-09 16:44:19,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +30: [2023-05-09 16:44:19,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +30: [2023-05-09 16:44:19,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +30: [2023-05-09 16:44:19,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +30: [2023-05-09 16:44:19,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +30: [2023-05-09 16:44:19,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +25: [2023-05-09 16:44:19,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +30: [2023-05-09 16:44:19,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +25: [2023-05-09 16:44:19,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +25: [2023-05-09 16:44:19,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +25: [2023-05-09 16:44:19,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +25: [2023-05-09 16:44:19,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +25: [2023-05-09 16:44:19,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +24: [2023-05-09 16:44:19,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +24: [2023-05-09 16:44:19,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +18: [2023-05-09 16:44:19,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +15: [2023-05-09 16:44:19,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +24: [2023-05-09 16:44:19,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +24: [2023-05-09 16:44:19,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +24: [2023-05-09 16:44:19,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +25: [2023-05-09 16:44:19,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +24: [2023-05-09 16:44:19,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +15: [2023-05-09 16:44:19,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +15: [2023-05-09 16:44:19,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +25: [2023-05-09 16:44:19,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +24: [2023-05-09 16:44:19,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +18: [2023-05-09 16:44:19,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +24: [2023-05-09 16:44:19,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +17: [2023-05-09 16:44:19,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +17: [2023-05-09 16:44:19,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +18: [2023-05-09 16:44:19,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +17: [2023-05-09 16:44:19,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +28: [2023-05-09 16:44:19,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +17: [2023-05-09 16:44:19,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +15: [2023-05-09 16:44:19,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +30: [2023-05-09 16:44:19,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +30: [2023-05-09 16:44:19,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +24: [2023-05-09 16:44:19,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +22: [2023-05-09 16:44:19,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +22: [2023-05-09 16:44:19,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +22: [2023-05-09 16:44:19,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +22: [2023-05-09 16:44:19,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +22: [2023-05-09 16:44:19,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +22: [2023-05-09 16:44:19,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +22: [2023-05-09 16:44:19,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +22: [2023-05-09 16:44:19,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +30: [2023-05-09 16:44:19,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +30: [2023-05-09 16:44:19,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +22: [2023-05-09 16:44:19,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +22: [2023-05-09 16:44:19,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +18: [2023-05-09 16:44:19,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +22: [2023-05-09 16:44:19,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +22: [2023-05-09 16:44:19,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 7: [2023-05-09 16:44:19,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 7: [2023-05-09 16:44:19,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +28: [2023-05-09 16:44:19,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +25: [2023-05-09 16:44:19,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +25: [2023-05-09 16:44:19,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +15: [2023-05-09 16:44:19,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +24: [2023-05-09 16:44:19,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +12: [2023-05-09 16:44:19,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +12: [2023-05-09 16:44:19,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +12: [2023-05-09 16:44:19,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +30: [2023-05-09 16:44:19,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +12: [2023-05-09 16:44:19,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +12: [2023-05-09 16:44:19,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +12: [2023-05-09 16:44:19,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +12: [2023-05-09 16:44:19,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +12: [2023-05-09 16:44:19,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +12: [2023-05-09 16:44:19,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +12: [2023-05-09 16:44:19,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +12: [2023-05-09 16:44:19,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +12: [2023-05-09 16:44:19,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +12: [2023-05-09 16:44:19,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +12: [2023-05-09 16:44:19,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +12: [2023-05-09 16:44:19,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +30: [2023-05-09 16:44:19,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +15: [2023-05-09 16:44:19,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +28: [2023-05-09 16:44:19,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +12: [2023-05-09 16:44:19,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +28: [2023-05-09 16:44:19,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 3: [2023-05-09 16:44:19,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +28: [2023-05-09 16:44:19,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +24: [2023-05-09 16:44:19,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +28: [2023-05-09 16:44:19,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +18: [2023-05-09 16:44:19,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +11: [2023-05-09 16:44:19,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +11: [2023-05-09 16:44:19,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +11: [2023-05-09 16:44:19,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +22: [2023-05-09 16:44:19,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +30: [2023-05-09 16:44:19,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +11: [2023-05-09 16:44:19,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +25: [2023-05-09 16:44:19,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +30: [2023-05-09 16:44:19,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 2: [2023-05-09 16:44:19,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 2: [2023-05-09 16:44:19,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +15: [2023-05-09 16:44:19,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +22: [2023-05-09 16:44:19,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +15: [2023-05-09 16:44:19,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +26: [2023-05-09 16:44:19,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +26: [2023-05-09 16:44:19,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +26: [2023-05-09 16:44:19,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +26: [2023-05-09 16:44:19,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +26: [2023-05-09 16:44:19,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +26: [2023-05-09 16:44:19,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +26: [2023-05-09 16:44:19,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +26: [2023-05-09 16:44:19,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +26: [2023-05-09 16:44:19,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +26: [2023-05-09 16:44:19,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +26: [2023-05-09 16:44:19,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +28: [2023-05-09 16:44:19,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +26: [2023-05-09 16:44:19,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +26: [2023-05-09 16:44:19,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +22: [2023-05-09 16:44:19,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +22: [2023-05-09 16:44:19,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +26: [2023-05-09 16:44:19,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 4: [2023-05-09 16:44:19,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +26: [2023-05-09 16:44:19,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 2: [2023-05-09 16:44:19,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +26: [2023-05-09 16:44:19,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +25: [2023-05-09 16:44:19,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +25: [2023-05-09 16:44:19,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +31: [2023-05-09 16:44:19,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +31: [2023-05-09 16:44:19,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +31: [2023-05-09 16:44:19,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +31: [2023-05-09 16:44:19,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +31: [2023-05-09 16:44:19,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +31: [2023-05-09 16:44:19,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +31: [2023-05-09 16:44:19,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +31: [2023-05-09 16:44:19,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 5: [2023-05-09 16:44:19,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +24: [2023-05-09 16:44:19,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +22: [2023-05-09 16:44:19,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +25: [2023-05-09 16:44:19,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +28: [2023-05-09 16:44:19,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +28: [2023-05-09 16:44:19,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +23: [2023-05-09 16:44:19,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +23: [2023-05-09 16:44:19,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +23: [2023-05-09 16:44:19,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +25: [2023-05-09 16:44:19,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 3: [2023-05-09 16:44:19,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 3: [2023-05-09 16:44:19,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 4: [2023-05-09 16:44:19,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 7: [2023-05-09 16:44:19,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +11: [2023-05-09 16:44:19,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +11: [2023-05-09 16:44:19,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +25: [2023-05-09 16:44:19,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 7: [2023-05-09 16:44:19,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +28: [2023-05-09 16:44:19,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 3: [2023-05-09 16:44:19,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 5: [2023-05-09 16:44:19,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +25: [2023-05-09 16:44:19,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +11: [2023-05-09 16:44:19,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +11: [2023-05-09 16:44:19,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +28: [2023-05-09 16:44:19,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +22: [2023-05-09 16:44:19,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +31: [2023-05-09 16:44:19,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +31: [2023-05-09 16:44:19,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +23: [2023-05-09 16:44:19,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +24: [2023-05-09 16:44:19,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +24: [2023-05-09 16:44:19,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +15: [2023-05-09 16:44:19,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +25: [2023-05-09 16:44:19,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +15: [2023-05-09 16:44:19,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +28: [2023-05-09 16:44:19,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +28: [2023-05-09 16:44:19,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +31: [2023-05-09 16:44:19,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 4: [2023-05-09 16:44:19,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +31: [2023-05-09 16:44:19,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 4: [2023-05-09 16:44:19,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +28: [2023-05-09 16:44:19,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +15: [2023-05-09 16:44:19,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +15: [2023-05-09 16:44:19,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +15: [2023-05-09 16:44:19,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +16: [2023-05-09 16:44:19,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +16: [2023-05-09 16:44:19,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +25: [2023-05-09 16:44:19,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +22: [2023-05-09 16:44:19,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +12: [2023-05-09 16:44:19,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +12: [2023-05-09 16:44:19,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +12: [2023-05-09 16:44:19,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +26: [2023-05-09 16:44:19,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +16: [2023-05-09 16:44:19,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +28: [2023-05-09 16:44:19,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +16: [2023-05-09 16:44:19,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +23: [2023-05-09 16:44:19,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +23: [2023-05-09 16:44:19,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +28: [2023-05-09 16:44:19,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +22: [2023-05-09 16:44:19,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +23: [2023-05-09 16:44:19,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +22: [2023-05-09 16:44:19,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +25: [2023-05-09 16:44:19,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +22: [2023-05-09 16:44:19,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +30: [2023-05-09 16:44:19,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +30: [2023-05-09 16:44:19,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +15: [2023-05-09 16:44:19,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 5: [2023-05-09 16:44:19,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +26: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +26: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 6: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 6: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 5: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +23: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +23: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +24: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +24: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +23: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +23: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +23: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +23: [2023-05-09 16:44:19,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +23: [2023-05-09 16:44:19,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +23: [2023-05-09 16:44:19,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +31: [2023-05-09 16:44:19,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +30: [2023-05-09 16:44:19,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +15: [2023-05-09 16:44:19,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +11: [2023-05-09 16:44:19,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +31: [2023-05-09 16:44:19,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +25: [2023-05-09 16:44:19,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +11: [2023-05-09 16:44:19,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 7: [2023-05-09 16:44:19,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +12: [2023-05-09 16:44:19,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +23: [2023-05-09 16:44:19,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +25: [2023-05-09 16:44:19,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +30: [2023-05-09 16:44:19,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +13: [2023-05-09 16:44:19,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +13: [2023-05-09 16:44:19,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +11: [2023-05-09 16:44:19,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +25: [2023-05-09 16:44:19,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +11: [2023-05-09 16:44:19,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 4: [2023-05-09 16:44:19,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +31: [2023-05-09 16:44:19,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +31: [2023-05-09 16:44:19,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +31: [2023-05-09 16:44:19,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +13: [2023-05-09 16:44:19,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +13: [2023-05-09 16:44:19,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +31: [2023-05-09 16:44:19,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +28: [2023-05-09 16:44:19,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +22: [2023-05-09 16:44:19,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +15: [2023-05-09 16:44:19,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +16: [2023-05-09 16:44:19,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +22: [2023-05-09 16:44:19,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +15: [2023-05-09 16:44:19,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +15: [2023-05-09 16:44:19,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +22: [2023-05-09 16:44:19,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +22: [2023-05-09 16:44:19,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 2: [2023-05-09 16:44:19,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +16: [2023-05-09 16:44:19,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +26: [2023-05-09 16:44:19,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 2: [2023-05-09 16:44:19,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +30: [2023-05-09 16:44:19,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +16: [2023-05-09 16:44:19,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +16: [2023-05-09 16:44:19,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +12: [2023-05-09 16:44:19,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +28: [2023-05-09 16:44:19,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 8: [2023-05-09 16:44:19,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +31: [2023-05-09 16:44:19,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +26: [2023-05-09 16:44:19,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +15: [2023-05-09 16:44:19,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 8: [2023-05-09 16:44:19,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +22: [2023-05-09 16:44:19,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +22: [2023-05-09 16:44:19,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +12: [2023-05-09 16:44:19,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. + 4: [2023-05-09 16:44:19,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +30: [2023-05-09 16:44:19,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +26: [2023-05-09 16:44:19,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +30: [2023-05-09 16:44:19,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +22: [2023-05-09 16:44:19,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +22: [2023-05-09 16:44:19,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +12: [2023-05-09 16:44:19,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +12: [2023-05-09 16:44:19,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +31: [2023-05-09 16:44:19,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +30: [2023-05-09 16:44:19,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +26: [2023-05-09 16:44:19,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +31: [2023-05-09 16:44:19,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +13: [2023-05-09 16:44:19,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +13: [2023-05-09 16:44:19,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +11: [2023-05-09 16:44:19,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +11: [2023-05-09 16:44:19,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 6: [2023-05-09 16:44:19,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 7: [2023-05-09 16:44:19,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +25: [2023-05-09 16:44:19,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +10: [2023-05-09 16:44:19,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +10: [2023-05-09 16:44:19,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +13: [2023-05-09 16:44:19,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +13: [2023-05-09 16:44:19,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +11: [2023-05-09 16:44:19,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +11: [2023-05-09 16:44:19,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +31: [2023-05-09 16:44:19,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +23: [2023-05-09 16:44:19,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +25: [2023-05-09 16:44:19,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +23: [2023-05-09 16:44:19,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 3: [2023-05-09 16:44:19,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt. +25: [2023-05-09 16:44:19,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 5: [2023-05-09 16:44:19,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +10: [2023-05-09 16:44:19,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +28: [2023-05-09 16:44:19,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +28: [2023-05-09 16:44:19,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +28: [2023-05-09 16:44:19,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +28: [2023-05-09 16:44:19,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 5: [2023-05-09 16:44:19,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +31: [2023-05-09 16:44:19,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +10: [2023-05-09 16:44:19,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +31: [2023-05-09 16:44:19,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +22: [2023-05-09 16:44:19,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +22: [2023-05-09 16:44:19,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 5: [2023-05-09 16:44:19,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +10: [2023-05-09 16:44:19,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +10: [2023-05-09 16:44:19,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +10: [2023-05-09 16:44:19,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +10: [2023-05-09 16:44:19,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +28: [2023-05-09 16:44:19,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +28: [2023-05-09 16:44:19,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 6: [2023-05-09 16:44:19,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +10: [2023-05-09 16:44:19,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +10: [2023-05-09 16:44:19,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +23: [2023-05-09 16:44:19,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +23: [2023-05-09 16:44:19,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +23: [2023-05-09 16:44:19,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +10: [2023-05-09 16:44:19,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 3: [2023-05-09 16:44:19,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +31: [2023-05-09 16:44:19,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +10: [2023-05-09 16:44:19,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +16: [2023-05-09 16:44:19,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +10: [2023-05-09 16:44:19,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +26: [2023-05-09 16:44:19,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 4: [2023-05-09 16:44:19,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +16: [2023-05-09 16:44:19,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +10: [2023-05-09 16:44:19,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 8: [2023-05-09 16:44:19,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +16: [2023-05-09 16:44:19,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +16: [2023-05-09 16:44:19,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +16: [2023-05-09 16:44:19,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +16: [2023-05-09 16:44:19,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +10: [2023-05-09 16:44:19,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +23: [2023-05-09 16:44:19,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +23: [2023-05-09 16:44:19,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 8: [2023-05-09 16:44:19,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +31: [2023-05-09 16:44:19,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +15: [2023-05-09 16:44:19,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +15: [2023-05-09 16:44:19,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +23: [2023-05-09 16:44:19,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +10: [2023-05-09 16:44:19,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +15: [2023-05-09 16:44:19,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +15: [2023-05-09 16:44:19,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 2: [2023-05-09 16:44:19,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +15: [2023-05-09 16:44:19,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +15: [2023-05-09 16:44:19,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +30: [2023-05-09 16:44:19,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +23: [2023-05-09 16:44:19,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +30: [2023-05-09 16:44:19,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +23: [2023-05-09 16:44:19,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +30: [2023-05-09 16:44:19,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +30: [2023-05-09 16:44:19,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +15: [2023-05-09 16:44:19,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +15: [2023-05-09 16:44:19,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +23: [2023-05-09 16:44:19,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 2: [2023-05-09 16:44:19,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +13: [2023-05-09 16:44:19,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt... +10: [2023-05-09 16:44:19,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +13: [2023-05-09 16:44:19,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 7: [2023-05-09 16:44:19,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +25: [2023-05-09 16:44:19,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +25: [2023-05-09 16:44:19,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +30: [2023-05-09 16:44:19,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +31: [2023-05-09 16:44:19,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +31: [2023-05-09 16:44:19,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +16: [2023-05-09 16:44:19,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +16: [2023-05-09 16:44:19,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +16: [2023-05-09 16:44:19,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +16: [2023-05-09 16:44:19,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +16: [2023-05-09 16:44:19,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +30: [2023-05-09 16:44:19,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +16: [2023-05-09 16:44:19,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +16: [2023-05-09 16:44:19,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +16: [2023-05-09 16:44:19,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +13: [2023-05-09 16:44:19,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +13: [2023-05-09 16:44:19,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +23: [2023-05-09 16:44:19,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +23: [2023-05-09 16:44:19,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +25: [2023-05-09 16:44:19,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +25: [2023-05-09 16:44:19,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 5: [2023-05-09 16:44:19,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +23: [2023-05-09 16:44:19,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +13: [2023-05-09 16:44:19,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +13: [2023-05-09 16:44:19,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +13: [2023-05-09 16:44:19,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +13: [2023-05-09 16:44:19,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 6: [2023-05-09 16:44:19,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +23: [2023-05-09 16:44:19,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +16: [2023-05-09 16:44:19,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +31: [2023-05-09 16:44:19,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +31: [2023-05-09 16:44:19,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +16: [2023-05-09 16:44:19,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 5: [2023-05-09 16:44:19,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +10: [2023-05-09 16:44:19,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +14: [2023-05-09 16:44:19,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +14: [2023-05-09 16:44:19,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +14: [2023-05-09 16:44:19,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +14: [2023-05-09 16:44:19,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +30: [2023-05-09 16:44:19,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +30: [2023-05-09 16:44:19,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +25: [2023-05-09 16:44:19,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +23: [2023-05-09 16:44:19,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +31: [2023-05-09 16:44:19,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +10: [2023-05-09 16:44:19,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +25: [2023-05-09 16:44:19,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +16: [2023-05-09 16:44:19,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +31: [2023-05-09 16:44:19,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... + 5: [2023-05-09 16:44:19,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +14: [2023-05-09 16:44:19,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +14: [2023-05-09 16:44:19,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +14: [2023-05-09 16:44:19,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +14: [2023-05-09 16:44:19,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +16: [2023-05-09 16:44:19,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +16: [2023-05-09 16:44:19,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +20: [2023-05-09 16:44:19,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +20: [2023-05-09 16:44:19,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +20: [2023-05-09 16:44:19,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +20: [2023-05-09 16:44:19,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +20: [2023-05-09 16:44:19,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +20: [2023-05-09 16:44:19,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +20: [2023-05-09 16:44:19,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +20: [2023-05-09 16:44:19,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +16: [2023-05-09 16:44:19,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +14: [2023-05-09 16:44:19,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +14: [2023-05-09 16:44:19,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +14: [2023-05-09 16:44:19,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +14: [2023-05-09 16:44:19,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +14: [2023-05-09 16:44:19,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +14: [2023-05-09 16:44:19,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +10: [2023-05-09 16:44:19,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +16: [2023-05-09 16:44:19,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +14: [2023-05-09 16:44:19,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +16: [2023-05-09 16:44:19,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +16: [2023-05-09 16:44:19,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +16: [2023-05-09 16:44:19,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +14: [2023-05-09 16:44:19,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt. +10: [2023-05-09 16:44:19,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +20: [2023-05-09 16:44:19,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +10: [2023-05-09 16:44:19,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +20: [2023-05-09 16:44:19,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt... +20: [2023-05-09 16:44:19,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +20: [2023-05-09 16:44:19,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +14: [2023-05-09 16:44:19,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +14: [2023-05-09 16:44:19,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +20: [2023-05-09 16:44:19,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +20: [2023-05-09 16:44:19,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +10: [2023-05-09 16:44:19,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +14: [2023-05-09 16:44:19,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +19: [2023-05-09 16:44:19,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +19: [2023-05-09 16:44:19,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +14: [2023-05-09 16:44:19,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +19: [2023-05-09 16:44:19,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +19: [2023-05-09 16:44:19,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +19: [2023-05-09 16:44:19,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +19: [2023-05-09 16:44:19,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +19: [2023-05-09 16:44:19,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +14: [2023-05-09 16:44:19,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +10: [2023-05-09 16:44:19,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +14: [2023-05-09 16:44:19,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +14: [2023-05-09 16:44:19,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +13: [2023-05-09 16:44:19,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +19: [2023-05-09 16:44:19,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +13: [2023-05-09 16:44:19,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +13: [2023-05-09 16:44:19,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +13: [2023-05-09 16:44:19,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +13: [2023-05-09 16:44:19,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +13: [2023-05-09 16:44:19,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +13: [2023-05-09 16:44:19,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +14: [2023-05-09 16:44:19,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 0: [2023-05-09 16:44:19,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +13: [2023-05-09 16:44:19,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +20: [2023-05-09 16:44:19,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +20: [2023-05-09 16:44:19,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +14: [2023-05-09 16:44:19,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +14: [2023-05-09 16:44:19,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +14: [2023-05-09 16:44:19,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +19: [2023-05-09 16:44:19,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +13: [2023-05-09 16:44:19,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +14: [2023-05-09 16:44:19,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +14: [2023-05-09 16:44:19,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +14: [2023-05-09 16:44:19,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +19: [2023-05-09 16:44:19,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +14: [2023-05-09 16:44:19,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +13: [2023-05-09 16:44:19,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +13: [2023-05-09 16:44:19,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +20: [2023-05-09 16:44:19,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +20: [2023-05-09 16:44:19,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +20: [2023-05-09 16:44:19,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +19: [2023-05-09 16:44:19,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +20: [2023-05-09 16:44:19,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 8: [2023-05-09 16:44:19,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 8: [2023-05-09 16:44:19,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 8: [2023-05-09 16:44:19,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 8: [2023-05-09 16:44:19,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +21: [2023-05-09 16:44:19,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +21: [2023-05-09 16:44:19,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +21: [2023-05-09 16:44:19,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +21: [2023-05-09 16:44:19,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +11: [2023-05-09 16:44:19,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +11: [2023-05-09 16:44:19,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +11: [2023-05-09 16:44:19,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +11: [2023-05-09 16:44:19,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +11: [2023-05-09 16:44:19,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +11: [2023-05-09 16:44:19,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 9: [2023-05-09 16:44:19,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +21: [2023-05-09 16:44:19,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +21: [2023-05-09 16:44:19,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +19: [2023-05-09 16:44:19,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +13: [2023-05-09 16:44:19,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +21: [2023-05-09 16:44:19,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +21: [2023-05-09 16:44:19,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +21: [2023-05-09 16:44:19,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +21: [2023-05-09 16:44:19,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +21: [2023-05-09 16:44:19,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +21: [2023-05-09 16:44:19,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +24: [2023-05-09 16:44:19,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +24: [2023-05-09 16:44:19,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +24: [2023-05-09 16:44:19,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +11: [2023-05-09 16:44:19,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +11: [2023-05-09 16:44:19,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +24: [2023-05-09 16:44:19,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 1: [2023-05-09 16:44:19,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 1: [2023-05-09 16:44:19,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +29: [2023-05-09 16:44:19,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +29: [2023-05-09 16:44:19,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +14: [2023-05-09 16:44:19,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +19: [2023-05-09 16:44:19,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +29: [2023-05-09 16:44:19,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +29: [2023-05-09 16:44:19,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +29: [2023-05-09 16:44:19,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 7: [2023-05-09 16:44:19,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 7: [2023-05-09 16:44:19,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +29: [2023-05-09 16:44:19,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +13: [2023-05-09 16:44:19,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 0: [2023-05-09 16:44:19,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 7: [2023-05-09 16:44:19,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 0: [2023-05-09 16:44:19,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +29: [2023-05-09 16:44:19,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 1: [2023-05-09 16:44:19,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +19: [2023-05-09 16:44:19,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +19: [2023-05-09 16:44:19,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +29: [2023-05-09 16:44:19,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +19: [2023-05-09 16:44:19,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +20: [2023-05-09 16:44:19,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +13: [2023-05-09 16:44:19,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +21: [2023-05-09 16:44:19,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +19: [2023-05-09 16:44:19,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 8: [2023-05-09 16:44:19,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +20: [2023-05-09 16:44:19,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +20: [2023-05-09 16:44:19,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +20: [2023-05-09 16:44:19,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +21: [2023-05-09 16:44:19,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +21: [2023-05-09 16:44:19,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +21: [2023-05-09 16:44:19,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +24: [2023-05-09 16:44:19,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +19: [2023-05-09 16:44:19,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +24: [2023-05-09 16:44:19,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +24: [2023-05-09 16:44:19,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +19: [2023-05-09 16:44:19,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +24: [2023-05-09 16:44:19,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +21: [2023-05-09 16:44:19,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +13: [2023-05-09 16:44:19,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +19: [2023-05-09 16:44:19,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +19: [2023-05-09 16:44:19,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 8: [2023-05-09 16:44:19,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 1: [2023-05-09 16:44:19,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +29: [2023-05-09 16:44:19,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +29: [2023-05-09 16:44:19,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +19: [2023-05-09 16:44:19,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +19: [2023-05-09 16:44:19,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +20: [2023-05-09 16:44:19,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +20: [2023-05-09 16:44:19,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +11: [2023-05-09 16:44:19,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +11: [2023-05-09 16:44:19,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +19: [2023-05-09 16:44:19,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +20: [2023-05-09 16:44:19,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +20: [2023-05-09 16:44:19,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +21: [2023-05-09 16:44:19,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +13: [2023-05-09 16:44:19,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +24: [2023-05-09 16:44:19,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +24: [2023-05-09 16:44:19,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +24: [2023-05-09 16:44:19,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +29: [2023-05-09 16:44:19,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +24: [2023-05-09 16:44:19,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +24: [2023-05-09 16:44:19,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +24: [2023-05-09 16:44:19,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +24: [2023-05-09 16:44:19,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 1: [2023-05-09 16:44:19,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +21: [2023-05-09 16:44:19,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +24: [2023-05-09 16:44:19,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 1: [2023-05-09 16:44:19,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +29: [2023-05-09 16:44:19,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +20: [2023-05-09 16:44:19,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +21: [2023-05-09 16:44:19,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +20: [2023-05-09 16:44:19,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +29: [2023-05-09 16:44:19,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 8: [2023-05-09 16:44:19,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 8: [2023-05-09 16:44:19,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 7: [2023-05-09 16:44:19,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 9: [2023-05-09 16:44:19,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +11: [2023-05-09 16:44:19,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +19: [2023-05-09 16:44:19,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +19: [2023-05-09 16:44:19,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 7: [2023-05-09 16:44:19,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 9: [2023-05-09 16:44:19,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +21: [2023-05-09 16:44:19,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +21: [2023-05-09 16:44:19,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +20: [2023-05-09 16:44:19,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +11: [2023-05-09 16:44:19,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +19: [2023-05-09 16:44:19,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +19: [2023-05-09 16:44:19,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +29: [2023-05-09 16:44:19,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +20: [2023-05-09 16:44:19,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 7: [2023-05-09 16:44:19,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +21: [2023-05-09 16:44:19,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +29: [2023-05-09 16:44:19,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +21: [2023-05-09 16:44:19,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +19: [2023-05-09 16:44:19,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +19: [2023-05-09 16:44:19,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +11: [2023-05-09 16:44:19,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +11: [2023-05-09 16:44:19,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 0: [2023-05-09 16:44:19,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 0: [2023-05-09 16:44:19,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +21: [2023-05-09 16:44:19,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +21: [2023-05-09 16:44:19,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +21: [2023-05-09 16:44:19,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +21: [2023-05-09 16:44:19,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 9: [2023-05-09 16:44:19,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +29: [2023-05-09 16:44:19,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +21: [2023-05-09 16:44:19,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +21: [2023-05-09 16:44:19,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +21: [2023-05-09 16:44:19,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +19: [2023-05-09 16:44:19,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +19: [2023-05-09 16:44:19,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +11: [2023-05-09 16:44:19,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +11: [2023-05-09 16:44:19,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +21: [2023-05-09 16:44:19,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +22: [2023-05-09 16:44:19,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +22: [2023-05-09 16:44:19,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +22: [2023-05-09 16:44:19,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +22: [2023-05-09 16:44:19,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +28: [2023-05-09 16:44:19,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +28: [2023-05-09 16:44:19,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +28: [2023-05-09 16:44:19,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +28: [2023-05-09 16:44:19,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +24: [2023-05-09 16:44:19,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +24: [2023-05-09 16:44:19,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +24: [2023-05-09 16:44:19,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +24: [2023-05-09 16:44:19,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +24: [2023-05-09 16:44:19,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +24: [2023-05-09 16:44:19,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +15: [2023-05-09 16:44:19,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +15: [2023-05-09 16:44:19,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +15: [2023-05-09 16:44:19,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +15: [2023-05-09 16:44:19,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +24: [2023-05-09 16:44:19,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +24: [2023-05-09 16:44:19,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +27: [2023-05-09 16:44:19,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +27: [2023-05-09 16:44:19,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 9: [2023-05-09 16:44:19,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +27: [2023-05-09 16:44:19,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 9: [2023-05-09 16:44:19,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +15: [2023-05-09 16:44:19,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +15: [2023-05-09 16:44:19,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +15: [2023-05-09 16:44:19,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +27: [2023-05-09 16:44:19,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +22: [2023-05-09 16:44:19,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +27: [2023-05-09 16:44:19,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +27: [2023-05-09 16:44:19,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +15: [2023-05-09 16:44:19,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +27: [2023-05-09 16:44:19,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +27: [2023-05-09 16:44:19,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +28: [2023-05-09 16:44:19,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +22: [2023-05-09 16:44:19,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +22: [2023-05-09 16:44:19,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +22: [2023-05-09 16:44:19,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +28: [2023-05-09 16:44:19,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +28: [2023-05-09 16:44:19,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +28: [2023-05-09 16:44:19,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 5: [2023-05-09 16:44:19,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 5: [2023-05-09 16:44:19,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 5: [2023-05-09 16:44:19,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +27: [2023-05-09 16:44:19,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +11: [2023-05-09 16:44:19,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +11: [2023-05-09 16:44:19,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +15: [2023-05-09 16:44:19,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +11: [2023-05-09 16:44:19,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +11: [2023-05-09 16:44:19,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +15: [2023-05-09 16:44:19,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +15: [2023-05-09 16:44:19,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 2: [2023-05-09 16:44:19,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 2: [2023-05-09 16:44:19,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 2: [2023-05-09 16:44:19,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 2: [2023-05-09 16:44:19,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +24: [2023-05-09 16:44:19,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +24: [2023-05-09 16:44:19,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +27: [2023-05-09 16:44:19,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +15: [2023-05-09 16:44:19,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +15: [2023-05-09 16:44:19,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +27: [2023-05-09 16:44:19,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +15: [2023-05-09 16:44:19,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +15: [2023-05-09 16:44:19,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +13: [2023-05-09 16:44:19,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +13: [2023-05-09 16:44:19,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +13: [2023-05-09 16:44:19,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +13: [2023-05-09 16:44:19,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +16: [2023-05-09 16:44:19,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +16: [2023-05-09 16:44:19,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 5: [2023-05-09 16:44:19,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +16: [2023-05-09 16:44:19,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +16: [2023-05-09 16:44:19,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +12: [2023-05-09 16:44:19,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +22: [2023-05-09 16:44:19,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +12: [2023-05-09 16:44:19,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +22: [2023-05-09 16:44:19,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +12: [2023-05-09 16:44:19,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +12: [2023-05-09 16:44:19,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +27: [2023-05-09 16:44:19,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +11: [2023-05-09 16:44:19,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +11: [2023-05-09 16:44:19,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +22: [2023-05-09 16:44:19,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +22: [2023-05-09 16:44:19,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +15: [2023-05-09 16:44:19,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +22: [2023-05-09 16:44:19,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +22: [2023-05-09 16:44:19,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 3: [2023-05-09 16:44:19,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 3: [2023-05-09 16:44:19,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 3: [2023-05-09 16:44:19,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +15: [2023-05-09 16:44:19,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +31: [2023-05-09 16:44:19,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +11: [2023-05-09 16:44:19,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +11: [2023-05-09 16:44:19,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +24: [2023-05-09 16:44:19,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +28: [2023-05-09 16:44:19,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +28: [2023-05-09 16:44:19,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +15: [2023-05-09 16:44:19,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +31: [2023-05-09 16:44:19,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +31: [2023-05-09 16:44:19,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +31: [2023-05-09 16:44:19,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 4: [2023-05-09 16:44:19,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +24: [2023-05-09 16:44:19,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 4: [2023-05-09 16:44:19,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 2: [2023-05-09 16:44:19,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +27: [2023-05-09 16:44:19,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +13: [2023-05-09 16:44:19,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +27: [2023-05-09 16:44:19,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +27: [2023-05-09 16:44:19,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +17: [2023-05-09 16:44:19,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 2: [2023-05-09 16:44:19,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +17: [2023-05-09 16:44:19,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +15: [2023-05-09 16:44:19,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +13: [2023-05-09 16:44:19,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +24: [2023-05-09 16:44:19,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +13: [2023-05-09 16:44:19,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +15: [2023-05-09 16:44:19,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +15: [2023-05-09 16:44:19,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +16: [2023-05-09 16:44:19,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +16: [2023-05-09 16:44:19,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +28: [2023-05-09 16:44:19,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +13: [2023-05-09 16:44:19,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +17: [2023-05-09 16:44:19,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +15: [2023-05-09 16:44:19,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +16: [2023-05-09 16:44:19,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +16: [2023-05-09 16:44:19,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +17: [2023-05-09 16:44:19,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +28: [2023-05-09 16:44:19,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +25: [2023-05-09 16:44:19,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +28: [2023-05-09 16:44:19,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +25: [2023-05-09 16:44:19,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +25: [2023-05-09 16:44:19,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +25: [2023-05-09 16:44:19,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +12: [2023-05-09 16:44:19,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +28: [2023-05-09 16:44:19,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +30: [2023-05-09 16:44:19,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +30: [2023-05-09 16:44:19,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 5: [2023-05-09 16:44:19,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +30: [2023-05-09 16:44:19,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. + 3: [2023-05-09 16:44:19,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 5: [2023-05-09 16:44:19,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 5: [2023-05-09 16:44:19,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 3: [2023-05-09 16:44:19,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +22: [2023-05-09 16:44:19,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +12: [2023-05-09 16:44:19,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +30: [2023-05-09 16:44:19,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +24: [2023-05-09 16:44:19,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +25: [2023-05-09 16:44:19,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +12: [2023-05-09 16:44:19,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +23: [2023-05-09 16:44:19,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +25: [2023-05-09 16:44:19,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +23: [2023-05-09 16:44:19,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +23: [2023-05-09 16:44:19,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +15: [2023-05-09 16:44:19,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +23: [2023-05-09 16:44:19,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt. +12: [2023-05-09 16:44:19,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +22: [2023-05-09 16:44:19,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +27: [2023-05-09 16:44:19,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +25: [2023-05-09 16:44:19,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +25: [2023-05-09 16:44:19,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +24: [2023-05-09 16:44:19,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +11: [2023-05-09 16:44:19,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +11: [2023-05-09 16:44:19,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 4: [2023-05-09 16:44:19,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +17: [2023-05-09 16:44:19,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +31: [2023-05-09 16:44:19,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +31: [2023-05-09 16:44:19,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +31: [2023-05-09 16:44:19,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +25: [2023-05-09 16:44:19,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +25: [2023-05-09 16:44:19,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +18: [2023-05-09 16:44:19,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +17: [2023-05-09 16:44:19,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +25: [2023-05-09 16:44:19,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +25: [2023-05-09 16:44:19,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +18: [2023-05-09 16:44:19,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +31: [2023-05-09 16:44:19,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +18: [2023-05-09 16:44:19,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 2: [2023-05-09 16:44:19,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +28: [2023-05-09 16:44:19,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +16: [2023-05-09 16:44:19,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +18: [2023-05-09 16:44:19,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +26: [2023-05-09 16:44:19,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +26: [2023-05-09 16:44:19,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +30: [2023-05-09 16:44:19,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +13: [2023-05-09 16:44:19,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +26: [2023-05-09 16:44:19,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +28: [2023-05-09 16:44:19,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +30: [2023-05-09 16:44:19,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +16: [2023-05-09 16:44:19,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 3: [2023-05-09 16:44:19,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 2: [2023-05-09 16:44:19,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +25: [2023-05-09 16:44:19,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +30: [2023-05-09 16:44:19,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +25: [2023-05-09 16:44:19,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +15: [2023-05-09 16:44:19,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +13: [2023-05-09 16:44:19,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 3: [2023-05-09 16:44:19,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +25: [2023-05-09 16:44:19,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +25: [2023-05-09 16:44:19,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +15: [2023-05-09 16:44:19,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +15: [2023-05-09 16:44:19,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +30: [2023-05-09 16:44:19,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +26: [2023-05-09 16:44:19,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 5: [2023-05-09 16:44:19,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +24: [2023-05-09 16:44:19,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +23: [2023-05-09 16:44:19,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +23: [2023-05-09 16:44:19,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +23: [2023-05-09 16:44:19,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +15: [2023-05-09 16:44:19,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +15: [2023-05-09 16:44:19,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 5: [2023-05-09 16:44:19,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +17: [2023-05-09 16:44:19,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +17: [2023-05-09 16:44:19,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +25: [2023-05-09 16:44:19,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 5: [2023-05-09 16:44:19,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 5: [2023-05-09 16:44:19,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +23: [2023-05-09 16:44:19,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt... +16: [2023-05-09 16:44:19,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +16: [2023-05-09 16:44:19,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +13: [2023-05-09 16:44:19,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +18: [2023-05-09 16:44:19,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +13: [2023-05-09 16:44:19,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +25: [2023-05-09 16:44:19,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +18: [2023-05-09 16:44:19,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +11: [2023-05-09 16:44:19,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +26: [2023-05-09 16:44:19,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +26: [2023-05-09 16:44:19,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +10: [2023-05-09 16:44:19,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +25: [2023-05-09 16:44:19,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +11: [2023-05-09 16:44:19,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +27: [2023-05-09 16:44:19,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 1: [2023-05-09 16:44:19,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +18: [2023-05-09 16:44:19,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +13: [2023-05-09 16:44:19,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 1: [2023-05-09 16:44:19,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +18: [2023-05-09 16:44:19,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +26: [2023-05-09 16:44:19,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +26: [2023-05-09 16:44:19,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +11: [2023-05-09 16:44:19,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +30: [2023-05-09 16:44:19,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +13: [2023-05-09 16:44:19,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +13: [2023-05-09 16:44:19,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +15: [2023-05-09 16:44:19,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +30: [2023-05-09 16:44:19,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +13: [2023-05-09 16:44:19,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +10: [2023-05-09 16:44:19,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +11: [2023-05-09 16:44:19,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +27: [2023-05-09 16:44:19,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +15: [2023-05-09 16:44:19,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 2: [2023-05-09 16:44:19,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +27: [2023-05-09 16:44:19,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +11: [2023-05-09 16:44:19,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +25: [2023-05-09 16:44:19,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +16: [2023-05-09 16:44:19,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +16: [2023-05-09 16:44:19,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +30: [2023-05-09 16:44:19,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +30: [2023-05-09 16:44:19,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +30: [2023-05-09 16:44:19,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +30: [2023-05-09 16:44:19,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 4: [2023-05-09 16:44:19,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +11: [2023-05-09 16:44:19,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +16: [2023-05-09 16:44:19,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +10: [2023-05-09 16:44:19,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +16: [2023-05-09 16:44:19,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 3: [2023-05-09 16:44:19,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +27: [2023-05-09 16:44:19,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 3: [2023-05-09 16:44:19,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +23: [2023-05-09 16:44:19,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +23: [2023-05-09 16:44:19,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +28: [2023-05-09 16:44:19,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +28: [2023-05-09 16:44:19,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +23: [2023-05-09 16:44:19,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +28: [2023-05-09 16:44:19,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +28: [2023-05-09 16:44:19,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +28: [2023-05-09 16:44:19,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +23: [2023-05-09 16:44:19,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 3: [2023-05-09 16:44:19,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +28: [2023-05-09 16:44:19,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +28: [2023-05-09 16:44:19,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +28: [2023-05-09 16:44:19,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 3: [2023-05-09 16:44:19,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +15: [2023-05-09 16:44:19,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +27: [2023-05-09 16:44:19,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +10: [2023-05-09 16:44:19,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +10: [2023-05-09 16:44:19,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +31: [2023-05-09 16:44:19,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +31: [2023-05-09 16:44:19,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +31: [2023-05-09 16:44:19,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +23: [2023-05-09 16:44:19,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +15: [2023-05-09 16:44:19,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 1: [2023-05-09 16:44:19,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +23: [2023-05-09 16:44:19,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +23: [2023-05-09 16:44:19,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +23: [2023-05-09 16:44:19,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +10: [2023-05-09 16:44:19,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +31: [2023-05-09 16:44:19,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 7: [2023-05-09 16:44:19,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +27: [2023-05-09 16:44:19,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 4: [2023-05-09 16:44:19,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 4: [2023-05-09 16:44:19,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +27: [2023-05-09 16:44:19,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +31: [2023-05-09 16:44:19,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +31: [2023-05-09 16:44:19,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +27: [2023-05-09 16:44:19,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +30: [2023-05-09 16:44:19,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +31: [2023-05-09 16:44:19,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +30: [2023-05-09 16:44:19,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 3: [2023-05-09 16:44:19,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +31: [2023-05-09 16:44:19,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +10: [2023-05-09 16:44:19,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 3: [2023-05-09 16:44:19,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +10: [2023-05-09 16:44:19,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +25: [2023-05-09 16:44:19,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +28: [2023-05-09 16:44:19,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 3: [2023-05-09 16:44:19,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 1: [2023-05-09 16:44:19,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +28: [2023-05-09 16:44:19,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +27: [2023-05-09 16:44:19,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +27: [2023-05-09 16:44:19,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +28: [2023-05-09 16:44:19,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +25: [2023-05-09 16:44:19,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +25: [2023-05-09 16:44:19,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +25: [2023-05-09 16:44:19,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +25: [2023-05-09 16:44:19,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... +25: [2023-05-09 16:44:19,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt. +28: [2023-05-09 16:44:19,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +25: [2023-05-09 16:44:19,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt... + 7: [2023-05-09 16:44:19,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +25: [2023-05-09 16:44:19,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 1: [2023-05-09 16:44:19,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 1: [2023-05-09 16:44:19,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 1: [2023-05-09 16:44:19,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +27: [2023-05-09 16:44:19,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +27: [2023-05-09 16:44:19,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +28: [2023-05-09 16:44:19,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +27: [2023-05-09 16:44:19,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +27: [2023-05-09 16:44:19,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +28: [2023-05-09 16:44:19,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +28: [2023-05-09 16:44:19,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +27: [2023-05-09 16:44:19,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +27: [2023-05-09 16:44:19,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +25: [2023-05-09 16:44:19,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +28: [2023-05-09 16:44:19,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +25: [2023-05-09 16:44:19,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +25: [2023-05-09 16:44:19,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 7: [2023-05-09 16:44:19,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 6: [2023-05-09 16:44:19,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +31: [2023-05-09 16:44:19,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +31: [2023-05-09 16:44:19,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +31: [2023-05-09 16:44:19,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +31: [2023-05-09 16:44:19,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +31: [2023-05-09 16:44:19,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +31: [2023-05-09 16:44:19,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +31: [2023-05-09 16:44:19,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +31: [2023-05-09 16:44:19,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +25: [2023-05-09 16:44:19,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +22: [2023-05-09 16:44:19,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +22: [2023-05-09 16:44:19,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +22: [2023-05-09 16:44:19,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +22: [2023-05-09 16:44:19,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +22: [2023-05-09 16:44:19,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +22: [2023-05-09 16:44:19,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +22: [2023-05-09 16:44:19,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +22: [2023-05-09 16:44:19,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 7: [2023-05-09 16:44:19,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 6: [2023-05-09 16:44:19,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +14: [2023-05-09 16:44:19,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +14: [2023-05-09 16:44:19,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +14: [2023-05-09 16:44:19,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +14: [2023-05-09 16:44:19,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +14: [2023-05-09 16:44:19,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +12: [2023-05-09 16:44:19,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +12: [2023-05-09 16:44:19,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +12: [2023-05-09 16:44:19,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +12: [2023-05-09 16:44:19,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +12: [2023-05-09 16:44:19,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +12: [2023-05-09 16:44:19,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +12: [2023-05-09 16:44:19,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +12: [2023-05-09 16:44:19,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +12: [2023-05-09 16:44:19,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +12: [2023-05-09 16:44:19,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +12: [2023-05-09 16:44:19,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +14: [2023-05-09 16:44:19,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +12: [2023-05-09 16:44:19,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +12: [2023-05-09 16:44:19,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +17: [2023-05-09 16:44:19,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +14: [2023-05-09 16:44:19,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +17: [2023-05-09 16:44:19,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +17: [2023-05-09 16:44:19,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +20: [2023-05-09 16:44:19,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +20: [2023-05-09 16:44:19,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 0: [2023-05-09 16:44:19,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +20: [2023-05-09 16:44:19,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +12: [2023-05-09 16:44:19,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +17: [2023-05-09 16:44:19,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +14: [2023-05-09 16:44:19,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +20: [2023-05-09 16:44:19,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +20: [2023-05-09 16:44:19,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +12: [2023-05-09 16:44:19,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +17: [2023-05-09 16:44:19,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +20: [2023-05-09 16:44:19,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +17: [2023-05-09 16:44:19,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +17: [2023-05-09 16:44:19,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +17: [2023-05-09 16:44:19,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +17: [2023-05-09 16:44:19,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +12: [2023-05-09 16:44:19,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +17: [2023-05-09 16:44:19,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +30: [2023-05-09 16:44:19,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +30: [2023-05-09 16:44:19,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +30: [2023-05-09 16:44:19,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +30: [2023-05-09 16:44:19,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +30: [2023-05-09 16:44:19,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +13: [2023-05-09 16:44:19,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +13: [2023-05-09 16:44:19,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +30: [2023-05-09 16:44:19,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +13: [2023-05-09 16:44:19,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +13: [2023-05-09 16:44:19,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +30: [2023-05-09 16:44:19,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +17: [2023-05-09 16:44:19,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +17: [2023-05-09 16:44:19,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +17: [2023-05-09 16:44:19,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +17: [2023-05-09 16:44:19,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +20: [2023-05-09 16:44:19,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +17: [2023-05-09 16:44:19,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +20: [2023-05-09 16:44:19,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +17: [2023-05-09 16:44:19,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +30: [2023-05-09 16:44:19,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +31: [2023-05-09 16:44:19,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +31: [2023-05-09 16:44:19,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +13: [2023-05-09 16:44:19,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +13: [2023-05-09 16:44:19,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 4: [2023-05-09 16:44:19,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 4: [2023-05-09 16:44:19,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +31: [2023-05-09 16:44:19,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +13: [2023-05-09 16:44:19,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +18: [2023-05-09 16:44:19,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +13: [2023-05-09 16:44:19,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 2: [2023-05-09 16:44:19,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 2: [2023-05-09 16:44:19,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +18: [2023-05-09 16:44:19,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +18: [2023-05-09 16:44:19,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +18: [2023-05-09 16:44:19,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +18: [2023-05-09 16:44:19,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +18: [2023-05-09 16:44:19,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +18: [2023-05-09 16:44:19,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +18: [2023-05-09 16:44:19,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +18: [2023-05-09 16:44:19,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +18: [2023-05-09 16:44:19,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +18: [2023-05-09 16:44:19,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +18: [2023-05-09 16:44:19,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +18: [2023-05-09 16:44:19,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +18: [2023-05-09 16:44:19,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 6: [2023-05-09 16:44:19,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +10: [2023-05-09 16:44:19,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +10: [2023-05-09 16:44:19,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +23: [2023-05-09 16:44:19,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +23: [2023-05-09 16:44:19,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +10: [2023-05-09 16:44:19,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +10: [2023-05-09 16:44:19,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +23: [2023-05-09 16:44:19,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +23: [2023-05-09 16:44:19,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +10: [2023-05-09 16:44:19,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +10: [2023-05-09 16:44:19,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +10: [2023-05-09 16:44:19,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +10: [2023-05-09 16:44:19,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +10: [2023-05-09 16:44:19,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +10: [2023-05-09 16:44:19,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +23: [2023-05-09 16:44:19,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +18: [2023-05-09 16:44:19,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +10: [2023-05-09 16:44:19,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +10: [2023-05-09 16:44:19,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +10: [2023-05-09 16:44:19,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +10: [2023-05-09 16:44:19,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +10: [2023-05-09 16:44:19,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +23: [2023-05-09 16:44:19,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +10: [2023-05-09 16:44:19,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +18: [2023-05-09 16:44:19,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +23: [2023-05-09 16:44:19,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +23: [2023-05-09 16:44:19,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +22: [2023-05-09 16:44:19,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +26: [2023-05-09 16:44:19,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +26: [2023-05-09 16:44:19,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +26: [2023-05-09 16:44:19,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +26: [2023-05-09 16:44:19,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +26: [2023-05-09 16:44:19,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +26: [2023-05-09 16:44:19,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +26: [2023-05-09 16:44:19,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +26: [2023-05-09 16:44:19,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +26: [2023-05-09 16:44:19,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +26: [2023-05-09 16:44:19,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +26: [2023-05-09 16:44:19,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +26: [2023-05-09 16:44:19,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +26: [2023-05-09 16:44:19,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +26: [2023-05-09 16:44:19,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +22: [2023-05-09 16:44:19,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +26: [2023-05-09 16:44:19,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +26: [2023-05-09 16:44:19,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +12: [2023-05-09 16:44:19,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +17: [2023-05-09 16:44:19,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +14: [2023-05-09 16:44:19,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +14: [2023-05-09 16:44:19,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +22: [2023-05-09 16:44:19,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +31: [2023-05-09 16:44:19,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +22: [2023-05-09 16:44:19,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +20: [2023-05-09 16:44:19,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +31: [2023-05-09 16:44:19,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +31: [2023-05-09 16:44:19,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 0: [2023-05-09 16:44:19,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +17: [2023-05-09 16:44:20,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +17: [2023-05-09 16:44:20,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +30: [2023-05-09 16:44:20,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +30: [2023-05-09 16:44:20,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +30: [2023-05-09 16:44:20,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +22: [2023-05-09 16:44:20,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +18: [2023-05-09 16:44:20,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +22: [2023-05-09 16:44:20,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +10: [2023-05-09 16:44:20,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +12: [2023-05-09 16:44:20,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +12: [2023-05-09 16:44:20,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +23: [2023-05-09 16:44:20,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +31: [2023-05-09 16:44:20,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +23: [2023-05-09 16:44:20,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +13: [2023-05-09 16:44:20,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +13: [2023-05-09 16:44:20,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +12: [2023-05-09 16:44:20,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +14: [2023-05-09 16:44:20,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +20: [2023-05-09 16:44:20,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +26: [2023-05-09 16:44:20,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +19: [2023-05-09 16:44:20,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +19: [2023-05-09 16:44:20,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +23: [2023-05-09 16:44:20,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +23: [2023-05-09 16:44:20,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +14: [2023-05-09 16:44:20,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +18: [2023-05-09 16:44:20,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 6: [2023-05-09 16:44:20,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +19: [2023-05-09 16:44:20,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +19: [2023-05-09 16:44:20,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +19: [2023-05-09 16:44:20,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +19: [2023-05-09 16:44:20,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +19: [2023-05-09 16:44:20,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +19: [2023-05-09 16:44:20,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +22: [2023-05-09 16:44:20,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +22: [2023-05-09 16:44:20,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +10: [2023-05-09 16:44:20,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +17: [2023-05-09 16:44:20,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +14: [2023-05-09 16:44:20,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +26: [2023-05-09 16:44:20,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +26: [2023-05-09 16:44:20,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +14: [2023-05-09 16:44:20,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +18: [2023-05-09 16:44:20,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +17: [2023-05-09 16:44:20,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +18: [2023-05-09 16:44:20,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +30: [2023-05-09 16:44:20,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +17: [2023-05-09 16:44:20,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +30: [2023-05-09 16:44:20,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +30: [2023-05-09 16:44:20,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +17: [2023-05-09 16:44:20,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +10: [2023-05-09 16:44:20,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +30: [2023-05-09 16:44:20,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +10: [2023-05-09 16:44:20,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +10: [2023-05-09 16:44:20,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +20: [2023-05-09 16:44:20,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +20: [2023-05-09 16:44:20,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +31: [2023-05-09 16:44:20,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +12: [2023-05-09 16:44:20,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +12: [2023-05-09 16:44:20,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +23: [2023-05-09 16:44:20,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +23: [2023-05-09 16:44:20,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +20: [2023-05-09 16:44:20,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +13: [2023-05-09 16:44:20,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +23: [2023-05-09 16:44:20,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +23: [2023-05-09 16:44:20,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +26: [2023-05-09 16:44:20,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +26: [2023-05-09 16:44:20,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +14: [2023-05-09 16:44:20,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +14: [2023-05-09 16:44:20,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +26: [2023-05-09 16:44:20,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +10: [2023-05-09 16:44:20,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +18: [2023-05-09 16:44:20,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +18: [2023-05-09 16:44:20,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +19: [2023-05-09 16:44:20,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +13: [2023-05-09 16:44:20,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +13: [2023-05-09 16:44:20,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +18: [2023-05-09 16:44:20,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +30: [2023-05-09 16:44:20,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +17: [2023-05-09 16:44:20,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +12: [2023-05-09 16:44:20,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +26: [2023-05-09 16:44:20,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +10: [2023-05-09 16:44:20,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +13: [2023-05-09 16:44:20,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +20: [2023-05-09 16:44:20,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +20: [2023-05-09 16:44:20,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +10: [2023-05-09 16:44:20,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +20: [2023-05-09 16:44:20,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +13: [2023-05-09 16:44:20,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +19: [2023-05-09 16:44:20,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +19: [2023-05-09 16:44:20,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +19: [2023-05-09 16:44:20,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +19: [2023-05-09 16:44:20,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +18: [2023-05-09 16:44:20,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +12: [2023-05-09 16:44:20,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +26: [2023-05-09 16:44:20,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +13: [2023-05-09 16:44:20,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +29: [2023-05-09 16:44:20,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 6: [2023-05-09 16:44:20,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +29: [2023-05-09 16:44:20,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +29: [2023-05-09 16:44:20,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +29: [2023-05-09 16:44:20,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 5: [2023-05-09 16:44:20,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +19: [2023-05-09 16:44:20,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +19: [2023-05-09 16:44:20,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 5: [2023-05-09 16:44:20,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 8: [2023-05-09 16:44:20,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +19: [2023-05-09 16:44:20,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 8: [2023-05-09 16:44:20,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +29: [2023-05-09 16:44:20,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +29: [2023-05-09 16:44:20,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +29: [2023-05-09 16:44:20,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +29: [2023-05-09 16:44:20,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +14: [2023-05-09 16:44:20,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +14: [2023-05-09 16:44:20,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +14: [2023-05-09 16:44:20,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +14: [2023-05-09 16:44:20,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +19: [2023-05-09 16:44:20,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +19: [2023-05-09 16:44:20,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +19: [2023-05-09 16:44:20,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +19: [2023-05-09 16:44:20,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 6: [2023-05-09 16:44:20,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 8: [2023-05-09 16:44:20,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +29: [2023-05-09 16:44:20,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +29: [2023-05-09 16:44:20,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +21: [2023-05-09 16:44:20,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +21: [2023-05-09 16:44:20,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +21: [2023-05-09 16:44:20,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +21: [2023-05-09 16:44:20,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +21: [2023-05-09 16:44:20,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +21: [2023-05-09 16:44:20,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +21: [2023-05-09 16:44:20,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +14: [2023-05-09 16:44:20,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +19: [2023-05-09 16:44:20,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +14: [2023-05-09 16:44:20,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +14: [2023-05-09 16:44:20,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +21: [2023-05-09 16:44:20,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +14: [2023-05-09 16:44:20,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +19: [2023-05-09 16:44:20,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +29: [2023-05-09 16:44:20,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +19: [2023-05-09 16:44:20,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +29: [2023-05-09 16:44:20,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +11: [2023-05-09 16:44:20,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +11: [2023-05-09 16:44:20,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +11: [2023-05-09 16:44:20,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +11: [2023-05-09 16:44:20,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +11: [2023-05-09 16:44:20,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +11: [2023-05-09 16:44:20,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +11: [2023-05-09 16:44:20,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +19: [2023-05-09 16:44:20,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +29: [2023-05-09 16:44:20,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +29: [2023-05-09 16:44:20,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +11: [2023-05-09 16:44:20,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +29: [2023-05-09 16:44:20,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +29: [2023-05-09 16:44:20,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 8: [2023-05-09 16:44:20,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +16: [2023-05-09 16:44:20,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +16: [2023-05-09 16:44:20,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +16: [2023-05-09 16:44:20,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +16: [2023-05-09 16:44:20,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +16: [2023-05-09 16:44:20,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +16: [2023-05-09 16:44:20,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +16: [2023-05-09 16:44:20,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +16: [2023-05-09 16:44:20,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +21: [2023-05-09 16:44:20,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +14: [2023-05-09 16:44:20,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +14: [2023-05-09 16:44:20,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 8: [2023-05-09 16:44:20,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +19: [2023-05-09 16:44:20,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +19: [2023-05-09 16:44:20,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +19: [2023-05-09 16:44:20,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 8: [2023-05-09 16:44:20,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 8: [2023-05-09 16:44:20,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +19: [2023-05-09 16:44:20,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 8: [2023-05-09 16:44:20,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 9: [2023-05-09 16:44:20,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 9: [2023-05-09 16:44:20,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 9: [2023-05-09 16:44:20,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 9: [2023-05-09 16:44:20,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +14: [2023-05-09 16:44:20,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +14: [2023-05-09 16:44:20,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +14: [2023-05-09 16:44:20,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +14: [2023-05-09 16:44:20,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 1: [2023-05-09 16:44:20,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +21: [2023-05-09 16:44:20,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +21: [2023-05-09 16:44:20,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +16: [2023-05-09 16:44:20,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +21: [2023-05-09 16:44:20,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +16: [2023-05-09 16:44:20,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +11: [2023-05-09 16:44:20,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +11: [2023-05-09 16:44:20,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +11: [2023-05-09 16:44:20,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +14: [2023-05-09 16:44:20,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +14: [2023-05-09 16:44:20,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +19: [2023-05-09 16:44:20,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +24: [2023-05-09 16:44:20,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +24: [2023-05-09 16:44:20,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +24: [2023-05-09 16:44:20,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +24: [2023-05-09 16:44:20,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 8: [2023-05-09 16:44:20,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +19: [2023-05-09 16:44:20,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 9: [2023-05-09 16:44:20,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +21: [2023-05-09 16:44:20,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +21: [2023-05-09 16:44:20,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +16: [2023-05-09 16:44:20,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +16: [2023-05-09 16:44:20,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +19: [2023-05-09 16:44:20,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +19: [2023-05-09 16:44:20,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +11: [2023-05-09 16:44:20,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +21: [2023-05-09 16:44:20,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +21: [2023-05-09 16:44:20,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +21: [2023-05-09 16:44:20,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +21: [2023-05-09 16:44:20,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +21: [2023-05-09 16:44:20,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +11: [2023-05-09 16:44:20,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +11: [2023-05-09 16:44:20,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +24: [2023-05-09 16:44:20,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +24: [2023-05-09 16:44:20,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +16: [2023-05-09 16:44:20,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +16: [2023-05-09 16:44:20,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +24: [2023-05-09 16:44:20,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +24: [2023-05-09 16:44:20,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +28: [2023-05-09 16:44:20,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +28: [2023-05-09 16:44:20,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 9: [2023-05-09 16:44:20,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +21: [2023-05-09 16:44:20,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +21: [2023-05-09 16:44:20,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +21: [2023-05-09 16:44:20,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +21: [2023-05-09 16:44:20,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +28: [2023-05-09 16:44:20,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +28: [2023-05-09 16:44:20,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +11: [2023-05-09 16:44:20,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +21: [2023-05-09 16:44:20,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +16: [2023-05-09 16:44:20,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +16: [2023-05-09 16:44:20,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +24: [2023-05-09 16:44:20,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +24: [2023-05-09 16:44:20,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +24: [2023-05-09 16:44:20,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +28: [2023-05-09 16:44:20,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +28: [2023-05-09 16:44:20,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +24: [2023-05-09 16:44:20,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +28: [2023-05-09 16:44:20,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +28: [2023-05-09 16:44:20,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +11: [2023-05-09 16:44:20,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +24: [2023-05-09 16:44:20,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +24: [2023-05-09 16:44:20,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +24: [2023-05-09 16:44:20,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +24: [2023-05-09 16:44:20,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +21: [2023-05-09 16:44:20,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +21: [2023-05-09 16:44:20,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +21: [2023-05-09 16:44:20,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +21: [2023-05-09 16:44:20,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +21: [2023-05-09 16:44:20,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +21: [2023-05-09 16:44:20,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 9: [2023-05-09 16:44:20,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 9: [2023-05-09 16:44:20,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +29: [2023-05-09 16:44:20,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +29: [2023-05-09 16:44:20,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +29: [2023-05-09 16:44:20,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +29: [2023-05-09 16:44:20,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +29: [2023-05-09 16:44:20,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +29: [2023-05-09 16:44:20,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +29: [2023-05-09 16:44:20,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +29: [2023-05-09 16:44:20,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +28: [2023-05-09 16:44:20,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +28: [2023-05-09 16:44:20,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +21: [2023-05-09 16:44:20,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +28: [2023-05-09 16:44:20,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +21: [2023-05-09 16:44:20,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +28: [2023-05-09 16:44:20,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +28: [2023-05-09 16:44:20,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +28: [2023-05-09 16:44:20,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 5: [2023-05-09 16:44:20,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 5: [2023-05-09 16:44:20,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 5: [2023-05-09 16:44:20,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +28: [2023-05-09 16:44:20,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +28: [2023-05-09 16:44:20,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 5: [2023-05-09 16:44:20,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 0: [2023-05-09 16:44:20,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 0: [2023-05-09 16:44:20,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 0: [2023-05-09 16:44:20,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 0: [2023-05-09 16:44:20,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 7: [2023-05-09 16:44:20,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 7: [2023-05-09 16:44:20,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 7: [2023-05-09 16:44:20,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +22: [2023-05-09 16:44:20,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +22: [2023-05-09 16:44:20,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +22: [2023-05-09 16:44:20,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 7: [2023-05-09 16:44:20,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +22: [2023-05-09 16:44:20,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +20: [2023-05-09 16:44:20,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +20: [2023-05-09 16:44:20,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +20: [2023-05-09 16:44:20,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +20: [2023-05-09 16:44:20,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +29: [2023-05-09 16:44:20,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +29: [2023-05-09 16:44:20,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +29: [2023-05-09 16:44:20,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +13: [2023-05-09 16:44:20,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 0: [2023-05-09 16:44:20,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +13: [2023-05-09 16:44:20,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 7: [2023-05-09 16:44:20,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +22: [2023-05-09 16:44:20,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +22: [2023-05-09 16:44:20,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +22: [2023-05-09 16:44:20,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +20: [2023-05-09 16:44:20,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +20: [2023-05-09 16:44:20,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +13: [2023-05-09 16:44:20,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +13: [2023-05-09 16:44:20,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +20: [2023-05-09 16:44:20,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +20: [2023-05-09 16:44:20,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +22: [2023-05-09 16:44:20,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +29: [2023-05-09 16:44:20,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +29: [2023-05-09 16:44:20,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +29: [2023-05-09 16:44:20,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +13: [2023-05-09 16:44:20,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 5: [2023-05-09 16:44:20,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 5: [2023-05-09 16:44:20,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +13: [2023-05-09 16:44:20,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +29: [2023-05-09 16:44:20,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +13: [2023-05-09 16:44:20,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +13: [2023-05-09 16:44:20,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +20: [2023-05-09 16:44:20,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +20: [2023-05-09 16:44:20,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +22: [2023-05-09 16:44:20,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 7: [2023-05-09 16:44:20,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 7: [2023-05-09 16:44:20,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 7: [2023-05-09 16:44:20,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +22: [2023-05-09 16:44:20,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +29: [2023-05-09 16:44:20,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +22: [2023-05-09 16:44:20,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +13: [2023-05-09 16:44:20,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +20: [2023-05-09 16:44:20,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +20: [2023-05-09 16:44:20,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +20: [2023-05-09 16:44:20,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +22: [2023-05-09 16:44:20,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +22: [2023-05-09 16:44:20,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +22: [2023-05-09 16:44:20,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 0: [2023-05-09 16:44:20,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 0: [2023-05-09 16:44:20,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +20: [2023-05-09 16:44:20,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +13: [2023-05-09 16:44:20,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +13: [2023-05-09 16:44:20,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +20: [2023-05-09 16:44:20,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +20: [2023-05-09 16:44:20,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +13: [2023-05-09 16:44:20,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +13: [2023-05-09 16:44:20,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +13: [2023-05-09 16:44:20,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 0: [2023-05-09 16:44:20,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +13: [2023-05-09 16:44:20,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +13: [2023-05-09 16:44:20,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +22: [2023-05-09 16:44:20,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +22: [2023-05-09 16:44:20,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 0: [2023-05-09 16:44:20,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 7: [2023-05-09 16:44:20,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +25: [2023-05-09 16:44:20,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +25: [2023-05-09 16:44:20,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +25: [2023-05-09 16:44:20,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +25: [2023-05-09 16:44:20,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +25: [2023-05-09 16:44:20,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +25: [2023-05-09 16:44:20,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +25: [2023-05-09 16:44:20,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +28: [2023-05-09 16:44:20,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +28: [2023-05-09 16:44:20,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +28: [2023-05-09 16:44:20,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +28: [2023-05-09 16:44:20,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +28: [2023-05-09 16:44:20,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +28: [2023-05-09 16:44:20,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +25: [2023-05-09 16:44:20,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +28: [2023-05-09 16:44:20,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +28: [2023-05-09 16:44:20,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +30: [2023-05-09 16:44:20,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +30: [2023-05-09 16:44:20,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +30: [2023-05-09 16:44:20,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +30: [2023-05-09 16:44:20,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +30: [2023-05-09 16:44:20,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +30: [2023-05-09 16:44:20,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +30: [2023-05-09 16:44:20,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +30: [2023-05-09 16:44:20,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +15: [2023-05-09 16:44:20,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +15: [2023-05-09 16:44:20,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +15: [2023-05-09 16:44:20,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +15: [2023-05-09 16:44:20,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +15: [2023-05-09 16:44:20,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +15: [2023-05-09 16:44:20,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +15: [2023-05-09 16:44:20,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +28: [2023-05-09 16:44:20,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +15: [2023-05-09 16:44:20,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +20: [2023-05-09 16:44:20,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +20: [2023-05-09 16:44:20,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +20: [2023-05-09 16:44:20,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +20: [2023-05-09 16:44:20,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +20: [2023-05-09 16:44:20,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +20: [2023-05-09 16:44:20,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +20: [2023-05-09 16:44:20,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +20: [2023-05-09 16:44:20,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +25: [2023-05-09 16:44:20,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +25: [2023-05-09 16:44:20,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +25: [2023-05-09 16:44:20,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +30: [2023-05-09 16:44:20,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +30: [2023-05-09 16:44:20,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +28: [2023-05-09 16:44:20,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +31: [2023-05-09 16:44:20,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +31: [2023-05-09 16:44:20,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 3: [2023-05-09 16:44:20,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +15: [2023-05-09 16:44:20,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +15: [2023-05-09 16:44:20,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +31: [2023-05-09 16:44:20,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 7: [2023-05-09 16:44:20,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +31: [2023-05-09 16:44:20,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +28: [2023-05-09 16:44:20,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +30: [2023-05-09 16:44:20,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +30: [2023-05-09 16:44:20,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +28: [2023-05-09 16:44:20,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +27: [2023-05-09 16:44:20,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +27: [2023-05-09 16:44:20,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +27: [2023-05-09 16:44:20,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +27: [2023-05-09 16:44:20,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +27: [2023-05-09 16:44:20,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +27: [2023-05-09 16:44:20,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +27: [2023-05-09 16:44:20,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +27: [2023-05-09 16:44:20,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +25: [2023-05-09 16:44:20,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +30: [2023-05-09 16:44:20,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +25: [2023-05-09 16:44:20,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +10: [2023-05-09 16:44:20,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +10: [2023-05-09 16:44:20,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +10: [2023-05-09 16:44:20,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +10: [2023-05-09 16:44:20,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +15: [2023-05-09 16:44:20,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +19: [2023-05-09 16:44:20,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +10: [2023-05-09 16:44:20,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +10: [2023-05-09 16:44:20,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +19: [2023-05-09 16:44:20,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +19: [2023-05-09 16:44:20,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +19: [2023-05-09 16:44:20,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +19: [2023-05-09 16:44:20,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +19: [2023-05-09 16:44:20,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +19: [2023-05-09 16:44:20,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +20: [2023-05-09 16:44:20,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +15: [2023-05-09 16:44:20,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +25: [2023-05-09 16:44:20,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +20: [2023-05-09 16:44:20,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +28: [2023-05-09 16:44:20,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +19: [2023-05-09 16:44:20,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +31: [2023-05-09 16:44:20,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +31: [2023-05-09 16:44:20,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +31: [2023-05-09 16:44:20,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +30: [2023-05-09 16:44:20,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +31: [2023-05-09 16:44:20,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +10: [2023-05-09 16:44:20,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +10: [2023-05-09 16:44:20,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +30: [2023-05-09 16:44:20,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +24: [2023-05-09 16:44:20,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +24: [2023-05-09 16:44:20,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +24: [2023-05-09 16:44:20,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +15: [2023-05-09 16:44:20,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +24: [2023-05-09 16:44:20,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +24: [2023-05-09 16:44:20,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +24: [2023-05-09 16:44:20,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +30: [2023-05-09 16:44:20,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +20: [2023-05-09 16:44:20,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +28: [2023-05-09 16:44:20,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +20: [2023-05-09 16:44:20,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +24: [2023-05-09 16:44:20,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +28: [2023-05-09 16:44:20,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +24: [2023-05-09 16:44:20,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +25: [2023-05-09 16:44:20,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +13: [2023-05-09 16:44:20,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +13: [2023-05-09 16:44:20,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 2: [2023-05-09 16:44:20,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 2: [2023-05-09 16:44:20,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 6: [2023-05-09 16:44:20,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +13: [2023-05-09 16:44:20,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +13: [2023-05-09 16:44:20,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +13: [2023-05-09 16:44:20,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +13: [2023-05-09 16:44:20,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +13: [2023-05-09 16:44:20,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +27: [2023-05-09 16:44:20,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +15: [2023-05-09 16:44:20,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +10: [2023-05-09 16:44:20,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +27: [2023-05-09 16:44:20,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +18: [2023-05-09 16:44:20,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +27: [2023-05-09 16:44:20,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +13: [2023-05-09 16:44:20,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +18: [2023-05-09 16:44:20,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +18: [2023-05-09 16:44:20,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +15: [2023-05-09 16:44:20,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +18: [2023-05-09 16:44:20,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +18: [2023-05-09 16:44:20,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +18: [2023-05-09 16:44:20,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +18: [2023-05-09 16:44:20,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +20: [2023-05-09 16:44:20,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +19: [2023-05-09 16:44:20,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +30: [2023-05-09 16:44:20,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +20: [2023-05-09 16:44:20,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +30: [2023-05-09 16:44:20,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +30: [2023-05-09 16:44:20,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +28: [2023-05-09 16:44:20,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +18: [2023-05-09 16:44:20,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +31: [2023-05-09 16:44:20,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +31: [2023-05-09 16:44:20,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +10: [2023-05-09 16:44:20,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +20: [2023-05-09 16:44:20,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +17: [2023-05-09 16:44:20,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +17: [2023-05-09 16:44:20,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +25: [2023-05-09 16:44:20,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +17: [2023-05-09 16:44:20,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +17: [2023-05-09 16:44:20,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +24: [2023-05-09 16:44:20,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +30: [2023-05-09 16:44:20,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 2: [2023-05-09 16:44:20,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +20: [2023-05-09 16:44:20,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +17: [2023-05-09 16:44:20,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +17: [2023-05-09 16:44:20,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +17: [2023-05-09 16:44:20,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +11: [2023-05-09 16:44:20,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 2: [2023-05-09 16:44:20,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +17: [2023-05-09 16:44:20,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +13: [2023-05-09 16:44:20,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +27: [2023-05-09 16:44:20,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +16: [2023-05-09 16:44:20,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +16: [2023-05-09 16:44:20,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +27: [2023-05-09 16:44:20,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +11: [2023-05-09 16:44:20,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +25: [2023-05-09 16:44:20,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 2: [2023-05-09 16:44:20,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +25: [2023-05-09 16:44:20,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +25: [2023-05-09 16:44:20,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +19: [2023-05-09 16:44:20,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +27: [2023-05-09 16:44:20,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +15: [2023-05-09 16:44:20,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +19: [2023-05-09 16:44:20,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +25: [2023-05-09 16:44:20,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +27: [2023-05-09 16:44:20,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +10: [2023-05-09 16:44:20,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +24: [2023-05-09 16:44:20,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +31: [2023-05-09 16:44:20,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +31: [2023-05-09 16:44:20,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +11: [2023-05-09 16:44:20,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +31: [2023-05-09 16:44:20,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +31: [2023-05-09 16:44:20,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +11: [2023-05-09 16:44:20,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +16: [2023-05-09 16:44:20,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +19: [2023-05-09 16:44:20,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +16: [2023-05-09 16:44:20,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +18: [2023-05-09 16:44:20,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +18: [2023-05-09 16:44:20,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +30: [2023-05-09 16:44:20,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +30: [2023-05-09 16:44:20,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +30: [2023-05-09 16:44:20,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +30: [2023-05-09 16:44:20,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +24: [2023-05-09 16:44:20,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +11: [2023-05-09 16:44:20,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +10: [2023-05-09 16:44:20,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +31: [2023-05-09 16:44:20,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +31: [2023-05-09 16:44:20,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +10: [2023-05-09 16:44:20,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +18: [2023-05-09 16:44:20,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +15: [2023-05-09 16:44:20,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +27: [2023-05-09 16:44:20,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +11: [2023-05-09 16:44:20,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +15: [2023-05-09 16:44:20,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +17: [2023-05-09 16:44:20,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +13: [2023-05-09 16:44:20,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +13: [2023-05-09 16:44:20,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +16: [2023-05-09 16:44:20,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +22: [2023-05-09 16:44:20,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +13: [2023-05-09 16:44:20,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +22: [2023-05-09 16:44:20,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +22: [2023-05-09 16:44:20,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +22: [2023-05-09 16:44:20,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +22: [2023-05-09 16:44:20,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +22: [2023-05-09 16:44:20,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +22: [2023-05-09 16:44:20,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +22: [2023-05-09 16:44:20,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +25: [2023-05-09 16:44:20,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +19: [2023-05-09 16:44:20,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +19: [2023-05-09 16:44:20,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +15: [2023-05-09 16:44:20,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +15: [2023-05-09 16:44:20,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 2: [2023-05-09 16:44:20,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +17: [2023-05-09 16:44:20,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +25: [2023-05-09 16:44:20,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +17: [2023-05-09 16:44:20,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +16: [2023-05-09 16:44:20,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +23: [2023-05-09 16:44:20,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +23: [2023-05-09 16:44:20,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +23: [2023-05-09 16:44:20,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 3: [2023-05-09 16:44:20,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 3: [2023-05-09 16:44:20,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 3: [2023-05-09 16:44:20,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +23: [2023-05-09 16:44:20,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 3: [2023-05-09 16:44:20,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +24: [2023-05-09 16:44:20,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +25: [2023-05-09 16:44:20,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +25: [2023-05-09 16:44:20,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +11: [2023-05-09 16:44:20,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +11: [2023-05-09 16:44:20,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +16: [2023-05-09 16:44:20,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +19: [2023-05-09 16:44:20,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +12: [2023-05-09 16:44:20,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +16: [2023-05-09 16:44:20,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +12: [2023-05-09 16:44:20,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +12: [2023-05-09 16:44:20,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 2: [2023-05-09 16:44:20,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 4: [2023-05-09 16:44:20,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. +12: [2023-05-09 16:44:20,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 4: [2023-05-09 16:44:20,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt. + 2: [2023-05-09 16:44:20,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +18: [2023-05-09 16:44:20,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +18: [2023-05-09 16:44:20,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +24: [2023-05-09 16:44:20,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +10: [2023-05-09 16:44:20,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +18: [2023-05-09 16:44:20,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +13: [2023-05-09 16:44:20,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +13: [2023-05-09 16:44:20,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +17: [2023-05-09 16:44:20,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +10: [2023-05-09 16:44:20,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +15: [2023-05-09 16:44:20,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +15: [2023-05-09 16:44:20,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +15: [2023-05-09 16:44:20,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +15: [2023-05-09 16:44:20,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +17: [2023-05-09 16:44:20,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +24: [2023-05-09 16:44:20,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +17: [2023-05-09 16:44:20,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +30: [2023-05-09 16:44:20,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +22: [2023-05-09 16:44:20,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +30: [2023-05-09 16:44:20,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +17: [2023-05-09 16:44:20,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +23: [2023-05-09 16:44:20,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +11: [2023-05-09 16:44:20,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +23: [2023-05-09 16:44:20,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +22: [2023-05-09 16:44:20,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +19: [2023-05-09 16:44:20,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +11: [2023-05-09 16:44:20,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +16: [2023-05-09 16:44:20,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +12: [2023-05-09 16:44:20,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +12: [2023-05-09 16:44:20,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +23: [2023-05-09 16:44:20,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +13: [2023-05-09 16:44:20,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +30: [2023-05-09 16:44:20,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +30: [2023-05-09 16:44:20,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +24: [2023-05-09 16:44:20,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +12: [2023-05-09 16:44:20,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +12: [2023-05-09 16:44:20,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +23: [2023-05-09 16:44:20,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +22: [2023-05-09 16:44:20,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +16: [2023-05-09 16:44:20,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +22: [2023-05-09 16:44:20,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +16: [2023-05-09 16:44:20,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +16: [2023-05-09 16:44:20,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 2: [2023-05-09 16:44:20,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... +16: [2023-05-09 16:44:20,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +16: [2023-05-09 16:44:20,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +18: [2023-05-09 16:44:20,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +11: [2023-05-09 16:44:20,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +10: [2023-05-09 16:44:20,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +11: [2023-05-09 16:44:20,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +11: [2023-05-09 16:44:20,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +11: [2023-05-09 16:44:20,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +11: [2023-05-09 16:44:20,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +11: [2023-05-09 16:44:20,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 4: [2023-05-09 16:44:20,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +24: [2023-05-09 16:44:20,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +17: [2023-05-09 16:44:20,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 0: [2023-05-09 16:44:20,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +25: [2023-05-09 16:44:20,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +25: [2023-05-09 16:44:20,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 2: [2023-05-09 16:44:20,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +22: [2023-05-09 16:44:20,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +22: [2023-05-09 16:44:20,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +13: [2023-05-09 16:44:20,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +30: [2023-05-09 16:44:20,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +30: [2023-05-09 16:44:20,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +23: [2023-05-09 16:44:20,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +23: [2023-05-09 16:44:20,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +18: [2023-05-09 16:44:20,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +22: [2023-05-09 16:44:20,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +30: [2023-05-09 16:44:20,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +30: [2023-05-09 16:44:20,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +15: [2023-05-09 16:44:20,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +22: [2023-05-09 16:44:20,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +15: [2023-05-09 16:44:20,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +16: [2023-05-09 16:44:20,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +15: [2023-05-09 16:44:20,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +15: [2023-05-09 16:44:20,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +15: [2023-05-09 16:44:20,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +15: [2023-05-09 16:44:20,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +16: [2023-05-09 16:44:20,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +25: [2023-05-09 16:44:20,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +25: [2023-05-09 16:44:20,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +23: [2023-05-09 16:44:20,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +23: [2023-05-09 16:44:20,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +25: [2023-05-09 16:44:20,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +25: [2023-05-09 16:44:20,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 3: [2023-05-09 16:44:20,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 3: [2023-05-09 16:44:20,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +15: [2023-05-09 16:44:20,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +15: [2023-05-09 16:44:20,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 4: [2023-05-09 16:44:20,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +25: [2023-05-09 16:44:20,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +25: [2023-05-09 16:44:20,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 4: [2023-05-09 16:44:20,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 3: [2023-05-09 16:44:20,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +23: [2023-05-09 16:44:20,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +23: [2023-05-09 16:44:20,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 3: [2023-05-09 16:44:20,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +23: [2023-05-09 16:44:20,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +23: [2023-05-09 16:44:20,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +23: [2023-05-09 16:44:20,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +23: [2023-05-09 16:44:20,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +31: [2023-05-09 16:44:20,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +31: [2023-05-09 16:44:20,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +31: [2023-05-09 16:44:20,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +31: [2023-05-09 16:44:20,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +31: [2023-05-09 16:44:20,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +31: [2023-05-09 16:44:20,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +31: [2023-05-09 16:44:20,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +23: [2023-05-09 16:44:20,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +23: [2023-05-09 16:44:20,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +23: [2023-05-09 16:44:20,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +23: [2023-05-09 16:44:20,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +31: [2023-05-09 16:44:20,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +23: [2023-05-09 16:44:20,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +26: [2023-05-09 16:44:20,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +26: [2023-05-09 16:44:20,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +14: [2023-05-09 16:44:20,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +14: [2023-05-09 16:44:20,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +14: [2023-05-09 16:44:20,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +14: [2023-05-09 16:44:20,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +26: [2023-05-09 16:44:20,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +26: [2023-05-09 16:44:20,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +26: [2023-05-09 16:44:20,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +26: [2023-05-09 16:44:20,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +12: [2023-05-09 16:44:20,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +12: [2023-05-09 16:44:20,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +12: [2023-05-09 16:44:20,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +26: [2023-05-09 16:44:20,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +23: [2023-05-09 16:44:20,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +12: [2023-05-09 16:44:20,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +12: [2023-05-09 16:44:20,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +12: [2023-05-09 16:44:20,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +12: [2023-05-09 16:44:20,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +12: [2023-05-09 16:44:20,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +12: [2023-05-09 16:44:20,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +12: [2023-05-09 16:44:20,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +12: [2023-05-09 16:44:20,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +12: [2023-05-09 16:44:20,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +12: [2023-05-09 16:44:20,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +12: [2023-05-09 16:44:20,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +26: [2023-05-09 16:44:20,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +12: [2023-05-09 16:44:20,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 5: [2023-05-09 16:44:20,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +12: [2023-05-09 16:44:20,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... + 4: [2023-05-09 16:44:20,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +14: [2023-05-09 16:44:20,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +14: [2023-05-09 16:44:20,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +14: [2023-05-09 16:44:20,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +14: [2023-05-09 16:44:20,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +23: [2023-05-09 16:44:20,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +31: [2023-05-09 16:44:20,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +31: [2023-05-09 16:44:20,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +31: [2023-05-09 16:44:20,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +23: [2023-05-09 16:44:20,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +23: [2023-05-09 16:44:20,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +26: [2023-05-09 16:44:20,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +14: [2023-05-09 16:44:20,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +14: [2023-05-09 16:44:20,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +26: [2023-05-09 16:44:20,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +26: [2023-05-09 16:44:20,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +23: [2023-05-09 16:44:20,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt. +31: [2023-05-09 16:44:20,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +12: [2023-05-09 16:44:20,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +12: [2023-05-09 16:44:20,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +31: [2023-05-09 16:44:20,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt... +31: [2023-05-09 16:44:20,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +26: [2023-05-09 16:44:20,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +26: [2023-05-09 16:44:20,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +19: [2023-05-09 16:44:20,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +19: [2023-05-09 16:44:20,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +26: [2023-05-09 16:44:20,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +23: [2023-05-09 16:44:20,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +23: [2023-05-09 16:44:20,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +19: [2023-05-09 16:44:20,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +19: [2023-05-09 16:44:20,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +23: [2023-05-09 16:44:20,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +14: [2023-05-09 16:44:20,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +14: [2023-05-09 16:44:20,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +31: [2023-05-09 16:44:20,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +26: [2023-05-09 16:44:20,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +12: [2023-05-09 16:44:20,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +17: [2023-05-09 16:44:20,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +17: [2023-05-09 16:44:20,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +17: [2023-05-09 16:44:20,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +17: [2023-05-09 16:44:20,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +12: [2023-05-09 16:44:20,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +14: [2023-05-09 16:44:20,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +12: [2023-05-09 16:44:20,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +27: [2023-05-09 16:44:20,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +27: [2023-05-09 16:44:20,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +27: [2023-05-09 16:44:20,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +27: [2023-05-09 16:44:20,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +12: [2023-05-09 16:44:20,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +23: [2023-05-09 16:44:20,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +19: [2023-05-09 16:44:20,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +19: [2023-05-09 16:44:20,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +31: [2023-05-09 16:44:20,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +19: [2023-05-09 16:44:20,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +19: [2023-05-09 16:44:20,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +14: [2023-05-09 16:44:20,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +26: [2023-05-09 16:44:20,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +12: [2023-05-09 16:44:20,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +12: [2023-05-09 16:44:20,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +17: [2023-05-09 16:44:20,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +17: [2023-05-09 16:44:20,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +14: [2023-05-09 16:44:20,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +27: [2023-05-09 16:44:20,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +27: [2023-05-09 16:44:20,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +17: [2023-05-09 16:44:20,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +17: [2023-05-09 16:44:20,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +27: [2023-05-09 16:44:20,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +18: [2023-05-09 16:44:20,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +18: [2023-05-09 16:44:20,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +18: [2023-05-09 16:44:20,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +18: [2023-05-09 16:44:20,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 1: [2023-05-09 16:44:20,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +27: [2023-05-09 16:44:20,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +14: [2023-05-09 16:44:20,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +19: [2023-05-09 16:44:20,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +19: [2023-05-09 16:44:20,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +19: [2023-05-09 16:44:20,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +19: [2023-05-09 16:44:20,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +19: [2023-05-09 16:44:20,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +19: [2023-05-09 16:44:20,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +10: [2023-05-09 16:44:20,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +10: [2023-05-09 16:44:20,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +10: [2023-05-09 16:44:20,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +10: [2023-05-09 16:44:20,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 1: [2023-05-09 16:44:20,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +18: [2023-05-09 16:44:20,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +27: [2023-05-09 16:44:20,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +27: [2023-05-09 16:44:20,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +19: [2023-05-09 16:44:20,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +19: [2023-05-09 16:44:20,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +18: [2023-05-09 16:44:20,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +18: [2023-05-09 16:44:20,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +18: [2023-05-09 16:44:20,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +17: [2023-05-09 16:44:20,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +17: [2023-05-09 16:44:20,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +17: [2023-05-09 16:44:20,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +17: [2023-05-09 16:44:20,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +17: [2023-05-09 16:44:20,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +17: [2023-05-09 16:44:20,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +27: [2023-05-09 16:44:20,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +27: [2023-05-09 16:44:20,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 1: [2023-05-09 16:44:20,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +10: [2023-05-09 16:44:20,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +10: [2023-05-09 16:44:20,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +10: [2023-05-09 16:44:20,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +10: [2023-05-09 16:44:20,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +18: [2023-05-09 16:44:20,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +18: [2023-05-09 16:44:20,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +17: [2023-05-09 16:44:20,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +17: [2023-05-09 16:44:20,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +27: [2023-05-09 16:44:20,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +18: [2023-05-09 16:44:20,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +18: [2023-05-09 16:44:20,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +18: [2023-05-09 16:44:20,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +18: [2023-05-09 16:44:20,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +27: [2023-05-09 16:44:20,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +27: [2023-05-09 16:44:20,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +27: [2023-05-09 16:44:20,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +14: [2023-05-09 16:44:20,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +14: [2023-05-09 16:44:20,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +14: [2023-05-09 16:44:20,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +14: [2023-05-09 16:44:20,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +29: [2023-05-09 16:44:20,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +18: [2023-05-09 16:44:20,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +29: [2023-05-09 16:44:20,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +29: [2023-05-09 16:44:20,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +29: [2023-05-09 16:44:20,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +18: [2023-05-09 16:44:20,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +10: [2023-05-09 16:44:20,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +10: [2023-05-09 16:44:20,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +10: [2023-05-09 16:44:20,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +10: [2023-05-09 16:44:20,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +14: [2023-05-09 16:44:20,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +14: [2023-05-09 16:44:20,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +14: [2023-05-09 16:44:20,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +14: [2023-05-09 16:44:20,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +29: [2023-05-09 16:44:20,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +29: [2023-05-09 16:44:20,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +10: [2023-05-09 16:44:20,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +10: [2023-05-09 16:44:20,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +29: [2023-05-09 16:44:20,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +29: [2023-05-09 16:44:20,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +10: [2023-05-09 16:44:20,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +10: [2023-05-09 16:44:20,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +13: [2023-05-09 16:44:20,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +13: [2023-05-09 16:44:20,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +13: [2023-05-09 16:44:20,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +13: [2023-05-09 16:44:20,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 1: [2023-05-09 16:44:20,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 1: [2023-05-09 16:44:20,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 1: [2023-05-09 16:44:20,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 1: [2023-05-09 16:44:20,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +29: [2023-05-09 16:44:20,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +14: [2023-05-09 16:44:20,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +14: [2023-05-09 16:44:20,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +11: [2023-05-09 16:44:20,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +11: [2023-05-09 16:44:20,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +11: [2023-05-09 16:44:20,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +14: [2023-05-09 16:44:20,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +11: [2023-05-09 16:44:20,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +11: [2023-05-09 16:44:20,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +14: [2023-05-09 16:44:20,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +11: [2023-05-09 16:44:20,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +29: [2023-05-09 16:44:20,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +11: [2023-05-09 16:44:20,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +11: [2023-05-09 16:44:20,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +13: [2023-05-09 16:44:20,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +13: [2023-05-09 16:44:20,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +13: [2023-05-09 16:44:20,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +13: [2023-05-09 16:44:20,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +29: [2023-05-09 16:44:20,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +29: [2023-05-09 16:44:20,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +14: [2023-05-09 16:44:20,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +14: [2023-05-09 16:44:20,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +29: [2023-05-09 16:44:20,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +29: [2023-05-09 16:44:20,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 1: [2023-05-09 16:44:20,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +29: [2023-05-09 16:44:20,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +14: [2023-05-09 16:44:20,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +14: [2023-05-09 16:44:20,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +29: [2023-05-09 16:44:20,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 8: [2023-05-09 16:44:20,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +11: [2023-05-09 16:44:20,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +13: [2023-05-09 16:44:20,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +13: [2023-05-09 16:44:20,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +24: [2023-05-09 16:44:20,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +24: [2023-05-09 16:44:20,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +24: [2023-05-09 16:44:20,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +24: [2023-05-09 16:44:20,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +16: [2023-05-09 16:44:20,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +16: [2023-05-09 16:44:20,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +16: [2023-05-09 16:44:20,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +16: [2023-05-09 16:44:20,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +16: [2023-05-09 16:44:20,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +16: [2023-05-09 16:44:20,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +16: [2023-05-09 16:44:20,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +11: [2023-05-09 16:44:20,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +16: [2023-05-09 16:44:20,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +13: [2023-05-09 16:44:20,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +13: [2023-05-09 16:44:20,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +13: [2023-05-09 16:44:20,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +29: [2023-05-09 16:44:20,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +29: [2023-05-09 16:44:20,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +29: [2023-05-09 16:44:20,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +29: [2023-05-09 16:44:20,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +29: [2023-05-09 16:44:20,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +21: [2023-05-09 16:44:20,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +21: [2023-05-09 16:44:20,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +21: [2023-05-09 16:44:20,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +29: [2023-05-09 16:44:20,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +21: [2023-05-09 16:44:20,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +21: [2023-05-09 16:44:20,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +21: [2023-05-09 16:44:20,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +21: [2023-05-09 16:44:20,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +21: [2023-05-09 16:44:20,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +13: [2023-05-09 16:44:20,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 9: [2023-05-09 16:44:20,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +29: [2023-05-09 16:44:20,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +29: [2023-05-09 16:44:20,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 8: [2023-05-09 16:44:20,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +11: [2023-05-09 16:44:20,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +24: [2023-05-09 16:44:20,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +24: [2023-05-09 16:44:20,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +24: [2023-05-09 16:44:20,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +13: [2023-05-09 16:44:20,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +24: [2023-05-09 16:44:20,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 5: [2023-05-09 16:44:20,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 5: [2023-05-09 16:44:20,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +13: [2023-05-09 16:44:20,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 1: [2023-05-09 16:44:20,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +11: [2023-05-09 16:44:20,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +16: [2023-05-09 16:44:20,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 1: [2023-05-09 16:44:20,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 8: [2023-05-09 16:44:20,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +11: [2023-05-09 16:44:20,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +11: [2023-05-09 16:44:20,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +16: [2023-05-09 16:44:20,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +16: [2023-05-09 16:44:20,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +21: [2023-05-09 16:44:20,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +29: [2023-05-09 16:44:20,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +29: [2023-05-09 16:44:20,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 7: [2023-05-09 16:44:20,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +16: [2023-05-09 16:44:20,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 5: [2023-05-09 16:44:20,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +24: [2023-05-09 16:44:20,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +24: [2023-05-09 16:44:20,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +16: [2023-05-09 16:44:20,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +24: [2023-05-09 16:44:20,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +24: [2023-05-09 16:44:20,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 5: [2023-05-09 16:44:20,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +11: [2023-05-09 16:44:20,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +11: [2023-05-09 16:44:20,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +26: [2023-05-09 16:44:20,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +26: [2023-05-09 16:44:20,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +26: [2023-05-09 16:44:20,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 6: [2023-05-09 16:44:20,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +26: [2023-05-09 16:44:20,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +21: [2023-05-09 16:44:20,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +21: [2023-05-09 16:44:20,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +21: [2023-05-09 16:44:20,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +21: [2023-05-09 16:44:20,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +25: [2023-05-09 16:44:20,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +25: [2023-05-09 16:44:20,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +16: [2023-05-09 16:44:20,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +16: [2023-05-09 16:44:20,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +29: [2023-05-09 16:44:20,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +16: [2023-05-09 16:44:20,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +25: [2023-05-09 16:44:20,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +25: [2023-05-09 16:44:20,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +25: [2023-05-09 16:44:20,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +25: [2023-05-09 16:44:20,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +24: [2023-05-09 16:44:20,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +24: [2023-05-09 16:44:20,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +24: [2023-05-09 16:44:20,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +24: [2023-05-09 16:44:20,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 9: [2023-05-09 16:44:20,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +29: [2023-05-09 16:44:20,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +25: [2023-05-09 16:44:20,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +25: [2023-05-09 16:44:20,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +29: [2023-05-09 16:44:20,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 9: [2023-05-09 16:44:20,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 9: [2023-05-09 16:44:20,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 7: [2023-05-09 16:44:20,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 0: [2023-05-09 16:44:20,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 0: [2023-05-09 16:44:20,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 0: [2023-05-09 16:44:20,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +29: [2023-05-09 16:44:20,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +26: [2023-05-09 16:44:20,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +21: [2023-05-09 16:44:20,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +26: [2023-05-09 16:44:20,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +26: [2023-05-09 16:44:20,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +26: [2023-05-09 16:44:20,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +21: [2023-05-09 16:44:20,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +25: [2023-05-09 16:44:20,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +21: [2023-05-09 16:44:20,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +29: [2023-05-09 16:44:20,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +29: [2023-05-09 16:44:20,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +25: [2023-05-09 16:44:20,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +25: [2023-05-09 16:44:20,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 5: [2023-05-09 16:44:20,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 5: [2023-05-09 16:44:20,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 9: [2023-05-09 16:44:20,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 5: [2023-05-09 16:44:20,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 7: [2023-05-09 16:44:20,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 9: [2023-05-09 16:44:20,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +26: [2023-05-09 16:44:20,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +26: [2023-05-09 16:44:20,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +26: [2023-05-09 16:44:20,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +26: [2023-05-09 16:44:20,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +26: [2023-05-09 16:44:20,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +26: [2023-05-09 16:44:20,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +25: [2023-05-09 16:44:20,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +22: [2023-05-09 16:44:20,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +25: [2023-05-09 16:44:20,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +22: [2023-05-09 16:44:20,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 9: [2023-05-09 16:44:20,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 8: [2023-05-09 16:44:20,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 8: [2023-05-09 16:44:20,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 8: [2023-05-09 16:44:20,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 8: [2023-05-09 16:44:20,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +22: [2023-05-09 16:44:20,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +22: [2023-05-09 16:44:20,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +21: [2023-05-09 16:44:20,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +21: [2023-05-09 16:44:20,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +21: [2023-05-09 16:44:20,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +31: [2023-05-09 16:44:20,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 6: [2023-05-09 16:44:20,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 6: [2023-05-09 16:44:20,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 6: [2023-05-09 16:44:20,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 6: [2023-05-09 16:44:20,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +21: [2023-05-09 16:44:20,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +31: [2023-05-09 16:44:20,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +31: [2023-05-09 16:44:20,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +16: [2023-05-09 16:44:20,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +31: [2023-05-09 16:44:20,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +16: [2023-05-09 16:44:20,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +26: [2023-05-09 16:44:20,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 2: [2023-05-09 16:44:20,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 2: [2023-05-09 16:44:20,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 7: [2023-05-09 16:44:20,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 2: [2023-05-09 16:44:20,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +16: [2023-05-09 16:44:20,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +16: [2023-05-09 16:44:20,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 7: [2023-05-09 16:44:20,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +26: [2023-05-09 16:44:20,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 6: [2023-05-09 16:44:20,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +25: [2023-05-09 16:44:20,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +25: [2023-05-09 16:44:20,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +22: [2023-05-09 16:44:20,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 0: [2023-05-09 16:44:20,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 0: [2023-05-09 16:44:20,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 8: [2023-05-09 16:44:20,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +22: [2023-05-09 16:44:20,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +31: [2023-05-09 16:44:20,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +31: [2023-05-09 16:44:20,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +21: [2023-05-09 16:44:20,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +21: [2023-05-09 16:44:20,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +21: [2023-05-09 16:44:20,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +21: [2023-05-09 16:44:20,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +16: [2023-05-09 16:44:20,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 6: [2023-05-09 16:44:20,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +22: [2023-05-09 16:44:20,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +31: [2023-05-09 16:44:20,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +16: [2023-05-09 16:44:20,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +22: [2023-05-09 16:44:20,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +31: [2023-05-09 16:44:20,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +23: [2023-05-09 16:44:20,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +30: [2023-05-09 16:44:20,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 6: [2023-05-09 16:44:20,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +23: [2023-05-09 16:44:20,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +30: [2023-05-09 16:44:20,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +23: [2023-05-09 16:44:20,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +23: [2023-05-09 16:44:20,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +30: [2023-05-09 16:44:20,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +30: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +20: [2023-05-09 16:44:20,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +25: [2023-05-09 16:44:20,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +20: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +20: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +28: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +16: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +11: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +16: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +11: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +11: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +28: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +28: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +11: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 4: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +20: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 3: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 3: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 3: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +25: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +25: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +28: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 4: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 4: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +25: [2023-05-09 16:44:20,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 4: [2023-05-09 16:44:20,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +25: [2023-05-09 16:44:20,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +15: [2023-05-09 16:44:20,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +15: [2023-05-09 16:44:20,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +15: [2023-05-09 16:44:20,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. +15: [2023-05-09 16:44:20,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt. + 2: [2023-05-09 16:44:20,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +20: [2023-05-09 16:44:20,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +20: [2023-05-09 16:44:20,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +20: [2023-05-09 16:44:20,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +20: [2023-05-09 16:44:20,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +20: [2023-05-09 16:44:20,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +20: [2023-05-09 16:44:20,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +20: [2023-05-09 16:44:20,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +30: [2023-05-09 16:44:20,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +25: [2023-05-09 16:44:20,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +20: [2023-05-09 16:44:20,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +30: [2023-05-09 16:44:20,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +23: [2023-05-09 16:44:20,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +30: [2023-05-09 16:44:20,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +30: [2023-05-09 16:44:20,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +23: [2023-05-09 16:44:20,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +11: [2023-05-09 16:44:20,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +23: [2023-05-09 16:44:20,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +28: [2023-05-09 16:44:20,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +16: [2023-05-09 16:44:20,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +20: [2023-05-09 16:44:20,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +28: [2023-05-09 16:44:20,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +11: [2023-05-09 16:44:20,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +16: [2023-05-09 16:44:20,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +25: [2023-05-09 16:44:20,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 3: [2023-05-09 16:44:20,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +15: [2023-05-09 16:44:20,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +22: [2023-05-09 16:44:20,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +15: [2023-05-09 16:44:20,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +23: [2023-05-09 16:44:20,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +25: [2023-05-09 16:44:20,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +21: [2023-05-09 16:44:20,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +20: [2023-05-09 16:44:20,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +20: [2023-05-09 16:44:20,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +25: [2023-05-09 16:44:20,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +28: [2023-05-09 16:44:20,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +22: [2023-05-09 16:44:20,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +21: [2023-05-09 16:44:20,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 6: [2023-05-09 16:44:20,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +11: [2023-05-09 16:44:20,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +11: [2023-05-09 16:44:20,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 6: [2023-05-09 16:44:20,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +20: [2023-05-09 16:44:20,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +28: [2023-05-09 16:44:20,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +16: [2023-05-09 16:44:20,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +16: [2023-05-09 16:44:20,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 8: [2023-05-09 16:44:20,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 0: [2023-05-09 16:44:20,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +15: [2023-05-09 16:44:20,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... +22: [2023-05-09 16:44:20,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +22: [2023-05-09 16:44:20,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 8: [2023-05-09 16:44:20,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 8: [2023-05-09 16:44:20,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +21: [2023-05-09 16:44:20,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +21: [2023-05-09 16:44:20,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +21: [2023-05-09 16:44:20,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +21: [2023-05-09 16:44:20,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +22: [2023-05-09 16:44:20,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +22: [2023-05-09 16:44:20,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 0: [2023-05-09 16:44:20,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +15: [2023-05-09 16:44:20,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +31: [2023-05-09 16:44:20,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +31: [2023-05-09 16:44:20,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +31: [2023-05-09 16:44:20,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +31: [2023-05-09 16:44:20,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +22: [2023-05-09 16:44:20,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +22: [2023-05-09 16:44:20,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 2: [2023-05-09 16:44:20,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +19: [2023-05-09 16:44:20,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +19: [2023-05-09 16:44:20,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +16: [2023-05-09 16:44:20,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +16: [2023-05-09 16:44:20,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 2: [2023-05-09 16:44:20,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +18: [2023-05-09 16:44:20,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +18: [2023-05-09 16:44:20,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +18: [2023-05-09 16:44:20,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +18: [2023-05-09 16:44:20,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +19: [2023-05-09 16:44:20,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 2: [2023-05-09 16:44:20,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +19: [2023-05-09 16:44:20,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +19: [2023-05-09 16:44:20,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +19: [2023-05-09 16:44:20,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +19: [2023-05-09 16:44:20,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +19: [2023-05-09 16:44:20,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +25: [2023-05-09 16:44:20,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +25: [2023-05-09 16:44:20,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +28: [2023-05-09 16:44:20,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +28: [2023-05-09 16:44:20,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +28: [2023-05-09 16:44:20,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +21: [2023-05-09 16:44:20,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +28: [2023-05-09 16:44:20,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +28: [2023-05-09 16:44:20,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +11: [2023-05-09 16:44:20,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +18: [2023-05-09 16:44:20,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +18: [2023-05-09 16:44:20,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +18: [2023-05-09 16:44:20,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +30: [2023-05-09 16:44:20,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +30: [2023-05-09 16:44:20,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +30: [2023-05-09 16:44:20,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +30: [2023-05-09 16:44:20,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +28: [2023-05-09 16:44:20,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +11: [2023-05-09 16:44:20,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +21: [2023-05-09 16:44:20,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 7: [2023-05-09 16:44:20,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +23: [2023-05-09 16:44:20,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +28: [2023-05-09 16:44:20,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +28: [2023-05-09 16:44:20,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +10: [2023-05-09 16:44:20,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +30: [2023-05-09 16:44:20,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +30: [2023-05-09 16:44:20,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +30: [2023-05-09 16:44:20,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +23: [2023-05-09 16:44:20,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +10: [2023-05-09 16:44:20,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +30: [2023-05-09 16:44:20,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +30: [2023-05-09 16:44:20,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +23: [2023-05-09 16:44:20,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +23: [2023-05-09 16:44:20,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +27: [2023-05-09 16:44:20,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +30: [2023-05-09 16:44:20,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +27: [2023-05-09 16:44:20,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +15: [2023-05-09 16:44:20,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +18: [2023-05-09 16:44:20,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +15: [2023-05-09 16:44:20,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +15: [2023-05-09 16:44:20,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +31: [2023-05-09 16:44:20,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +31: [2023-05-09 16:44:20,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +15: [2023-05-09 16:44:20,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +31: [2023-05-09 16:44:20,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +31: [2023-05-09 16:44:20,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +31: [2023-05-09 16:44:20,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +31: [2023-05-09 16:44:20,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +13: [2023-05-09 16:44:20,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +13: [2023-05-09 16:44:20,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +13: [2023-05-09 16:44:20,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +13: [2023-05-09 16:44:20,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +15: [2023-05-09 16:44:20,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +10: [2023-05-09 16:44:20,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +10: [2023-05-09 16:44:20,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +15: [2023-05-09 16:44:20,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +27: [2023-05-09 16:44:20,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +24: [2023-05-09 16:44:20,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +30: [2023-05-09 16:44:20,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +24: [2023-05-09 16:44:20,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +24: [2023-05-09 16:44:20,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +15: [2023-05-09 16:44:20,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +23: [2023-05-09 16:44:20,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +23: [2023-05-09 16:44:20,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +15: [2023-05-09 16:44:20,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +13: [2023-05-09 16:44:20,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +30: [2023-05-09 16:44:20,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +30: [2023-05-09 16:44:20,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +30: [2023-05-09 16:44:20,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +30: [2023-05-09 16:44:20,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +23: [2023-05-09 16:44:20,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +23: [2023-05-09 16:44:20,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +13: [2023-05-09 16:44:20,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +23: [2023-05-09 16:44:20,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +23: [2023-05-09 16:44:20,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +20: [2023-05-09 16:44:20,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +31: [2023-05-09 16:44:20,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +22: [2023-05-09 16:44:20,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +31: [2023-05-09 16:44:20,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +22: [2023-05-09 16:44:20,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +27: [2023-05-09 16:44:20,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +30: [2023-05-09 16:44:20,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +16: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +24: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +24: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +24: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +24: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +28: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +27: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +13: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +22: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +27: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +27: [2023-05-09 16:44:20,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +28: [2023-05-09 16:44:20,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +27: [2023-05-09 16:44:20,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +10: [2023-05-09 16:44:20,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +10: [2023-05-09 16:44:20,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +15: [2023-05-09 16:44:20,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +20: [2023-05-09 16:44:20,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +31: [2023-05-09 16:44:20,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +23: [2023-05-09 16:44:20,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +31: [2023-05-09 16:44:20,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +23: [2023-05-09 16:44:20,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +16: [2023-05-09 16:44:20,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +15: [2023-05-09 16:44:20,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 7: [2023-05-09 16:44:20,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +17: [2023-05-09 16:44:20,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +17: [2023-05-09 16:44:20,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +17: [2023-05-09 16:44:20,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +17: [2023-05-09 16:44:20,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +10: [2023-05-09 16:44:20,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +22: [2023-05-09 16:44:20,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +10: [2023-05-09 16:44:20,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +22: [2023-05-09 16:44:20,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +22: [2023-05-09 16:44:20,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +13: [2023-05-09 16:44:20,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +26: [2023-05-09 16:44:20,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +26: [2023-05-09 16:44:20,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +26: [2023-05-09 16:44:20,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +26: [2023-05-09 16:44:20,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +26: [2023-05-09 16:44:20,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +26: [2023-05-09 16:44:20,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +22: [2023-05-09 16:44:20,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +26: [2023-05-09 16:44:20,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +26: [2023-05-09 16:44:20,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +24: [2023-05-09 16:44:20,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +25: [2023-05-09 16:44:20,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +25: [2023-05-09 16:44:20,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +31: [2023-05-09 16:44:20,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +25: [2023-05-09 16:44:20,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +25: [2023-05-09 16:44:20,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +20: [2023-05-09 16:44:20,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +19: [2023-05-09 16:44:20,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +14: [2023-05-09 16:44:20,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +14: [2023-05-09 16:44:20,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +22: [2023-05-09 16:44:20,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +14: [2023-05-09 16:44:20,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +14: [2023-05-09 16:44:20,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +14: [2023-05-09 16:44:20,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +14: [2023-05-09 16:44:20,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +31: [2023-05-09 16:44:20,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 3: [2023-05-09 16:44:20,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +20: [2023-05-09 16:44:20,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +23: [2023-05-09 16:44:20,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +23: [2023-05-09 16:44:20,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 3: [2023-05-09 16:44:20,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +14: [2023-05-09 16:44:20,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +14: [2023-05-09 16:44:20,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +28: [2023-05-09 16:44:20,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +28: [2023-05-09 16:44:20,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +17: [2023-05-09 16:44:20,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +17: [2023-05-09 16:44:20,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +17: [2023-05-09 16:44:20,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +17: [2023-05-09 16:44:20,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +20: [2023-05-09 16:44:20,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +20: [2023-05-09 16:44:20,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +20: [2023-05-09 16:44:20,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +20: [2023-05-09 16:44:20,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +18: [2023-05-09 16:44:20,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +20: [2023-05-09 16:44:20,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +18: [2023-05-09 16:44:20,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +18: [2023-05-09 16:44:20,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 4: [2023-05-09 16:44:20,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +11: [2023-05-09 16:44:20,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +11: [2023-05-09 16:44:20,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +11: [2023-05-09 16:44:20,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +11: [2023-05-09 16:44:20,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 4: [2023-05-09 16:44:20,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +25: [2023-05-09 16:44:20,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +25: [2023-05-09 16:44:20,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 3: [2023-05-09 16:44:20,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 3: [2023-05-09 16:44:20,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +23: [2023-05-09 16:44:20,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +15: [2023-05-09 16:44:20,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +31: [2023-05-09 16:44:20,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +27: [2023-05-09 16:44:20,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +31: [2023-05-09 16:44:20,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 0: [2023-05-09 16:44:20,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +31: [2023-05-09 16:44:20,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +20: [2023-05-09 16:44:20,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +28: [2023-05-09 16:44:20,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +28: [2023-05-09 16:44:20,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +15: [2023-05-09 16:44:20,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +23: [2023-05-09 16:44:20,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 7: [2023-05-09 16:44:20,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +15: [2023-05-09 16:44:20,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 7: [2023-05-09 16:44:20,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +22: [2023-05-09 16:44:20,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +28: [2023-05-09 16:44:20,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +19: [2023-05-09 16:44:20,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +19: [2023-05-09 16:44:20,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +19: [2023-05-09 16:44:20,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +23: [2023-05-09 16:44:20,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +10: [2023-05-09 16:44:20,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +27: [2023-05-09 16:44:20,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 4: [2023-05-09 16:44:20,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 4: [2023-05-09 16:44:20,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 2: [2023-05-09 16:44:20,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +15: [2023-05-09 16:44:20,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +19: [2023-05-09 16:44:20,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +30: [2023-05-09 16:44:20,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +30: [2023-05-09 16:44:20,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +15: [2023-05-09 16:44:20,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +22: [2023-05-09 16:44:20,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +14: [2023-05-09 16:44:20,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +30: [2023-05-09 16:44:20,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +20: [2023-05-09 16:44:20,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +23: [2023-05-09 16:44:20,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +23: [2023-05-09 16:44:20,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +20: [2023-05-09 16:44:20,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +14: [2023-05-09 16:44:20,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +18: [2023-05-09 16:44:20,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +23: [2023-05-09 16:44:20,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +11: [2023-05-09 16:44:20,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +13: [2023-05-09 16:44:20,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +13: [2023-05-09 16:44:20,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +18: [2023-05-09 16:44:20,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +18: [2023-05-09 16:44:20,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +17: [2023-05-09 16:44:20,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +17: [2023-05-09 16:44:20,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +26: [2023-05-09 16:44:20,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +26: [2023-05-09 16:44:20,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +26: [2023-05-09 16:44:20,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +15: [2023-05-09 16:44:20,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +20: [2023-05-09 16:44:20,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +13: [2023-05-09 16:44:20,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +12: [2023-05-09 16:44:20,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +12: [2023-05-09 16:44:20,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +17: [2023-05-09 16:44:20,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +15: [2023-05-09 16:44:20,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +11: [2023-05-09 16:44:20,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +31: [2023-05-09 16:44:20,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +17: [2023-05-09 16:44:20,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +10: [2023-05-09 16:44:20,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +28: [2023-05-09 16:44:20,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +10: [2023-05-09 16:44:20,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +24: [2023-05-09 16:44:20,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +24: [2023-05-09 16:44:20,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +27: [2023-05-09 16:44:20,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +31: [2023-05-09 16:44:20,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +28: [2023-05-09 16:44:20,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +27: [2023-05-09 16:44:20,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +28: [2023-05-09 16:44:20,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +31: [2023-05-09 16:44:20,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +12: [2023-05-09 16:44:20,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +20: [2023-05-09 16:44:20,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +12: [2023-05-09 16:44:20,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... + 7: [2023-05-09 16:44:20,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +28: [2023-05-09 16:44:20,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +19: [2023-05-09 16:44:20,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +28: [2023-05-09 16:44:20,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +28: [2023-05-09 16:44:20,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +10: [2023-05-09 16:44:20,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +30: [2023-05-09 16:44:20,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +30: [2023-05-09 16:44:20,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +28: [2023-05-09 16:44:20,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +12: [2023-05-09 16:44:20,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +12: [2023-05-09 16:44:20,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +12: [2023-05-09 16:44:20,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +15: [2023-05-09 16:44:20,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +26: [2023-05-09 16:44:20,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +15: [2023-05-09 16:44:20,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt. +24: [2023-05-09 16:44:20,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +27: [2023-05-09 16:44:20,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +28: [2023-05-09 16:44:20,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +14: [2023-05-09 16:44:20,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +15: [2023-05-09 16:44:20,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +19: [2023-05-09 16:44:20,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +22: [2023-05-09 16:44:20,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +23: [2023-05-09 16:44:20,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +30: [2023-05-09 16:44:20,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +22: [2023-05-09 16:44:20,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +19: [2023-05-09 16:44:20,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +30: [2023-05-09 16:44:20,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +15: [2023-05-09 16:44:20,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +12: [2023-05-09 16:44:20,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +20: [2023-05-09 16:44:20,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +20: [2023-05-09 16:44:20,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +23: [2023-05-09 16:44:20,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +23: [2023-05-09 16:44:20,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +18: [2023-05-09 16:44:20,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +14: [2023-05-09 16:44:20,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +15: [2023-05-09 16:44:20,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt... +15: [2023-05-09 16:44:20,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +27: [2023-05-09 16:44:20,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +13: [2023-05-09 16:44:20,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +13: [2023-05-09 16:44:20,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +10: [2023-05-09 16:44:20,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +13: [2023-05-09 16:44:20,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +23: [2023-05-09 16:44:20,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +27: [2023-05-09 16:44:20,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +10: [2023-05-09 16:44:20,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +26: [2023-05-09 16:44:20,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +15: [2023-05-09 16:44:20,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +22: [2023-05-09 16:44:20,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +26: [2023-05-09 16:44:20,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +28: [2023-05-09 16:44:20,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +26: [2023-05-09 16:44:20,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +22: [2023-05-09 16:44:20,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +14: [2023-05-09 16:44:20,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +14: [2023-05-09 16:44:20,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +31: [2023-05-09 16:44:20,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +17: [2023-05-09 16:44:20,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +13: [2023-05-09 16:44:20,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +24: [2023-05-09 16:44:20,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +17: [2023-05-09 16:44:20,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +26: [2023-05-09 16:44:20,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +17: [2023-05-09 16:44:20,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +10: [2023-05-09 16:44:20,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +30: [2023-05-09 16:44:20,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +12: [2023-05-09 16:44:20,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +24: [2023-05-09 16:44:20,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +24: [2023-05-09 16:44:20,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +18: [2023-05-09 16:44:20,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +27: [2023-05-09 16:44:20,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +12: [2023-05-09 16:44:20,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +12: [2023-05-09 16:44:20,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +17: [2023-05-09 16:44:20,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +22: [2023-05-09 16:44:20,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +22: [2023-05-09 16:44:20,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +31: [2023-05-09 16:44:20,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +14: [2023-05-09 16:44:20,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +14: [2023-05-09 16:44:20,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +24: [2023-05-09 16:44:20,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +13: [2023-05-09 16:44:20,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +10: [2023-05-09 16:44:20,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +12: [2023-05-09 16:44:20,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +12: [2023-05-09 16:44:20,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +12: [2023-05-09 16:44:20,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +24: [2023-05-09 16:44:20,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +12: [2023-05-09 16:44:20,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +12: [2023-05-09 16:44:20,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +16: [2023-05-09 16:44:20,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +16: [2023-05-09 16:44:20,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +16: [2023-05-09 16:44:20,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +16: [2023-05-09 16:44:20,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +16: [2023-05-09 16:44:20,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +16: [2023-05-09 16:44:20,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +16: [2023-05-09 16:44:20,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +16: [2023-05-09 16:44:20,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +16: [2023-05-09 16:44:20,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +16: [2023-05-09 16:44:20,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +25: [2023-05-09 16:44:20,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +25: [2023-05-09 16:44:20,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +25: [2023-05-09 16:44:20,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +25: [2023-05-09 16:44:20,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +25: [2023-05-09 16:44:20,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +25: [2023-05-09 16:44:20,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +25: [2023-05-09 16:44:20,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +25: [2023-05-09 16:44:20,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +16: [2023-05-09 16:44:20,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +16: [2023-05-09 16:44:20,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +16: [2023-05-09 16:44:20,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +16: [2023-05-09 16:44:20,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +25: [2023-05-09 16:44:20,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +25: [2023-05-09 16:44:20,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +25: [2023-05-09 16:44:20,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +16: [2023-05-09 16:44:20,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +16: [2023-05-09 16:44:20,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +25: [2023-05-09 16:44:20,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +25: [2023-05-09 16:44:20,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +25: [2023-05-09 16:44:20,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +25: [2023-05-09 16:44:20,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +25: [2023-05-09 16:44:20,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +21: [2023-05-09 16:44:20,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +21: [2023-05-09 16:44:20,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +21: [2023-05-09 16:44:20,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +21: [2023-05-09 16:44:20,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +21: [2023-05-09 16:44:20,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +21: [2023-05-09 16:44:20,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +21: [2023-05-09 16:44:20,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +21: [2023-05-09 16:44:20,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +21: [2023-05-09 16:44:20,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +21: [2023-05-09 16:44:20,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +21: [2023-05-09 16:44:20,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +21: [2023-05-09 16:44:20,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +21: [2023-05-09 16:44:20,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +21: [2023-05-09 16:44:20,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +21: [2023-05-09 16:44:20,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +21: [2023-05-09 16:44:20,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +24: [2023-05-09 16:44:20,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +24: [2023-05-09 16:44:20,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +13: [2023-05-09 16:44:20,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +13: [2023-05-09 16:44:20,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +13: [2023-05-09 16:44:20,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +19: [2023-05-09 16:44:20,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +19: [2023-05-09 16:44:20,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +24: [2023-05-09 16:44:20,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +10: [2023-05-09 16:44:20,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +19: [2023-05-09 16:44:20,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +10: [2023-05-09 16:44:20,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +18: [2023-05-09 16:44:20,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +24: [2023-05-09 16:44:20,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +17: [2023-05-09 16:44:20,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +13: [2023-05-09 16:44:20,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +19: [2023-05-09 16:44:20,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +17: [2023-05-09 16:44:20,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +10: [2023-05-09 16:44:20,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +18: [2023-05-09 16:44:20,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +18: [2023-05-09 16:44:20,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +17: [2023-05-09 16:44:20,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +10: [2023-05-09 16:44:20,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 6: [2023-05-09 16:44:20,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +17: [2023-05-09 16:44:20,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +18: [2023-05-09 16:44:20,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 1: [2023-05-09 16:44:20,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 1: [2023-05-09 16:44:20,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +27: [2023-05-09 16:44:20,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +27: [2023-05-09 16:44:20,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 9: [2023-05-09 16:44:20,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +27: [2023-05-09 16:44:20,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +27: [2023-05-09 16:44:20,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +14: [2023-05-09 16:44:20,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +14: [2023-05-09 16:44:20,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +14: [2023-05-09 16:44:20,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 1: [2023-05-09 16:44:20,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +14: [2023-05-09 16:44:20,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 9: [2023-05-09 16:44:20,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +19: [2023-05-09 16:44:20,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +13: [2023-05-09 16:44:20,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +10: [2023-05-09 16:44:20,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +13: [2023-05-09 16:44:20,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +13: [2023-05-09 16:44:20,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +24: [2023-05-09 16:44:20,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +24: [2023-05-09 16:44:20,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +10: [2023-05-09 16:44:20,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +13: [2023-05-09 16:44:20,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +10: [2023-05-09 16:44:20,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +10: [2023-05-09 16:44:20,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +19: [2023-05-09 16:44:20,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +19: [2023-05-09 16:44:20,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +18: [2023-05-09 16:44:20,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +18: [2023-05-09 16:44:20,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +19: [2023-05-09 16:44:20,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +18: [2023-05-09 16:44:20,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +18: [2023-05-09 16:44:20,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +24: [2023-05-09 16:44:20,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +24: [2023-05-09 16:44:20,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +27: [2023-05-09 16:44:20,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +27: [2023-05-09 16:44:20,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +27: [2023-05-09 16:44:20,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +17: [2023-05-09 16:44:20,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +17: [2023-05-09 16:44:20,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +27: [2023-05-09 16:44:20,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +14: [2023-05-09 16:44:20,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +14: [2023-05-09 16:44:20,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +14: [2023-05-09 16:44:20,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +14: [2023-05-09 16:44:20,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +17: [2023-05-09 16:44:20,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +17: [2023-05-09 16:44:20,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +12: [2023-05-09 16:44:20,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +12: [2023-05-09 16:44:20,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +12: [2023-05-09 16:44:20,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +12: [2023-05-09 16:44:20,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +29: [2023-05-09 16:44:20,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +29: [2023-05-09 16:44:20,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +29: [2023-05-09 16:44:20,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +29: [2023-05-09 16:44:20,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +13: [2023-05-09 16:44:20,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +13: [2023-05-09 16:44:20,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +24: [2023-05-09 16:44:20,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +19: [2023-05-09 16:44:20,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +24: [2023-05-09 16:44:20,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +10: [2023-05-09 16:44:20,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +19: [2023-05-09 16:44:20,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +10: [2023-05-09 16:44:20,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +24: [2023-05-09 16:44:20,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +24: [2023-05-09 16:44:20,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +13: [2023-05-09 16:44:20,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +13: [2023-05-09 16:44:20,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +13: [2023-05-09 16:44:20,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +13: [2023-05-09 16:44:20,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +27: [2023-05-09 16:44:20,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +18: [2023-05-09 16:44:20,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +18: [2023-05-09 16:44:20,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +24: [2023-05-09 16:44:20,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +24: [2023-05-09 16:44:20,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +19: [2023-05-09 16:44:20,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +27: [2023-05-09 16:44:20,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +27: [2023-05-09 16:44:20,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +27: [2023-05-09 16:44:20,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +12: [2023-05-09 16:44:20,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +27: [2023-05-09 16:44:20,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +27: [2023-05-09 16:44:20,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +10: [2023-05-09 16:44:20,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +10: [2023-05-09 16:44:20,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +10: [2023-05-09 16:44:20,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +10: [2023-05-09 16:44:20,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +10: [2023-05-09 16:44:20,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +10: [2023-05-09 16:44:20,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +12: [2023-05-09 16:44:20,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +19: [2023-05-09 16:44:20,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +17: [2023-05-09 16:44:20,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +14: [2023-05-09 16:44:20,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +29: [2023-05-09 16:44:20,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +17: [2023-05-09 16:44:20,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +14: [2023-05-09 16:44:20,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +29: [2023-05-09 16:44:20,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +24: [2023-05-09 16:44:20,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +24: [2023-05-09 16:44:20,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +12: [2023-05-09 16:44:20,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +12: [2023-05-09 16:44:20,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +13: [2023-05-09 16:44:20,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +18: [2023-05-09 16:44:20,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +19: [2023-05-09 16:44:20,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +29: [2023-05-09 16:44:20,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +29: [2023-05-09 16:44:20,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +13: [2023-05-09 16:44:20,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +19: [2023-05-09 16:44:20,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +27: [2023-05-09 16:44:20,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +18: [2023-05-09 16:44:20,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +27: [2023-05-09 16:44:20,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +14: [2023-05-09 16:44:20,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +14: [2023-05-09 16:44:20,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +14: [2023-05-09 16:44:20,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +14: [2023-05-09 16:44:20,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +19: [2023-05-09 16:44:20,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 1: [2023-05-09 16:44:20,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +17: [2023-05-09 16:44:20,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +17: [2023-05-09 16:44:20,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +19: [2023-05-09 16:44:20,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +18: [2023-05-09 16:44:20,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +18: [2023-05-09 16:44:20,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +18: [2023-05-09 16:44:20,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +18: [2023-05-09 16:44:20,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +17: [2023-05-09 16:44:20,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +17: [2023-05-09 16:44:20,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +14: [2023-05-09 16:44:20,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +14: [2023-05-09 16:44:20,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 1: [2023-05-09 16:44:20,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 1: [2023-05-09 16:44:20,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +29: [2023-05-09 16:44:20,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +29: [2023-05-09 16:44:20,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +12: [2023-05-09 16:44:20,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +29: [2023-05-09 16:44:20,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +29: [2023-05-09 16:44:20,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +12: [2023-05-09 16:44:20,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 3: [2023-05-09 16:44:20,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +17: [2023-05-09 16:44:20,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +29: [2023-05-09 16:44:20,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +29: [2023-05-09 16:44:20,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +12: [2023-05-09 16:44:20,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +12: [2023-05-09 16:44:20,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 3: [2023-05-09 16:44:20,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +17: [2023-05-09 16:44:20,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +29: [2023-05-09 16:44:20,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +29: [2023-05-09 16:44:20,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +20: [2023-05-09 16:44:20,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +20: [2023-05-09 16:44:20,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +20: [2023-05-09 16:44:20,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +20: [2023-05-09 16:44:20,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +20: [2023-05-09 16:44:20,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +20: [2023-05-09 16:44:20,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +20: [2023-05-09 16:44:20,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +20: [2023-05-09 16:44:20,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +12: [2023-05-09 16:44:20,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +29: [2023-05-09 16:44:20,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +29: [2023-05-09 16:44:20,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +29: [2023-05-09 16:44:20,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +29: [2023-05-09 16:44:20,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +29: [2023-05-09 16:44:20,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +29: [2023-05-09 16:44:20,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +29: [2023-05-09 16:44:20,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +12: [2023-05-09 16:44:20,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +11: [2023-05-09 16:44:20,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +11: [2023-05-09 16:44:20,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +11: [2023-05-09 16:44:20,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +11: [2023-05-09 16:44:20,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +14: [2023-05-09 16:44:20,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +14: [2023-05-09 16:44:20,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +14: [2023-05-09 16:44:20,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +14: [2023-05-09 16:44:20,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +14: [2023-05-09 16:44:20,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +29: [2023-05-09 16:44:20,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +14: [2023-05-09 16:44:20,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +14: [2023-05-09 16:44:20,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +14: [2023-05-09 16:44:20,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +31: [2023-05-09 16:44:20,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +31: [2023-05-09 16:44:20,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +31: [2023-05-09 16:44:20,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +31: [2023-05-09 16:44:20,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +31: [2023-05-09 16:44:20,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +31: [2023-05-09 16:44:20,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +31: [2023-05-09 16:44:20,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +11: [2023-05-09 16:44:20,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +11: [2023-05-09 16:44:20,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +11: [2023-05-09 16:44:20,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +12: [2023-05-09 16:44:20,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +31: [2023-05-09 16:44:20,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +27: [2023-05-09 16:44:20,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +27: [2023-05-09 16:44:20,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +27: [2023-05-09 16:44:20,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +27: [2023-05-09 16:44:20,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +27: [2023-05-09 16:44:20,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +27: [2023-05-09 16:44:20,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +27: [2023-05-09 16:44:20,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +12: [2023-05-09 16:44:20,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +11: [2023-05-09 16:44:20,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +27: [2023-05-09 16:44:20,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +20: [2023-05-09 16:44:20,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +20: [2023-05-09 16:44:20,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +29: [2023-05-09 16:44:20,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +29: [2023-05-09 16:44:20,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +14: [2023-05-09 16:44:20,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +14: [2023-05-09 16:44:20,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +20: [2023-05-09 16:44:20,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +20: [2023-05-09 16:44:20,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +31: [2023-05-09 16:44:20,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +31: [2023-05-09 16:44:20,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +31: [2023-05-09 16:44:20,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +14: [2023-05-09 16:44:20,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +14: [2023-05-09 16:44:20,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +29: [2023-05-09 16:44:20,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +27: [2023-05-09 16:44:20,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +27: [2023-05-09 16:44:20,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +20: [2023-05-09 16:44:20,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +27: [2023-05-09 16:44:20,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 7: [2023-05-09 16:44:20,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 7: [2023-05-09 16:44:20,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +20: [2023-05-09 16:44:20,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 0: [2023-05-09 16:44:20,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +11: [2023-05-09 16:44:20,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +11: [2023-05-09 16:44:20,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +31: [2023-05-09 16:44:20,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +14: [2023-05-09 16:44:20,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +14: [2023-05-09 16:44:20,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +29: [2023-05-09 16:44:20,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +29: [2023-05-09 16:44:20,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +20: [2023-05-09 16:44:20,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +20: [2023-05-09 16:44:20,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +31: [2023-05-09 16:44:20,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +27: [2023-05-09 16:44:20,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +11: [2023-05-09 16:44:20,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +29: [2023-05-09 16:44:20,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +31: [2023-05-09 16:44:20,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +31: [2023-05-09 16:44:20,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +27: [2023-05-09 16:44:20,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 9: [2023-05-09 16:44:20,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 9: [2023-05-09 16:44:20,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 8: [2023-05-09 16:44:20,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +29: [2023-05-09 16:44:20,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 7: [2023-05-09 16:44:20,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +14: [2023-05-09 16:44:20,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +14: [2023-05-09 16:44:20,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +26: [2023-05-09 16:44:20,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 7: [2023-05-09 16:44:20,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +26: [2023-05-09 16:44:20,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +26: [2023-05-09 16:44:20,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +26: [2023-05-09 16:44:20,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 5: [2023-05-09 16:44:20,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 5: [2023-05-09 16:44:20,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 5: [2023-05-09 16:44:20,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +27: [2023-05-09 16:44:20,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +27: [2023-05-09 16:44:20,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +11: [2023-05-09 16:44:20,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +11: [2023-05-09 16:44:20,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 5: [2023-05-09 16:44:20,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 0: [2023-05-09 16:44:20,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +11: [2023-05-09 16:44:20,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 8: [2023-05-09 16:44:20,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +11: [2023-05-09 16:44:20,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +31: [2023-05-09 16:44:20,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +27: [2023-05-09 16:44:20,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +29: [2023-05-09 16:44:20,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +11: [2023-05-09 16:44:20,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +26: [2023-05-09 16:44:20,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +11: [2023-05-09 16:44:20,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +11: [2023-05-09 16:44:20,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +26: [2023-05-09 16:44:20,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +30: [2023-05-09 16:44:20,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +30: [2023-05-09 16:44:20,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +30: [2023-05-09 16:44:20,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +30: [2023-05-09 16:44:20,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +11: [2023-05-09 16:44:20,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +12: [2023-05-09 16:44:20,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +26: [2023-05-09 16:44:20,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +12: [2023-05-09 16:44:20,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +11: [2023-05-09 16:44:20,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +26: [2023-05-09 16:44:20,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +10: [2023-05-09 16:44:20,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +10: [2023-05-09 16:44:20,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +10: [2023-05-09 16:44:20,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +10: [2023-05-09 16:44:20,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +10: [2023-05-09 16:44:20,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +12: [2023-05-09 16:44:20,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +12: [2023-05-09 16:44:20,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +10: [2023-05-09 16:44:20,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +10: [2023-05-09 16:44:20,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +10: [2023-05-09 16:44:20,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +22: [2023-05-09 16:44:20,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +22: [2023-05-09 16:44:20,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +22: [2023-05-09 16:44:20,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +22: [2023-05-09 16:44:20,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +22: [2023-05-09 16:44:20,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +22: [2023-05-09 16:44:20,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +22: [2023-05-09 16:44:20,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +22: [2023-05-09 16:44:20,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +12: [2023-05-09 16:44:20,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 0: [2023-05-09 16:44:20,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +12: [2023-05-09 16:44:20,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +12: [2023-05-09 16:44:20,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +11: [2023-05-09 16:44:20,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +12: [2023-05-09 16:44:20,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +11: [2023-05-09 16:44:20,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 4: [2023-05-09 16:44:20,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 4: [2023-05-09 16:44:20,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 4: [2023-05-09 16:44:20,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +30: [2023-05-09 16:44:20,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +30: [2023-05-09 16:44:20,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +30: [2023-05-09 16:44:20,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +30: [2023-05-09 16:44:20,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +11: [2023-05-09 16:44:20,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 7: [2023-05-09 16:44:20,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 2: [2023-05-09 16:44:20,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +11: [2023-05-09 16:44:20,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 2: [2023-05-09 16:44:20,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +11: [2023-05-09 16:44:20,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +11: [2023-05-09 16:44:20,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +22: [2023-05-09 16:44:20,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +10: [2023-05-09 16:44:20,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +11: [2023-05-09 16:44:20,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +23: [2023-05-09 16:44:20,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 2: [2023-05-09 16:44:20,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 2: [2023-05-09 16:44:20,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 2: [2023-05-09 16:44:20,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 5: [2023-05-09 16:44:20,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +23: [2023-05-09 16:44:20,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +23: [2023-05-09 16:44:20,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 5: [2023-05-09 16:44:20,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +26: [2023-05-09 16:44:20,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +23: [2023-05-09 16:44:20,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 2: [2023-05-09 16:44:20,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +26: [2023-05-09 16:44:20,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 5: [2023-05-09 16:44:20,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 5: [2023-05-09 16:44:20,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +22: [2023-05-09 16:44:20,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 2: [2023-05-09 16:44:20,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 0: [2023-05-09 16:44:20,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +22: [2023-05-09 16:44:20,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +10: [2023-05-09 16:44:20,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +10: [2023-05-09 16:44:20,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +30: [2023-05-09 16:44:20,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +22: [2023-05-09 16:44:20,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +30: [2023-05-09 16:44:20,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +12: [2023-05-09 16:44:20,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +26: [2023-05-09 16:44:20,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +26: [2023-05-09 16:44:20,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 9: [2023-05-09 16:44:20,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +22: [2023-05-09 16:44:20,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +23: [2023-05-09 16:44:20,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +23: [2023-05-09 16:44:20,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +12: [2023-05-09 16:44:20,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +10: [2023-05-09 16:44:20,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +23: [2023-05-09 16:44:20,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +11: [2023-05-09 16:44:20,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +11: [2023-05-09 16:44:20,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +11: [2023-05-09 16:44:20,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +30: [2023-05-09 16:44:20,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +30: [2023-05-09 16:44:20,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +26: [2023-05-09 16:44:20,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +23: [2023-05-09 16:44:20,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +22: [2023-05-09 16:44:20,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +12: [2023-05-09 16:44:20,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +18: [2023-05-09 16:44:20,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +18: [2023-05-09 16:44:20,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +18: [2023-05-09 16:44:20,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +18: [2023-05-09 16:44:20,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +10: [2023-05-09 16:44:20,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +18: [2023-05-09 16:44:20,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +18: [2023-05-09 16:44:20,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +18: [2023-05-09 16:44:20,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +18: [2023-05-09 16:44:20,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +26: [2023-05-09 16:44:20,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +26: [2023-05-09 16:44:20,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 2: [2023-05-09 16:44:20,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +26: [2023-05-09 16:44:20,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +10: [2023-05-09 16:44:20,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +12: [2023-05-09 16:44:20,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +19: [2023-05-09 16:44:20,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +19: [2023-05-09 16:44:20,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +10: [2023-05-09 16:44:20,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +11: [2023-05-09 16:44:20,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +12: [2023-05-09 16:44:20,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +19: [2023-05-09 16:44:20,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +11: [2023-05-09 16:44:20,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +19: [2023-05-09 16:44:20,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +19: [2023-05-09 16:44:20,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +22: [2023-05-09 16:44:20,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +22: [2023-05-09 16:44:20,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +19: [2023-05-09 16:44:20,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +30: [2023-05-09 16:44:20,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +19: [2023-05-09 16:44:20,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +19: [2023-05-09 16:44:20,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +12: [2023-05-09 16:44:20,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +30: [2023-05-09 16:44:20,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 7: [2023-05-09 16:44:20,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +12: [2023-05-09 16:44:20,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +10: [2023-05-09 16:44:20,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 7: [2023-05-09 16:44:20,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +23: [2023-05-09 16:44:20,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +23: [2023-05-09 16:44:20,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +30: [2023-05-09 16:44:20,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +12: [2023-05-09 16:44:20,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +30: [2023-05-09 16:44:20,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +18: [2023-05-09 16:44:20,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +23: [2023-05-09 16:44:20,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +23: [2023-05-09 16:44:20,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +23: [2023-05-09 16:44:20,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +23: [2023-05-09 16:44:20,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +18: [2023-05-09 16:44:20,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 7: [2023-05-09 16:44:20,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 2: [2023-05-09 16:44:20,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +18: [2023-05-09 16:44:20,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +18: [2023-05-09 16:44:20,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 2: [2023-05-09 16:44:20,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 2: [2023-05-09 16:44:20,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +18: [2023-05-09 16:44:20,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +19: [2023-05-09 16:44:20,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +25: [2023-05-09 16:44:20,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +25: [2023-05-09 16:44:20,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +25: [2023-05-09 16:44:20,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +25: [2023-05-09 16:44:20,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +25: [2023-05-09 16:44:20,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +25: [2023-05-09 16:44:20,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +25: [2023-05-09 16:44:20,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +25: [2023-05-09 16:44:20,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +19: [2023-05-09 16:44:20,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +18: [2023-05-09 16:44:20,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +18: [2023-05-09 16:44:20,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +23: [2023-05-09 16:44:20,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +15: [2023-05-09 16:44:20,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +15: [2023-05-09 16:44:20,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +15: [2023-05-09 16:44:20,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +15: [2023-05-09 16:44:20,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +18: [2023-05-09 16:44:20,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +15: [2023-05-09 16:44:20,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +15: [2023-05-09 16:44:20,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +15: [2023-05-09 16:44:20,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +23: [2023-05-09 16:44:20,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +15: [2023-05-09 16:44:20,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 2: [2023-05-09 16:44:20,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +13: [2023-05-09 16:44:20,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +13: [2023-05-09 16:44:20,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +13: [2023-05-09 16:44:20,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +13: [2023-05-09 16:44:20,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +13: [2023-05-09 16:44:20,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +13: [2023-05-09 16:44:20,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +13: [2023-05-09 16:44:20,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +19: [2023-05-09 16:44:20,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +30: [2023-05-09 16:44:20,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +30: [2023-05-09 16:44:20,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +30: [2023-05-09 16:44:20,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +30: [2023-05-09 16:44:20,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +30: [2023-05-09 16:44:20,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +30: [2023-05-09 16:44:20,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +30: [2023-05-09 16:44:20,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +30: [2023-05-09 16:44:20,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +13: [2023-05-09 16:44:20,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +23: [2023-05-09 16:44:20,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +23: [2023-05-09 16:44:20,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +23: [2023-05-09 16:44:20,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +23: [2023-05-09 16:44:20,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +23: [2023-05-09 16:44:20,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +23: [2023-05-09 16:44:20,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +28: [2023-05-09 16:44:20,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +28: [2023-05-09 16:44:20,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +23: [2023-05-09 16:44:20,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 4: [2023-05-09 16:44:20,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +28: [2023-05-09 16:44:20,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +28: [2023-05-09 16:44:20,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +28: [2023-05-09 16:44:20,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +28: [2023-05-09 16:44:20,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +28: [2023-05-09 16:44:20,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +28: [2023-05-09 16:44:20,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 2: [2023-05-09 16:44:20,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +24: [2023-05-09 16:44:20,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +24: [2023-05-09 16:44:20,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +24: [2023-05-09 16:44:20,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +24: [2023-05-09 16:44:20,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +23: [2023-05-09 16:44:20,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +24: [2023-05-09 16:44:20,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +24: [2023-05-09 16:44:20,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +24: [2023-05-09 16:44:20,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +24: [2023-05-09 16:44:20,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +19: [2023-05-09 16:44:20,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +17: [2023-05-09 16:44:20,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +17: [2023-05-09 16:44:20,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +17: [2023-05-09 16:44:20,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +17: [2023-05-09 16:44:20,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +17: [2023-05-09 16:44:20,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +17: [2023-05-09 16:44:20,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +21: [2023-05-09 16:44:20,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +17: [2023-05-09 16:44:20,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +21: [2023-05-09 16:44:20,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +21: [2023-05-09 16:44:20,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +21: [2023-05-09 16:44:20,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +21: [2023-05-09 16:44:20,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +21: [2023-05-09 16:44:20,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +17: [2023-05-09 16:44:20,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +21: [2023-05-09 16:44:20,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +21: [2023-05-09 16:44:20,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +19: [2023-05-09 16:44:20,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +19: [2023-05-09 16:44:20,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +15: [2023-05-09 16:44:20,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +25: [2023-05-09 16:44:20,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +26: [2023-05-09 16:44:20,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +26: [2023-05-09 16:44:20,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +26: [2023-05-09 16:44:20,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +26: [2023-05-09 16:44:20,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +26: [2023-05-09 16:44:20,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +26: [2023-05-09 16:44:20,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +26: [2023-05-09 16:44:20,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +15: [2023-05-09 16:44:20,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +15: [2023-05-09 16:44:20,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +26: [2023-05-09 16:44:20,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +13: [2023-05-09 16:44:20,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +25: [2023-05-09 16:44:20,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +25: [2023-05-09 16:44:20,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +25: [2023-05-09 16:44:20,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +30: [2023-05-09 16:44:20,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +30: [2023-05-09 16:44:20,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +23: [2023-05-09 16:44:20,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +28: [2023-05-09 16:44:20,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +24: [2023-05-09 16:44:20,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +19: [2023-05-09 16:44:20,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +21: [2023-05-09 16:44:20,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +17: [2023-05-09 16:44:20,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +30: [2023-05-09 16:44:20,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +30: [2023-05-09 16:44:20,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +19: [2023-05-09 16:44:20,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +17: [2023-05-09 16:44:20,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +23: [2023-05-09 16:44:20,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +23: [2023-05-09 16:44:20,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +25: [2023-05-09 16:44:20,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +28: [2023-05-09 16:44:20,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +13: [2023-05-09 16:44:20,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +13: [2023-05-09 16:44:20,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +15: [2023-05-09 16:44:20,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +17: [2023-05-09 16:44:20,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +17: [2023-05-09 16:44:20,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +15: [2023-05-09 16:44:20,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +15: [2023-05-09 16:44:20,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 5: [2023-05-09 16:44:20,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +28: [2023-05-09 16:44:20,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +28: [2023-05-09 16:44:20,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 6: [2023-05-09 16:44:20,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +25: [2023-05-09 16:44:20,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +28: [2023-05-09 16:44:20,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +30: [2023-05-09 16:44:20,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +13: [2023-05-09 16:44:20,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 4: [2023-05-09 16:44:20,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +28: [2023-05-09 16:44:20,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 6: [2023-05-09 16:44:20,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +25: [2023-05-09 16:44:20,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +25: [2023-05-09 16:44:20,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +21: [2023-05-09 16:44:20,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +21: [2023-05-09 16:44:20,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +24: [2023-05-09 16:44:20,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +24: [2023-05-09 16:44:20,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +21: [2023-05-09 16:44:20,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +24: [2023-05-09 16:44:20,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +24: [2023-05-09 16:44:20,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +28: [2023-05-09 16:44:20,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +26: [2023-05-09 16:44:20,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +26: [2023-05-09 16:44:20,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +28: [2023-05-09 16:44:20,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +30: [2023-05-09 16:44:20,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +23: [2023-05-09 16:44:20,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +30: [2023-05-09 16:44:20,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +28: [2023-05-09 16:44:20,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +30: [2023-05-09 16:44:20,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +23: [2023-05-09 16:44:20,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +23: [2023-05-09 16:44:20,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +15: [2023-05-09 16:44:20,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +17: [2023-05-09 16:44:20,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +21: [2023-05-09 16:44:20,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +13: [2023-05-09 16:44:20,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +26: [2023-05-09 16:44:20,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +28: [2023-05-09 16:44:20,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +13: [2023-05-09 16:44:20,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +16: [2023-05-09 16:44:20,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +16: [2023-05-09 16:44:20,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +16: [2023-05-09 16:44:20,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +16: [2023-05-09 16:44:20,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +16: [2023-05-09 16:44:20,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +16: [2023-05-09 16:44:20,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +16: [2023-05-09 16:44:20,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +16: [2023-05-09 16:44:20,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +17: [2023-05-09 16:44:20,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 5: [2023-05-09 16:44:20,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +24: [2023-05-09 16:44:20,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +17: [2023-05-09 16:44:20,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +24: [2023-05-09 16:44:20,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +23: [2023-05-09 16:44:20,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +28: [2023-05-09 16:44:20,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +28: [2023-05-09 16:44:20,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +13: [2023-05-09 16:44:20,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +28: [2023-05-09 16:44:20,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +26: [2023-05-09 16:44:20,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +21: [2023-05-09 16:44:20,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +28: [2023-05-09 16:44:20,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +24: [2023-05-09 16:44:20,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +21: [2023-05-09 16:44:20,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +21: [2023-05-09 16:44:20,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +28: [2023-05-09 16:44:20,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +26: [2023-05-09 16:44:20,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +17: [2023-05-09 16:44:20,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +28: [2023-05-09 16:44:20,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 4: [2023-05-09 16:44:20,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +15: [2023-05-09 16:44:20,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +26: [2023-05-09 16:44:20,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +26: [2023-05-09 16:44:20,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +16: [2023-05-09 16:44:20,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 5: [2023-05-09 16:44:20,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +16: [2023-05-09 16:44:20,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +16: [2023-05-09 16:44:20,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +16: [2023-05-09 16:44:20,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +23: [2023-05-09 16:44:20,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +13: [2023-05-09 16:44:20,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +16: [2023-05-09 16:44:20,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +16: [2023-05-09 16:44:20,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +26: [2023-05-09 16:44:20,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +28: [2023-05-09 16:44:20,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +28: [2023-05-09 16:44:20,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +15: [2023-05-09 16:44:20,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +15: [2023-05-09 16:44:20,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +15: [2023-05-09 16:44:20,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +15: [2023-05-09 16:44:20,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +31: [2023-05-09 16:44:20,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +31: [2023-05-09 16:44:20,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +31: [2023-05-09 16:44:20,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +31: [2023-05-09 16:44:20,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +16: [2023-05-09 16:44:20,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +16: [2023-05-09 16:44:20,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +16: [2023-05-09 16:44:20,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +16: [2023-05-09 16:44:20,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +16: [2023-05-09 16:44:20,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +20: [2023-05-09 16:44:20,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +20: [2023-05-09 16:44:20,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +20: [2023-05-09 16:44:20,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +20: [2023-05-09 16:44:20,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +21: [2023-05-09 16:44:20,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +21: [2023-05-09 16:44:20,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +21: [2023-05-09 16:44:20,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +21: [2023-05-09 16:44:20,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 1: [2023-05-09 16:44:20,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +16: [2023-05-09 16:44:20,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +28: [2023-05-09 16:44:20,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +16: [2023-05-09 16:44:20,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +28: [2023-05-09 16:44:20,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 1: [2023-05-09 16:44:20,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +16: [2023-05-09 16:44:20,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +22: [2023-05-09 16:44:20,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +22: [2023-05-09 16:44:20,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +22: [2023-05-09 16:44:20,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +22: [2023-05-09 16:44:20,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 1: [2023-05-09 16:44:20,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +31: [2023-05-09 16:44:20,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 3: [2023-05-09 16:44:20,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 3: [2023-05-09 16:44:20,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +15: [2023-05-09 16:44:20,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +15: [2023-05-09 16:44:20,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +15: [2023-05-09 16:44:20,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +15: [2023-05-09 16:44:20,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +31: [2023-05-09 16:44:20,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +31: [2023-05-09 16:44:20,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +31: [2023-05-09 16:44:20,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +25: [2023-05-09 16:44:20,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +28: [2023-05-09 16:44:20,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +28: [2023-05-09 16:44:20,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +28: [2023-05-09 16:44:20,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +28: [2023-05-09 16:44:20,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +25: [2023-05-09 16:44:20,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +25: [2023-05-09 16:44:20,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +25: [2023-05-09 16:44:20,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +21: [2023-05-09 16:44:20,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +21: [2023-05-09 16:44:20,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +20: [2023-05-09 16:44:20,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +20: [2023-05-09 16:44:20,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 8: [2023-05-09 16:44:20,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 8: [2023-05-09 16:44:20,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +20: [2023-05-09 16:44:20,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +20: [2023-05-09 16:44:20,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +16: [2023-05-09 16:44:20,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +21: [2023-05-09 16:44:20,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +16: [2023-05-09 16:44:20,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +21: [2023-05-09 16:44:20,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +16: [2023-05-09 16:44:20,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +16: [2023-05-09 16:44:20,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 3: [2023-05-09 16:44:20,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +22: [2023-05-09 16:44:20,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +22: [2023-05-09 16:44:20,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +22: [2023-05-09 16:44:20,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +22: [2023-05-09 16:44:20,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +16: [2023-05-09 16:44:20,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +25: [2023-05-09 16:44:20,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +16: [2023-05-09 16:44:20,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 3: [2023-05-09 16:44:20,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +25: [2023-05-09 16:44:20,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +16: [2023-05-09 16:44:20,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +16: [2023-05-09 16:44:20,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 3: [2023-05-09 16:44:20,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 6: [2023-05-09 16:44:20,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 3: [2023-05-09 16:44:20,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 6: [2023-05-09 16:44:20,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 6: [2023-05-09 16:44:20,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 3: [2023-05-09 16:44:20,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 0: [2023-05-09 16:44:20,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 0: [2023-05-09 16:44:20,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. + 8: [2023-05-09 16:44:20,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 0: [2023-05-09 16:44:20,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt. +25: [2023-05-09 16:44:20,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 8: [2023-05-09 16:44:20,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +25: [2023-05-09 16:44:20,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 1: [2023-05-09 16:44:20,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 1: [2023-05-09 16:44:20,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +15: [2023-05-09 16:44:20,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +15: [2023-05-09 16:44:20,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +15: [2023-05-09 16:44:20,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 3: [2023-05-09 16:44:20,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 3: [2023-05-09 16:44:20,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +15: [2023-05-09 16:44:20,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 3: [2023-05-09 16:44:20,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +15: [2023-05-09 16:44:20,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +15: [2023-05-09 16:44:20,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +31: [2023-05-09 16:44:20,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 8: [2023-05-09 16:44:20,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +31: [2023-05-09 16:44:20,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +31: [2023-05-09 16:44:20,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +31: [2023-05-09 16:44:20,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 1: [2023-05-09 16:44:20,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +21: [2023-05-09 16:44:20,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +21: [2023-05-09 16:44:20,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 9: [2023-05-09 16:44:20,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 9: [2023-05-09 16:44:20,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 9: [2023-05-09 16:44:20,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +15: [2023-05-09 16:44:20,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +15: [2023-05-09 16:44:20,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 9: [2023-05-09 16:44:20,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 6: [2023-05-09 16:44:20,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +20: [2023-05-09 16:44:21,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +20: [2023-05-09 16:44:21,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +21: [2023-05-09 16:44:21,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +21: [2023-05-09 16:44:21,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 0: [2023-05-09 16:44:21,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +20: [2023-05-09 16:44:21,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 8: [2023-05-09 16:44:21,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 3: [2023-05-09 16:44:21,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +20: [2023-05-09 16:44:21,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +31: [2023-05-09 16:44:21,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +31: [2023-05-09 16:44:21,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +31: [2023-05-09 16:44:21,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +31: [2023-05-09 16:44:21,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +20: [2023-05-09 16:44:21,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +20: [2023-05-09 16:44:21,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 0: [2023-05-09 16:44:21,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt... +16: [2023-05-09 16:44:21,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +22: [2023-05-09 16:44:21,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +22: [2023-05-09 16:44:21,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +16: [2023-05-09 16:44:21,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +21: [2023-05-09 16:44:21,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +21: [2023-05-09 16:44:21,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 3: [2023-05-09 16:44:21,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +21: [2023-05-09 16:44:21,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 8: [2023-05-09 16:44:21,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 8: [2023-05-09 16:44:21,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +25: [2023-05-09 16:44:21,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +25: [2023-05-09 16:44:21,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +25: [2023-05-09 16:44:21,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +25: [2023-05-09 16:44:21,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +22: [2023-05-09 16:44:21,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +21: [2023-05-09 16:44:21,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 3: [2023-05-09 16:44:21,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +22: [2023-05-09 16:44:21,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +22: [2023-05-09 16:44:21,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +22: [2023-05-09 16:44:21,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +20: [2023-05-09 16:44:21,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +22: [2023-05-09 16:44:21,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +22: [2023-05-09 16:44:21,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 6: [2023-05-09 16:44:21,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 6: [2023-05-09 16:44:21,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +20: [2023-05-09 16:44:21,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 1: [2023-05-09 16:44:21,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +25: [2023-05-09 16:44:21,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 6: [2023-05-09 16:44:21,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 6: [2023-05-09 16:44:21,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 8: [2023-05-09 16:44:21,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +25: [2023-05-09 16:44:21,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 6: [2023-05-09 16:44:21,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 6: [2023-05-09 16:44:21,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... +25: [2023-05-09 16:44:21,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. +25: [2023-05-09 16:44:21,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 3: [2023-05-09 16:44:21,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 0: [2023-05-09 16:44:21,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 9: [2023-05-09 16:44:21,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 6: [2023-05-09 16:44:21,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 3: [2023-05-09 16:44:21,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 6: [2023-05-09 16:44:21,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt... + 0: [2023-05-09 16:44:21,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +29: [2023-05-09 16:44:21,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +29: [2023-05-09 16:44:21,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +29: [2023-05-09 16:44:21,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +29: [2023-05-09 16:44:21,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +29: [2023-05-09 16:44:21,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +29: [2023-05-09 16:44:21,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +29: [2023-05-09 16:44:21,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +29: [2023-05-09 16:44:21,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +20: [2023-05-09 16:44:21,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +20: [2023-05-09 16:44:21,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +20: [2023-05-09 16:44:21,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +20: [2023-05-09 16:44:21,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +20: [2023-05-09 16:44:21,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +20: [2023-05-09 16:44:21,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +20: [2023-05-09 16:44:21,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +29: [2023-05-09 16:44:21,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +20: [2023-05-09 16:44:21,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +31: [2023-05-09 16:44:21,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +31: [2023-05-09 16:44:21,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +29: [2023-05-09 16:44:21,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +29: [2023-05-09 16:44:21,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +31: [2023-05-09 16:44:21,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +31: [2023-05-09 16:44:21,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +31: [2023-05-09 16:44:21,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +31: [2023-05-09 16:44:21,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +31: [2023-05-09 16:44:21,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +31: [2023-05-09 16:44:21,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +20: [2023-05-09 16:44:21,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +20: [2023-05-09 16:44:21,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +29: [2023-05-09 16:44:21,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +29: [2023-05-09 16:44:21,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +29: [2023-05-09 16:44:21,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +29: [2023-05-09 16:44:21,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +31: [2023-05-09 16:44:21,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +31: [2023-05-09 16:44:21,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +31: [2023-05-09 16:44:21,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +20: [2023-05-09 16:44:21,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +20: [2023-05-09 16:44:21,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +29: [2023-05-09 16:44:21,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +31: [2023-05-09 16:44:21,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +31: [2023-05-09 16:44:21,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +31: [2023-05-09 16:44:21,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +20: [2023-05-09 16:44:21,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +20: [2023-05-09 16:44:21,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +31: [2023-05-09 16:44:21,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +20: [2023-05-09 16:44:21,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +20: [2023-05-09 16:44:21,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +31: [2023-05-09 16:44:21,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +13: [2023-05-09 16:44:21,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +13: [2023-05-09 16:44:21,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +13: [2023-05-09 16:44:21,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +13: [2023-05-09 16:44:21,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +22: [2023-05-09 16:44:21,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +22: [2023-05-09 16:44:21,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +22: [2023-05-09 16:44:21,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +22: [2023-05-09 16:44:21,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +22: [2023-05-09 16:44:21,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +22: [2023-05-09 16:44:21,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +13: [2023-05-09 16:44:21,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +22: [2023-05-09 16:44:21,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +22: [2023-05-09 16:44:21,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +13: [2023-05-09 16:44:21,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +13: [2023-05-09 16:44:21,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +14: [2023-05-09 16:44:21,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +14: [2023-05-09 16:44:21,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +14: [2023-05-09 16:44:21,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +14: [2023-05-09 16:44:21,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +13: [2023-05-09 16:44:21,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +14: [2023-05-09 16:44:21,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +14: [2023-05-09 16:44:21,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +14: [2023-05-09 16:44:21,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +14: [2023-05-09 16:44:21,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +24: [2023-05-09 16:44:21,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +24: [2023-05-09 16:44:21,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +24: [2023-05-09 16:44:21,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +24: [2023-05-09 16:44:21,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +22: [2023-05-09 16:44:21,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +13: [2023-05-09 16:44:21,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +14: [2023-05-09 16:44:21,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +13: [2023-05-09 16:44:21,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +24: [2023-05-09 16:44:21,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +24: [2023-05-09 16:44:21,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +24: [2023-05-09 16:44:21,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +24: [2023-05-09 16:44:21,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +13: [2023-05-09 16:44:21,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +13: [2023-05-09 16:44:21,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +22: [2023-05-09 16:44:21,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +22: [2023-05-09 16:44:21,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 9: [2023-05-09 16:44:21,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 9: [2023-05-09 16:44:21,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 9: [2023-05-09 16:44:21,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +14: [2023-05-09 16:44:21,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +13: [2023-05-09 16:44:21,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +22: [2023-05-09 16:44:21,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +13: [2023-05-09 16:44:21,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +27: [2023-05-09 16:44:21,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +13: [2023-05-09 16:44:21,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +27: [2023-05-09 16:44:21,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +27: [2023-05-09 16:44:21,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +27: [2023-05-09 16:44:21,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +14: [2023-05-09 16:44:21,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +13: [2023-05-09 16:44:21,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +24: [2023-05-09 16:44:21,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +22: [2023-05-09 16:44:21,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +22: [2023-05-09 16:44:21,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +27: [2023-05-09 16:44:21,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +27: [2023-05-09 16:44:21,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +27: [2023-05-09 16:44:21,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +27: [2023-05-09 16:44:21,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +24: [2023-05-09 16:44:21,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +14: [2023-05-09 16:44:21,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +24: [2023-05-09 16:44:21,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +24: [2023-05-09 16:44:21,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +22: [2023-05-09 16:44:21,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +24: [2023-05-09 16:44:21,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +24: [2023-05-09 16:44:21,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 9: [2023-05-09 16:44:21,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +14: [2023-05-09 16:44:21,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +14: [2023-05-09 16:44:21,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +24: [2023-05-09 16:44:21,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +24: [2023-05-09 16:44:21,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +22: [2023-05-09 16:44:21,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +27: [2023-05-09 16:44:21,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +27: [2023-05-09 16:44:21,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +14: [2023-05-09 16:44:21,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +18: [2023-05-09 16:44:21,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +18: [2023-05-09 16:44:21,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +18: [2023-05-09 16:44:21,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +18: [2023-05-09 16:44:21,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +27: [2023-05-09 16:44:21,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +12: [2023-05-09 16:44:21,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +12: [2023-05-09 16:44:21,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +12: [2023-05-09 16:44:21,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +12: [2023-05-09 16:44:21,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +12: [2023-05-09 16:44:21,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +12: [2023-05-09 16:44:21,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +12: [2023-05-09 16:44:21,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +12: [2023-05-09 16:44:21,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +14: [2023-05-09 16:44:21,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 5: [2023-05-09 16:44:21,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 5: [2023-05-09 16:44:21,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +14: [2023-05-09 16:44:21,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 9: [2023-05-09 16:44:21,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +10: [2023-05-09 16:44:21,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +14: [2023-05-09 16:44:21,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +14: [2023-05-09 16:44:21,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +10: [2023-05-09 16:44:21,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +10: [2023-05-09 16:44:21,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +10: [2023-05-09 16:44:21,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +14: [2023-05-09 16:44:21,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +10: [2023-05-09 16:44:21,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +10: [2023-05-09 16:44:21,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 5: [2023-05-09 16:44:21,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +10: [2023-05-09 16:44:21,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 7: [2023-05-09 16:44:21,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +11: [2023-05-09 16:44:21,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +11: [2023-05-09 16:44:21,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +10: [2023-05-09 16:44:21,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +27: [2023-05-09 16:44:21,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +11: [2023-05-09 16:44:21,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +11: [2023-05-09 16:44:21,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +11: [2023-05-09 16:44:21,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +27: [2023-05-09 16:44:21,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +11: [2023-05-09 16:44:21,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +11: [2023-05-09 16:44:21,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +18: [2023-05-09 16:44:21,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +18: [2023-05-09 16:44:21,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +27: [2023-05-09 16:44:21,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +11: [2023-05-09 16:44:21,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +19: [2023-05-09 16:44:21,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +19: [2023-05-09 16:44:21,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +19: [2023-05-09 16:44:21,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +19: [2023-05-09 16:44:21,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +19: [2023-05-09 16:44:21,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +19: [2023-05-09 16:44:21,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +19: [2023-05-09 16:44:21,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +19: [2023-05-09 16:44:21,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +18: [2023-05-09 16:44:21,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +18: [2023-05-09 16:44:21,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +27: [2023-05-09 16:44:21,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 2: [2023-05-09 16:44:21,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +12: [2023-05-09 16:44:21,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +14: [2023-05-09 16:44:21,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +14: [2023-05-09 16:44:21,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +14: [2023-05-09 16:44:21,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +14: [2023-05-09 16:44:21,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +10: [2023-05-09 16:44:21,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +27: [2023-05-09 16:44:21,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +18: [2023-05-09 16:44:21,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +18: [2023-05-09 16:44:21,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +18: [2023-05-09 16:44:21,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 7: [2023-05-09 16:44:21,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +18: [2023-05-09 16:44:21,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +10: [2023-05-09 16:44:21,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +13: [2023-05-09 16:44:21,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +13: [2023-05-09 16:44:21,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +12: [2023-05-09 16:44:21,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +12: [2023-05-09 16:44:21,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +13: [2023-05-09 16:44:21,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +12: [2023-05-09 16:44:21,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +13: [2023-05-09 16:44:21,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +10: [2023-05-09 16:44:21,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +13: [2023-05-09 16:44:21,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +13: [2023-05-09 16:44:21,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +12: [2023-05-09 16:44:21,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +19: [2023-05-09 16:44:21,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +19: [2023-05-09 16:44:21,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +13: [2023-05-09 16:44:21,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +13: [2023-05-09 16:44:21,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +23: [2023-05-09 16:44:21,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +23: [2023-05-09 16:44:21,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +18: [2023-05-09 16:44:21,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +19: [2023-05-09 16:44:21,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +18: [2023-05-09 16:44:21,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +14: [2023-05-09 16:44:21,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +26: [2023-05-09 16:44:21,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +26: [2023-05-09 16:44:21,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +14: [2023-05-09 16:44:21,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +23: [2023-05-09 16:44:21,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +23: [2023-05-09 16:44:21,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +26: [2023-05-09 16:44:21,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +10: [2023-05-09 16:44:21,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +26: [2023-05-09 16:44:21,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 2: [2023-05-09 16:44:21,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +11: [2023-05-09 16:44:21,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +11: [2023-05-09 16:44:21,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +19: [2023-05-09 16:44:21,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +19: [2023-05-09 16:44:21,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 7: [2023-05-09 16:44:21,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +10: [2023-05-09 16:44:21,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +12: [2023-05-09 16:44:21,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +18: [2023-05-09 16:44:21,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +18: [2023-05-09 16:44:21,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 1: [2023-05-09 16:44:21,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 1: [2023-05-09 16:44:21,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 1: [2023-05-09 16:44:21,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +10: [2023-05-09 16:44:21,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +12: [2023-05-09 16:44:21,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +19: [2023-05-09 16:44:21,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +19: [2023-05-09 16:44:21,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +12: [2023-05-09 16:44:21,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 2: [2023-05-09 16:44:21,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 5: [2023-05-09 16:44:21,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +14: [2023-05-09 16:44:21,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +14: [2023-05-09 16:44:21,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +14: [2023-05-09 16:44:21,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +14: [2023-05-09 16:44:21,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 2: [2023-05-09 16:44:21,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +19: [2023-05-09 16:44:21,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +19: [2023-05-09 16:44:21,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +23: [2023-05-09 16:44:21,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +23: [2023-05-09 16:44:21,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +11: [2023-05-09 16:44:21,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +10: [2023-05-09 16:44:21,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +15: [2023-05-09 16:44:21,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +15: [2023-05-09 16:44:21,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +15: [2023-05-09 16:44:21,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +15: [2023-05-09 16:44:21,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +15: [2023-05-09 16:44:21,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +15: [2023-05-09 16:44:21,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +26: [2023-05-09 16:44:21,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +15: [2023-05-09 16:44:21,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +23: [2023-05-09 16:44:21,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +15: [2023-05-09 16:44:21,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +23: [2023-05-09 16:44:21,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +26: [2023-05-09 16:44:21,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +30: [2023-05-09 16:44:21,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +26: [2023-05-09 16:44:21,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +26: [2023-05-09 16:44:21,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +30: [2023-05-09 16:44:21,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +11: [2023-05-09 16:44:21,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 7: [2023-05-09 16:44:21,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 7: [2023-05-09 16:44:21,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +11: [2023-05-09 16:44:21,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +29: [2023-05-09 16:44:21,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +27: [2023-05-09 16:44:21,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +27: [2023-05-09 16:44:21,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +19: [2023-05-09 16:44:21,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +29: [2023-05-09 16:44:21,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +29: [2023-05-09 16:44:21,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 7: [2023-05-09 16:44:21,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +30: [2023-05-09 16:44:21,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +14: [2023-05-09 16:44:21,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +14: [2023-05-09 16:44:21,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +27: [2023-05-09 16:44:21,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +17: [2023-05-09 16:44:21,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 7: [2023-05-09 16:44:21,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +29: [2023-05-09 16:44:21,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +19: [2023-05-09 16:44:21,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +27: [2023-05-09 16:44:21,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +30: [2023-05-09 16:44:21,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +17: [2023-05-09 16:44:21,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 7: [2023-05-09 16:44:21,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +17: [2023-05-09 16:44:21,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +17: [2023-05-09 16:44:21,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +13: [2023-05-09 16:44:21,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +13: [2023-05-09 16:44:21,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +10: [2023-05-09 16:44:21,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +11: [2023-05-09 16:44:21,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 2: [2023-05-09 16:44:21,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +11: [2023-05-09 16:44:21,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 2: [2023-05-09 16:44:21,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +12: [2023-05-09 16:44:21,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +12: [2023-05-09 16:44:21,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +12: [2023-05-09 16:44:21,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 4: [2023-05-09 16:44:21,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +12: [2023-05-09 16:44:21,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 4: [2023-05-09 16:44:21,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 4: [2023-05-09 16:44:21,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +10: [2023-05-09 16:44:21,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +10: [2023-05-09 16:44:21,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 4: [2023-05-09 16:44:21,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 2: [2023-05-09 16:44:21,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +19: [2023-05-09 16:44:21,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +19: [2023-05-09 16:44:21,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +25: [2023-05-09 16:44:21,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +10: [2023-05-09 16:44:21,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +25: [2023-05-09 16:44:21,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +23: [2023-05-09 16:44:21,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +23: [2023-05-09 16:44:21,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +23: [2023-05-09 16:44:21,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +25: [2023-05-09 16:44:21,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +25: [2023-05-09 16:44:21,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +25: [2023-05-09 16:44:21,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +23: [2023-05-09 16:44:21,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +25: [2023-05-09 16:44:21,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +25: [2023-05-09 16:44:21,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +25: [2023-05-09 16:44:21,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +10: [2023-05-09 16:44:21,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +23: [2023-05-09 16:44:21,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +23: [2023-05-09 16:44:21,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +11: [2023-05-09 16:44:21,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +11: [2023-05-09 16:44:21,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +23: [2023-05-09 16:44:21,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +23: [2023-05-09 16:44:21,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +30: [2023-05-09 16:44:21,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 5: [2023-05-09 16:44:21,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +30: [2023-05-09 16:44:21,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +26: [2023-05-09 16:44:21,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +30: [2023-05-09 16:44:21,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +15: [2023-05-09 16:44:21,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +26: [2023-05-09 16:44:21,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 2: [2023-05-09 16:44:21,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +30: [2023-05-09 16:44:21,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +26: [2023-05-09 16:44:21,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +11: [2023-05-09 16:44:21,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +26: [2023-05-09 16:44:21,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +26: [2023-05-09 16:44:21,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +12: [2023-05-09 16:44:21,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +26: [2023-05-09 16:44:21,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +15: [2023-05-09 16:44:21,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +15: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +28: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +12: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +26: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +26: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +28: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +28: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +26: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +28: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +12: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +19: [2023-05-09 16:44:21,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +19: [2023-05-09 16:44:21,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +29: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +28: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +28: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +17: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +17: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +28: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +17: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +17: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +17: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +13: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +13: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +27: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +17: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +11: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +27: [2023-05-09 16:44:21,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +29: [2023-05-09 16:44:21,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +29: [2023-05-09 16:44:21,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +12: [2023-05-09 16:44:21,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +29: [2023-05-09 16:44:21,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +10: [2023-05-09 16:44:21,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +26: [2023-05-09 16:44:21,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +23: [2023-05-09 16:44:21,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +23: [2023-05-09 16:44:21,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +11: [2023-05-09 16:44:21,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +17: [2023-05-09 16:44:21,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +24: [2023-05-09 16:44:21,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +28: [2023-05-09 16:44:21,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +24: [2023-05-09 16:44:21,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +15: [2023-05-09 16:44:21,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +27: [2023-05-09 16:44:21,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +23: [2023-05-09 16:44:21,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +24: [2023-05-09 16:44:21,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +24: [2023-05-09 16:44:21,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +11: [2023-05-09 16:44:21,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +27: [2023-05-09 16:44:21,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +11: [2023-05-09 16:44:21,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +19: [2023-05-09 16:44:21,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +10: [2023-05-09 16:44:21,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +13: [2023-05-09 16:44:21,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +10: [2023-05-09 16:44:21,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +23: [2023-05-09 16:44:21,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +19: [2023-05-09 16:44:21,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +19: [2023-05-09 16:44:21,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +24: [2023-05-09 16:44:21,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +24: [2023-05-09 16:44:21,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +17: [2023-05-09 16:44:21,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +24: [2023-05-09 16:44:21,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +17: [2023-05-09 16:44:21,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +10: [2023-05-09 16:44:21,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +19: [2023-05-09 16:44:21,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +19: [2023-05-09 16:44:21,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 2: [2023-05-09 16:44:21,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +30: [2023-05-09 16:44:21,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +30: [2023-05-09 16:44:21,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +17: [2023-05-09 16:44:21,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +17: [2023-05-09 16:44:21,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +30: [2023-05-09 16:44:21,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +30: [2023-05-09 16:44:21,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +30: [2023-05-09 16:44:21,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +17: [2023-05-09 16:44:21,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +30: [2023-05-09 16:44:21,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +25: [2023-05-09 16:44:21,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +24: [2023-05-09 16:44:21,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +30: [2023-05-09 16:44:21,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +30: [2023-05-09 16:44:21,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +15: [2023-05-09 16:44:21,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +11: [2023-05-09 16:44:21,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 6: [2023-05-09 16:44:21,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +15: [2023-05-09 16:44:21,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +23: [2023-05-09 16:44:21,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +23: [2023-05-09 16:44:21,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +23: [2023-05-09 16:44:21,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +13: [2023-05-09 16:44:21,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 6: [2023-05-09 16:44:21,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 6: [2023-05-09 16:44:21,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +15: [2023-05-09 16:44:21,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 6: [2023-05-09 16:44:21,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +30: [2023-05-09 16:44:21,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +26: [2023-05-09 16:44:21,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +26: [2023-05-09 16:44:21,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +26: [2023-05-09 16:44:21,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +26: [2023-05-09 16:44:21,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +26: [2023-05-09 16:44:21,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +26: [2023-05-09 16:44:21,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +30: [2023-05-09 16:44:21,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 6: [2023-05-09 16:44:21,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +29: [2023-05-09 16:44:21,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +28: [2023-05-09 16:44:21,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +23: [2023-05-09 16:44:21,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +23: [2023-05-09 16:44:21,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +15: [2023-05-09 16:44:21,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +23: [2023-05-09 16:44:21,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +29: [2023-05-09 16:44:21,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +12: [2023-05-09 16:44:21,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +19: [2023-05-09 16:44:21,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +19: [2023-05-09 16:44:21,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +12: [2023-05-09 16:44:21,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +12: [2023-05-09 16:44:21,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +12: [2023-05-09 16:44:21,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +25: [2023-05-09 16:44:21,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +24: [2023-05-09 16:44:21,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +29: [2023-05-09 16:44:21,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +29: [2023-05-09 16:44:21,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 1: [2023-05-09 16:44:21,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +25: [2023-05-09 16:44:21,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 1: [2023-05-09 16:44:21,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +25: [2023-05-09 16:44:21,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +13: [2023-05-09 16:44:21,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +17: [2023-05-09 16:44:21,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +11: [2023-05-09 16:44:21,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +29: [2023-05-09 16:44:21,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +29: [2023-05-09 16:44:21,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +26: [2023-05-09 16:44:21,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +28: [2023-05-09 16:44:21,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +26: [2023-05-09 16:44:21,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 2: [2023-05-09 16:44:21,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +29: [2023-05-09 16:44:21,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +29: [2023-05-09 16:44:21,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +11: [2023-05-09 16:44:21,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +26: [2023-05-09 16:44:21,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +27: [2023-05-09 16:44:21,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +27: [2023-05-09 16:44:21,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +27: [2023-05-09 16:44:21,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +12: [2023-05-09 16:44:21,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +12: [2023-05-09 16:44:21,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +12: [2023-05-09 16:44:21,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +12: [2023-05-09 16:44:21,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +23: [2023-05-09 16:44:21,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +27: [2023-05-09 16:44:21,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +27: [2023-05-09 16:44:21,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +27: [2023-05-09 16:44:21,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 7: [2023-05-09 16:44:21,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +17: [2023-05-09 16:44:21,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +28: [2023-05-09 16:44:21,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +23: [2023-05-09 16:44:21,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +30: [2023-05-09 16:44:21,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +11: [2023-05-09 16:44:21,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +30: [2023-05-09 16:44:21,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +19: [2023-05-09 16:44:21,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +19: [2023-05-09 16:44:21,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 4: [2023-05-09 16:44:21,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +11: [2023-05-09 16:44:21,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +11: [2023-05-09 16:44:21,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 4: [2023-05-09 16:44:21,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +13: [2023-05-09 16:44:21,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +24: [2023-05-09 16:44:21,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +11: [2023-05-09 16:44:21,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +30: [2023-05-09 16:44:21,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +11: [2023-05-09 16:44:21,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +25: [2023-05-09 16:44:21,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +24: [2023-05-09 16:44:21,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +28: [2023-05-09 16:44:21,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +30: [2023-05-09 16:44:21,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +30: [2023-05-09 16:44:21,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +30: [2023-05-09 16:44:21,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +24: [2023-05-09 16:44:21,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +30: [2023-05-09 16:44:21,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +17: [2023-05-09 16:44:21,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +10: [2023-05-09 16:44:21,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +10: [2023-05-09 16:44:21,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +15: [2023-05-09 16:44:21,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +15: [2023-05-09 16:44:21,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 5: [2023-05-09 16:44:21,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +15: [2023-05-09 16:44:21,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +10: [2023-05-09 16:44:21,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +10: [2023-05-09 16:44:21,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +15: [2023-05-09 16:44:21,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +23: [2023-05-09 16:44:21,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +23: [2023-05-09 16:44:21,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +17: [2023-05-09 16:44:21,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 6: [2023-05-09 16:44:21,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +25: [2023-05-09 16:44:21,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +23: [2023-05-09 16:44:21,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +25: [2023-05-09 16:44:21,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +30: [2023-05-09 16:44:21,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +23: [2023-05-09 16:44:21,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +10: [2023-05-09 16:44:21,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +26: [2023-05-09 16:44:21,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +26: [2023-05-09 16:44:21,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +21: [2023-05-09 16:44:21,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +21: [2023-05-09 16:44:21,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +28: [2023-05-09 16:44:21,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +26: [2023-05-09 16:44:21,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +21: [2023-05-09 16:44:21,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +21: [2023-05-09 16:44:21,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +21: [2023-05-09 16:44:21,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +21: [2023-05-09 16:44:21,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +21: [2023-05-09 16:44:21,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +10: [2023-05-09 16:44:21,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +11: [2023-05-09 16:44:21,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +11: [2023-05-09 16:44:21,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +17: [2023-05-09 16:44:21,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +10: [2023-05-09 16:44:21,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +10: [2023-05-09 16:44:21,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 1: [2023-05-09 16:44:21,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +17: [2023-05-09 16:44:21,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 5: [2023-05-09 16:44:21,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +21: [2023-05-09 16:44:21,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +28: [2023-05-09 16:44:21,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +30: [2023-05-09 16:44:21,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +25: [2023-05-09 16:44:21,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +17: [2023-05-09 16:44:21,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +26: [2023-05-09 16:44:21,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +27: [2023-05-09 16:44:21,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +24: [2023-05-09 16:44:21,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +24: [2023-05-09 16:44:21,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +30: [2023-05-09 16:44:21,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +27: [2023-05-09 16:44:21,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +17: [2023-05-09 16:44:21,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +17: [2023-05-09 16:44:21,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +28: [2023-05-09 16:44:21,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +30: [2023-05-09 16:44:21,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +30: [2023-05-09 16:44:21,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +15: [2023-05-09 16:44:21,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +17: [2023-05-09 16:44:21,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +17: [2023-05-09 16:44:21,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +17: [2023-05-09 16:44:21,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +17: [2023-05-09 16:44:21,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +15: [2023-05-09 16:44:21,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +17: [2023-05-09 16:44:21,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +26: [2023-05-09 16:44:21,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +15: [2023-05-09 16:44:21,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +15: [2023-05-09 16:44:21,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +17: [2023-05-09 16:44:21,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +28: [2023-05-09 16:44:21,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +24: [2023-05-09 16:44:21,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +18: [2023-05-09 16:44:21,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +18: [2023-05-09 16:44:21,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +18: [2023-05-09 16:44:21,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +18: [2023-05-09 16:44:21,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +18: [2023-05-09 16:44:21,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +18: [2023-05-09 16:44:21,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +18: [2023-05-09 16:44:21,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +18: [2023-05-09 16:44:21,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +30: [2023-05-09 16:44:21,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +30: [2023-05-09 16:44:21,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +21: [2023-05-09 16:44:21,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +21: [2023-05-09 16:44:21,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +21: [2023-05-09 16:44:21,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 4: [2023-05-09 16:44:21,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 4: [2023-05-09 16:44:21,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +16: [2023-05-09 16:44:21,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +16: [2023-05-09 16:44:21,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +16: [2023-05-09 16:44:21,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +16: [2023-05-09 16:44:21,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 4: [2023-05-09 16:44:21,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +17: [2023-05-09 16:44:21,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +24: [2023-05-09 16:44:21,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +15: [2023-05-09 16:44:21,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +15: [2023-05-09 16:44:21,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 3: [2023-05-09 16:44:21,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +15: [2023-05-09 16:44:21,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +21: [2023-05-09 16:44:21,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +15: [2023-05-09 16:44:21,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 3: [2023-05-09 16:44:21,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +15: [2023-05-09 16:44:21,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +15: [2023-05-09 16:44:21,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 3: [2023-05-09 16:44:21,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +22: [2023-05-09 16:44:21,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +22: [2023-05-09 16:44:21,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +22: [2023-05-09 16:44:21,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 4: [2023-05-09 16:44:21,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +22: [2023-05-09 16:44:21,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 8: [2023-05-09 16:44:21,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +18: [2023-05-09 16:44:21,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +18: [2023-05-09 16:44:21,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +16: [2023-05-09 16:44:21,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +16: [2023-05-09 16:44:21,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +16: [2023-05-09 16:44:21,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +21: [2023-05-09 16:44:21,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +21: [2023-05-09 16:44:21,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +16: [2023-05-09 16:44:21,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +18: [2023-05-09 16:44:21,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +21: [2023-05-09 16:44:21,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +18: [2023-05-09 16:44:21,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +21: [2023-05-09 16:44:21,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +22: [2023-05-09 16:44:21,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +22: [2023-05-09 16:44:21,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +22: [2023-05-09 16:44:21,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +22: [2023-05-09 16:44:21,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +15: [2023-05-09 16:44:21,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 3: [2023-05-09 16:44:21,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 3: [2023-05-09 16:44:21,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +15: [2023-05-09 16:44:21,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 3: [2023-05-09 16:44:21,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 1: [2023-05-09 16:44:21,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 0: [2023-05-09 16:44:21,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +18: [2023-05-09 16:44:21,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 0: [2023-05-09 16:44:21,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +31: [2023-05-09 16:44:21,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 8: [2023-05-09 16:44:21,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +16: [2023-05-09 16:44:21,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +31: [2023-05-09 16:44:21,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +31: [2023-05-09 16:44:21,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +16: [2023-05-09 16:44:21,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 1: [2023-05-09 16:44:21,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +31: [2023-05-09 16:44:21,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +16: [2023-05-09 16:44:21,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +16: [2023-05-09 16:44:21,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +16: [2023-05-09 16:44:21,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +16: [2023-05-09 16:44:21,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +16: [2023-05-09 16:44:21,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +16: [2023-05-09 16:44:21,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +16: [2023-05-09 16:44:21,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +16: [2023-05-09 16:44:21,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +28: [2023-05-09 16:44:21,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +28: [2023-05-09 16:44:21,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +28: [2023-05-09 16:44:21,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +18: [2023-05-09 16:44:21,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +28: [2023-05-09 16:44:21,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +21: [2023-05-09 16:44:21,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +18: [2023-05-09 16:44:21,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +16: [2023-05-09 16:44:21,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +16: [2023-05-09 16:44:21,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +21: [2023-05-09 16:44:21,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +25: [2023-05-09 16:44:21,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +25: [2023-05-09 16:44:21,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +25: [2023-05-09 16:44:21,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 3: [2023-05-09 16:44:21,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +21: [2023-05-09 16:44:21,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 8: [2023-05-09 16:44:21,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 0: [2023-05-09 16:44:21,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 0: [2023-05-09 16:44:21,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +25: [2023-05-09 16:44:21,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 0: [2023-05-09 16:44:21,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 1: [2023-05-09 16:44:21,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +31: [2023-05-09 16:44:21,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +21: [2023-05-09 16:44:21,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 6: [2023-05-09 16:44:21,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 6: [2023-05-09 16:44:21,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. + 8: [2023-05-09 16:44:21,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +31: [2023-05-09 16:44:21,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +18: [2023-05-09 16:44:21,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +31: [2023-05-09 16:44:21,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +20: [2023-05-09 16:44:21,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +20: [2023-05-09 16:44:21,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +28: [2023-05-09 16:44:21,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +28: [2023-05-09 16:44:21,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +22: [2023-05-09 16:44:21,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +22: [2023-05-09 16:44:21,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +28: [2023-05-09 16:44:21,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +31: [2023-05-09 16:44:21,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +22: [2023-05-09 16:44:21,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +22: [2023-05-09 16:44:21,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +21: [2023-05-09 16:44:21,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +22: [2023-05-09 16:44:21,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +28: [2023-05-09 16:44:21,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +22: [2023-05-09 16:44:21,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 8: [2023-05-09 16:44:21,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +22: [2023-05-09 16:44:21,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 6: [2023-05-09 16:44:21,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +22: [2023-05-09 16:44:21,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +16: [2023-05-09 16:44:21,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +16: [2023-05-09 16:44:21,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +16: [2023-05-09 16:44:21,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 0: [2023-05-09 16:44:21,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +20: [2023-05-09 16:44:21,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +21: [2023-05-09 16:44:21,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +25: [2023-05-09 16:44:21,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +21: [2023-05-09 16:44:21,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +20: [2023-05-09 16:44:21,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt. +25: [2023-05-09 16:44:21,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +25: [2023-05-09 16:44:21,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +25: [2023-05-09 16:44:21,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +16: [2023-05-09 16:44:21,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +21: [2023-05-09 16:44:21,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +16: [2023-05-09 16:44:21,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +16: [2023-05-09 16:44:21,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 6: [2023-05-09 16:44:21,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +20: [2023-05-09 16:44:21,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +20: [2023-05-09 16:44:21,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +31: [2023-05-09 16:44:21,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +31: [2023-05-09 16:44:21,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +16: [2023-05-09 16:44:21,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +28: [2023-05-09 16:44:21,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +28: [2023-05-09 16:44:21,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 6: [2023-05-09 16:44:21,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +20: [2023-05-09 16:44:21,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 3: [2023-05-09 16:44:21,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +20: [2023-05-09 16:44:21,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt... +25: [2023-05-09 16:44:21,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +16: [2023-05-09 16:44:21,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +25: [2023-05-09 16:44:21,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +31: [2023-05-09 16:44:21,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +31: [2023-05-09 16:44:21,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +31: [2023-05-09 16:44:21,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +31: [2023-05-09 16:44:21,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +21: [2023-05-09 16:44:21,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +21: [2023-05-09 16:44:21,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 3: [2023-05-09 16:44:21,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 8: [2023-05-09 16:44:21,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +21: [2023-05-09 16:44:21,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +21: [2023-05-09 16:44:21,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +28: [2023-05-09 16:44:21,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +28: [2023-05-09 16:44:21,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +28: [2023-05-09 16:44:21,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +28: [2023-05-09 16:44:21,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 8: [2023-05-09 16:44:21,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 8: [2023-05-09 16:44:21,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 0: [2023-05-09 16:44:21,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +16: [2023-05-09 16:44:21,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +31: [2023-05-09 16:44:21,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +31: [2023-05-09 16:44:21,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +31: [2023-05-09 16:44:21,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +31: [2023-05-09 16:44:21,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +31: [2023-05-09 16:44:21,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +31: [2023-05-09 16:44:21,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +31: [2023-05-09 16:44:21,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +31: [2023-05-09 16:44:21,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +16: [2023-05-09 16:44:21,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +21: [2023-05-09 16:44:21,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +21: [2023-05-09 16:44:21,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +21: [2023-05-09 16:44:21,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +28: [2023-05-09 16:44:21,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +28: [2023-05-09 16:44:21,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +21: [2023-05-09 16:44:21,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +25: [2023-05-09 16:44:21,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +25: [2023-05-09 16:44:21,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +25: [2023-05-09 16:44:21,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +25: [2023-05-09 16:44:21,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 6: [2023-05-09 16:44:21,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 6: [2023-05-09 16:44:21,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +25: [2023-05-09 16:44:21,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 6: [2023-05-09 16:44:21,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 6: [2023-05-09 16:44:21,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +20: [2023-05-09 16:44:21,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +20: [2023-05-09 16:44:21,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +20: [2023-05-09 16:44:21,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 8: [2023-05-09 16:44:21,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +20: [2023-05-09 16:44:21,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +25: [2023-05-09 16:44:21,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 3: [2023-05-09 16:44:21,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +31: [2023-05-09 16:44:21,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +16: [2023-05-09 16:44:21,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +31: [2023-05-09 16:44:21,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +20: [2023-05-09 16:44:21,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +20: [2023-05-09 16:44:21,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 6: [2023-05-09 16:44:21,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +20: [2023-05-09 16:44:21,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +20: [2023-05-09 16:44:21,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 8: [2023-05-09 16:44:21,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. +16: [2023-05-09 16:44:21,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 0: [2023-05-09 16:44:21,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 0: [2023-05-09 16:44:21,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 6: [2023-05-09 16:44:21,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 8: [2023-05-09 16:44:21,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... + 6: [2023-05-09 16:44:21,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +31: [2023-05-09 16:44:21,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt... +31: [2023-05-09 16:44:21,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +31: [2023-05-09 16:44:21,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +31: [2023-05-09 16:44:21,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +31: [2023-05-09 16:44:21,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +31: [2023-05-09 16:44:21,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +31: [2023-05-09 16:44:21,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +31: [2023-05-09 16:44:21,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +29: [2023-05-09 16:44:21,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +29: [2023-05-09 16:44:21,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +29: [2023-05-09 16:44:21,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +29: [2023-05-09 16:44:21,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +29: [2023-05-09 16:44:21,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +29: [2023-05-09 16:44:21,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +29: [2023-05-09 16:44:21,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +29: [2023-05-09 16:44:21,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +22: [2023-05-09 16:44:21,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +22: [2023-05-09 16:44:21,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +22: [2023-05-09 16:44:21,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +22: [2023-05-09 16:44:21,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +22: [2023-05-09 16:44:21,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +22: [2023-05-09 16:44:21,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +22: [2023-05-09 16:44:21,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +22: [2023-05-09 16:44:21,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +20: [2023-05-09 16:44:21,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +20: [2023-05-09 16:44:21,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +20: [2023-05-09 16:44:21,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +14: [2023-05-09 16:44:21,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +20: [2023-05-09 16:44:21,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +20: [2023-05-09 16:44:21,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +20: [2023-05-09 16:44:21,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +14: [2023-05-09 16:44:21,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +20: [2023-05-09 16:44:21,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +14: [2023-05-09 16:44:21,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +20: [2023-05-09 16:44:21,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +14: [2023-05-09 16:44:21,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +14: [2023-05-09 16:44:21,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +14: [2023-05-09 16:44:21,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +14: [2023-05-09 16:44:21,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +14: [2023-05-09 16:44:21,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +22: [2023-05-09 16:44:21,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +29: [2023-05-09 16:44:21,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +29: [2023-05-09 16:44:21,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +22: [2023-05-09 16:44:21,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +20: [2023-05-09 16:44:21,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +29: [2023-05-09 16:44:21,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +29: [2023-05-09 16:44:21,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +20: [2023-05-09 16:44:21,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +22: [2023-05-09 16:44:21,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +22: [2023-05-09 16:44:21,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +20: [2023-05-09 16:44:21,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +20: [2023-05-09 16:44:21,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +14: [2023-05-09 16:44:21,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +14: [2023-05-09 16:44:21,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +14: [2023-05-09 16:44:21,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +14: [2023-05-09 16:44:21,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +29: [2023-05-09 16:44:21,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +29: [2023-05-09 16:44:21,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +22: [2023-05-09 16:44:21,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +22: [2023-05-09 16:44:21,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +20: [2023-05-09 16:44:21,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +20: [2023-05-09 16:44:21,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +20: [2023-05-09 16:44:21,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +29: [2023-05-09 16:44:21,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +29: [2023-05-09 16:44:21,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +22: [2023-05-09 16:44:21,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +20: [2023-05-09 16:44:21,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +14: [2023-05-09 16:44:21,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +22: [2023-05-09 16:44:21,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +14: [2023-05-09 16:44:21,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +14: [2023-05-09 16:44:21,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +14: [2023-05-09 16:44:21,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +13: [2023-05-09 16:44:21,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +13: [2023-05-09 16:44:21,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +13: [2023-05-09 16:44:21,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +13: [2023-05-09 16:44:21,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 9: [2023-05-09 16:44:21,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +13: [2023-05-09 16:44:21,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +13: [2023-05-09 16:44:21,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +13: [2023-05-09 16:44:21,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +13: [2023-05-09 16:44:21,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +10: [2023-05-09 16:44:21,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +10: [2023-05-09 16:44:21,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +10: [2023-05-09 16:44:21,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +10: [2023-05-09 16:44:21,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +19: [2023-05-09 16:44:21,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +10: [2023-05-09 16:44:21,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +19: [2023-05-09 16:44:21,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +19: [2023-05-09 16:44:21,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +10: [2023-05-09 16:44:21,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +10: [2023-05-09 16:44:21,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +19: [2023-05-09 16:44:21,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +19: [2023-05-09 16:44:21,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +10: [2023-05-09 16:44:21,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +19: [2023-05-09 16:44:21,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +19: [2023-05-09 16:44:21,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +19: [2023-05-09 16:44:21,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +12: [2023-05-09 16:44:21,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +27: [2023-05-09 16:44:21,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +12: [2023-05-09 16:44:21,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +27: [2023-05-09 16:44:21,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +12: [2023-05-09 16:44:21,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +12: [2023-05-09 16:44:21,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +27: [2023-05-09 16:44:21,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +12: [2023-05-09 16:44:21,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +12: [2023-05-09 16:44:21,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +27: [2023-05-09 16:44:21,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +27: [2023-05-09 16:44:21,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +27: [2023-05-09 16:44:21,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +27: [2023-05-09 16:44:21,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +12: [2023-05-09 16:44:21,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +12: [2023-05-09 16:44:21,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +27: [2023-05-09 16:44:21,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +24: [2023-05-09 16:44:21,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +24: [2023-05-09 16:44:21,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +24: [2023-05-09 16:44:21,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +24: [2023-05-09 16:44:21,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +13: [2023-05-09 16:44:21,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +13: [2023-05-09 16:44:21,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +18: [2023-05-09 16:44:21,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +10: [2023-05-09 16:44:21,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +18: [2023-05-09 16:44:21,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +18: [2023-05-09 16:44:21,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +18: [2023-05-09 16:44:21,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +13: [2023-05-09 16:44:21,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +13: [2023-05-09 16:44:21,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +19: [2023-05-09 16:44:21,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +13: [2023-05-09 16:44:21,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +13: [2023-05-09 16:44:21,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +13: [2023-05-09 16:44:21,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +13: [2023-05-09 16:44:21,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 2: [2023-05-09 16:44:21,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +10: [2023-05-09 16:44:21,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +24: [2023-05-09 16:44:21,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +10: [2023-05-09 16:44:21,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +10: [2023-05-09 16:44:21,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +27: [2023-05-09 16:44:21,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +24: [2023-05-09 16:44:21,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +24: [2023-05-09 16:44:21,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +19: [2023-05-09 16:44:21,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +19: [2023-05-09 16:44:21,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +27: [2023-05-09 16:44:21,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +10: [2023-05-09 16:44:21,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +24: [2023-05-09 16:44:21,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +27: [2023-05-09 16:44:21,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +18: [2023-05-09 16:44:21,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +18: [2023-05-09 16:44:21,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +19: [2023-05-09 16:44:21,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +18: [2023-05-09 16:44:21,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +18: [2023-05-09 16:44:21,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +12: [2023-05-09 16:44:21,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +12: [2023-05-09 16:44:21,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +10: [2023-05-09 16:44:21,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +27: [2023-05-09 16:44:21,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +27: [2023-05-09 16:44:21,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +19: [2023-05-09 16:44:21,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +19: [2023-05-09 16:44:21,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +10: [2023-05-09 16:44:21,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 9: [2023-05-09 16:44:21,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +10: [2023-05-09 16:44:21,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +27: [2023-05-09 16:44:21,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 1: [2023-05-09 16:44:21,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +27: [2023-05-09 16:44:21,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 1: [2023-05-09 16:44:21,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +24: [2023-05-09 16:44:21,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +24: [2023-05-09 16:44:21,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 7: [2023-05-09 16:44:21,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +24: [2023-05-09 16:44:21,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +24: [2023-05-09 16:44:21,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 1: [2023-05-09 16:44:21,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 5: [2023-05-09 16:44:21,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 9: [2023-05-09 16:44:21,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +12: [2023-05-09 16:44:21,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +19: [2023-05-09 16:44:21,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +12: [2023-05-09 16:44:21,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +18: [2023-05-09 16:44:21,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +14: [2023-05-09 16:44:21,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +14: [2023-05-09 16:44:21,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +14: [2023-05-09 16:44:21,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +14: [2023-05-09 16:44:21,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 2: [2023-05-09 16:44:21,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +18: [2023-05-09 16:44:21,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +18: [2023-05-09 16:44:21,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +18: [2023-05-09 16:44:21,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 7: [2023-05-09 16:44:21,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +12: [2023-05-09 16:44:21,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +24: [2023-05-09 16:44:21,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +24: [2023-05-09 16:44:21,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 5: [2023-05-09 16:44:21,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +12: [2023-05-09 16:44:21,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +27: [2023-05-09 16:44:21,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +19: [2023-05-09 16:44:21,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +12: [2023-05-09 16:44:21,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +14: [2023-05-09 16:44:21,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +14: [2023-05-09 16:44:21,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +24: [2023-05-09 16:44:21,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +14: [2023-05-09 16:44:21,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +11: [2023-05-09 16:44:21,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +11: [2023-05-09 16:44:21,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +11: [2023-05-09 16:44:21,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +11: [2023-05-09 16:44:21,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +11: [2023-05-09 16:44:21,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +11: [2023-05-09 16:44:21,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +11: [2023-05-09 16:44:21,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +11: [2023-05-09 16:44:21,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +24: [2023-05-09 16:44:21,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +14: [2023-05-09 16:44:21,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +13: [2023-05-09 16:44:21,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +18: [2023-05-09 16:44:21,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +13: [2023-05-09 16:44:21,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +13: [2023-05-09 16:44:21,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +13: [2023-05-09 16:44:21,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +13: [2023-05-09 16:44:21,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +13: [2023-05-09 16:44:21,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +13: [2023-05-09 16:44:21,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +18: [2023-05-09 16:44:21,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +12: [2023-05-09 16:44:21,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +18: [2023-05-09 16:44:21,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +18: [2023-05-09 16:44:21,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +13: [2023-05-09 16:44:21,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +10: [2023-05-09 16:44:21,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +10: [2023-05-09 16:44:21,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +10: [2023-05-09 16:44:21,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 9: [2023-05-09 16:44:21,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +10: [2023-05-09 16:44:21,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 9: [2023-05-09 16:44:21,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 1: [2023-05-09 16:44:21,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +23: [2023-05-09 16:44:21,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +23: [2023-05-09 16:44:21,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +23: [2023-05-09 16:44:21,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +14: [2023-05-09 16:44:21,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +23: [2023-05-09 16:44:21,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +23: [2023-05-09 16:44:21,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +14: [2023-05-09 16:44:21,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +23: [2023-05-09 16:44:21,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +23: [2023-05-09 16:44:21,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +23: [2023-05-09 16:44:21,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 5: [2023-05-09 16:44:21,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +13: [2023-05-09 16:44:21,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 1: [2023-05-09 16:44:21,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 9: [2023-05-09 16:44:21,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 1: [2023-05-09 16:44:21,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 1: [2023-05-09 16:44:21,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +10: [2023-05-09 16:44:21,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +10: [2023-05-09 16:44:21,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +14: [2023-05-09 16:44:21,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +14: [2023-05-09 16:44:21,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +10: [2023-05-09 16:44:21,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +13: [2023-05-09 16:44:21,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +10: [2023-05-09 16:44:21,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +11: [2023-05-09 16:44:21,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +11: [2023-05-09 16:44:21,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +11: [2023-05-09 16:44:21,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +11: [2023-05-09 16:44:21,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +23: [2023-05-09 16:44:21,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +23: [2023-05-09 16:44:21,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +22: [2023-05-09 16:44:21,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +22: [2023-05-09 16:44:21,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +13: [2023-05-09 16:44:21,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +22: [2023-05-09 16:44:21,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 9: [2023-05-09 16:44:21,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +22: [2023-05-09 16:44:21,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +14: [2023-05-09 16:44:21,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +14: [2023-05-09 16:44:21,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +28: [2023-05-09 16:44:21,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +28: [2023-05-09 16:44:21,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +28: [2023-05-09 16:44:21,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +28: [2023-05-09 16:44:21,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +28: [2023-05-09 16:44:21,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +28: [2023-05-09 16:44:21,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +28: [2023-05-09 16:44:21,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +28: [2023-05-09 16:44:21,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +13: [2023-05-09 16:44:21,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +13: [2023-05-09 16:44:21,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +14: [2023-05-09 16:44:21,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +14: [2023-05-09 16:44:21,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +10: [2023-05-09 16:44:21,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +10: [2023-05-09 16:44:21,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +11: [2023-05-09 16:44:21,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +11: [2023-05-09 16:44:21,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +23: [2023-05-09 16:44:21,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +11: [2023-05-09 16:44:21,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +11: [2023-05-09 16:44:21,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +23: [2023-05-09 16:44:21,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +22: [2023-05-09 16:44:21,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +22: [2023-05-09 16:44:21,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +22: [2023-05-09 16:44:21,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +22: [2023-05-09 16:44:21,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +13: [2023-05-09 16:44:21,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +23: [2023-05-09 16:44:21,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +28: [2023-05-09 16:44:21,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +26: [2023-05-09 16:44:21,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +26: [2023-05-09 16:44:21,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +26: [2023-05-09 16:44:21,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +26: [2023-05-09 16:44:21,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +26: [2023-05-09 16:44:21,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +26: [2023-05-09 16:44:21,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +26: [2023-05-09 16:44:21,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +26: [2023-05-09 16:44:21,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +13: [2023-05-09 16:44:21,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +23: [2023-05-09 16:44:21,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +10: [2023-05-09 16:44:21,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +10: [2023-05-09 16:44:21,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +10: [2023-05-09 16:44:21,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +10: [2023-05-09 16:44:21,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +28: [2023-05-09 16:44:21,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +29: [2023-05-09 16:44:21,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +29: [2023-05-09 16:44:21,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +29: [2023-05-09 16:44:21,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +29: [2023-05-09 16:44:21,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +28: [2023-05-09 16:44:21,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +28: [2023-05-09 16:44:21,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +17: [2023-05-09 16:44:21,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +17: [2023-05-09 16:44:21,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +17: [2023-05-09 16:44:21,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +17: [2023-05-09 16:44:21,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +23: [2023-05-09 16:44:21,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +10: [2023-05-09 16:44:21,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +22: [2023-05-09 16:44:21,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +28: [2023-05-09 16:44:21,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +10: [2023-05-09 16:44:21,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +13: [2023-05-09 16:44:21,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +23: [2023-05-09 16:44:21,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +22: [2023-05-09 16:44:21,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 4: [2023-05-09 16:44:21,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +11: [2023-05-09 16:44:21,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +11: [2023-05-09 16:44:21,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +11: [2023-05-09 16:44:21,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +19: [2023-05-09 16:44:21,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +19: [2023-05-09 16:44:21,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 4: [2023-05-09 16:44:21,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +19: [2023-05-09 16:44:21,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +11: [2023-05-09 16:44:21,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 4: [2023-05-09 16:44:21,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 4: [2023-05-09 16:44:21,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +15: [2023-05-09 16:44:21,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +15: [2023-05-09 16:44:21,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 4: [2023-05-09 16:44:21,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +12: [2023-05-09 16:44:21,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +12: [2023-05-09 16:44:21,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +12: [2023-05-09 16:44:21,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +12: [2023-05-09 16:44:21,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +19: [2023-05-09 16:44:21,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +15: [2023-05-09 16:44:21,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 7: [2023-05-09 16:44:21,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 7: [2023-05-09 16:44:21,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 7: [2023-05-09 16:44:21,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 7: [2023-05-09 16:44:21,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +30: [2023-05-09 16:44:21,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +15: [2023-05-09 16:44:21,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +30: [2023-05-09 16:44:21,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +30: [2023-05-09 16:44:21,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +30: [2023-05-09 16:44:21,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +15: [2023-05-09 16:44:21,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +15: [2023-05-09 16:44:21,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +15: [2023-05-09 16:44:21,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +15: [2023-05-09 16:44:21,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +15: [2023-05-09 16:44:21,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +15: [2023-05-09 16:44:21,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +26: [2023-05-09 16:44:21,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +26: [2023-05-09 16:44:21,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +28: [2023-05-09 16:44:21,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +15: [2023-05-09 16:44:21,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +28: [2023-05-09 16:44:21,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +15: [2023-05-09 16:44:21,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +26: [2023-05-09 16:44:21,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +29: [2023-05-09 16:44:21,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +28: [2023-05-09 16:44:21,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +29: [2023-05-09 16:44:21,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +29: [2023-05-09 16:44:21,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +29: [2023-05-09 16:44:21,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +17: [2023-05-09 16:44:21,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +17: [2023-05-09 16:44:21,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 2: [2023-05-09 16:44:21,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 4: [2023-05-09 16:44:21,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +17: [2023-05-09 16:44:21,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +17: [2023-05-09 16:44:21,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +11: [2023-05-09 16:44:21,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 8: [2023-05-09 16:44:21,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +11: [2023-05-09 16:44:21,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +26: [2023-05-09 16:44:21,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +22: [2023-05-09 16:44:21,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +11: [2023-05-09 16:44:21,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +19: [2023-05-09 16:44:21,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +12: [2023-05-09 16:44:21,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +19: [2023-05-09 16:44:21,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +22: [2023-05-09 16:44:21,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +11: [2023-05-09 16:44:21,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +19: [2023-05-09 16:44:21,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +30: [2023-05-09 16:44:21,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +30: [2023-05-09 16:44:21,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +26: [2023-05-09 16:44:21,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +22: [2023-05-09 16:44:21,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +30: [2023-05-09 16:44:21,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +30: [2023-05-09 16:44:21,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 2: [2023-05-09 16:44:21,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +26: [2023-05-09 16:44:21,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +19: [2023-05-09 16:44:21,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +12: [2023-05-09 16:44:21,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +22: [2023-05-09 16:44:21,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +22: [2023-05-09 16:44:21,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +12: [2023-05-09 16:44:21,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +12: [2023-05-09 16:44:21,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +26: [2023-05-09 16:44:21,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +22: [2023-05-09 16:44:21,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 4: [2023-05-09 16:44:21,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +15: [2023-05-09 16:44:21,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +15: [2023-05-09 16:44:21,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +26: [2023-05-09 16:44:21,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +15: [2023-05-09 16:44:21,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +17: [2023-05-09 16:44:21,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +11: [2023-05-09 16:44:21,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +17: [2023-05-09 16:44:21,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +15: [2023-05-09 16:44:21,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +11: [2023-05-09 16:44:21,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +11: [2023-05-09 16:44:21,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +11: [2023-05-09 16:44:21,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 2: [2023-05-09 16:44:21,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +15: [2023-05-09 16:44:21,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +15: [2023-05-09 16:44:21,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +29: [2023-05-09 16:44:21,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +15: [2023-05-09 16:44:21,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +12: [2023-05-09 16:44:21,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +29: [2023-05-09 16:44:21,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +12: [2023-05-09 16:44:21,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +15: [2023-05-09 16:44:21,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +24: [2023-05-09 16:44:21,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +24: [2023-05-09 16:44:21,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +19: [2023-05-09 16:44:21,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +24: [2023-05-09 16:44:21,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +24: [2023-05-09 16:44:21,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +19: [2023-05-09 16:44:21,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +17: [2023-05-09 16:44:21,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +17: [2023-05-09 16:44:21,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +11: [2023-05-09 16:44:21,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +11: [2023-05-09 16:44:21,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 5: [2023-05-09 16:44:21,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +30: [2023-05-09 16:44:21,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +30: [2023-05-09 16:44:21,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +30: [2023-05-09 16:44:21,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 5: [2023-05-09 16:44:21,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +30: [2023-05-09 16:44:21,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +17: [2023-05-09 16:44:21,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +24: [2023-05-09 16:44:21,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +17: [2023-05-09 16:44:21,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +19: [2023-05-09 16:44:21,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +24: [2023-05-09 16:44:21,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +19: [2023-05-09 16:44:21,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +24: [2023-05-09 16:44:21,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +30: [2023-05-09 16:44:21,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +30: [2023-05-09 16:44:21,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +24: [2023-05-09 16:44:21,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 7: [2023-05-09 16:44:21,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 7: [2023-05-09 16:44:21,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 7: [2023-05-09 16:44:21,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +30: [2023-05-09 16:44:21,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +30: [2023-05-09 16:44:21,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +30: [2023-05-09 16:44:21,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +30: [2023-05-09 16:44:21,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +30: [2023-05-09 16:44:21,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +30: [2023-05-09 16:44:21,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +30: [2023-05-09 16:44:21,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +29: [2023-05-09 16:44:21,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +30: [2023-05-09 16:44:21,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +15: [2023-05-09 16:44:21,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +29: [2023-05-09 16:44:21,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 8: [2023-05-09 16:44:21,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +12: [2023-05-09 16:44:21,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +12: [2023-05-09 16:44:21,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +18: [2023-05-09 16:44:21,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +18: [2023-05-09 16:44:21,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +18: [2023-05-09 16:44:21,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +18: [2023-05-09 16:44:21,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +18: [2023-05-09 16:44:21,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +18: [2023-05-09 16:44:21,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +27: [2023-05-09 16:44:21,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +27: [2023-05-09 16:44:21,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +11: [2023-05-09 16:44:21,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +11: [2023-05-09 16:44:21,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +16: [2023-05-09 16:44:21,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +16: [2023-05-09 16:44:21,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +23: [2023-05-09 16:44:21,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +18: [2023-05-09 16:44:21,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +23: [2023-05-09 16:44:21,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +18: [2023-05-09 16:44:21,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +23: [2023-05-09 16:44:21,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +23: [2023-05-09 16:44:21,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +26: [2023-05-09 16:44:21,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +26: [2023-05-09 16:44:21,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 2: [2023-05-09 16:44:21,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +26: [2023-05-09 16:44:21,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +30: [2023-05-09 16:44:21,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +30: [2023-05-09 16:44:21,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +29: [2023-05-09 16:44:21,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +29: [2023-05-09 16:44:21,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +26: [2023-05-09 16:44:21,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +16: [2023-05-09 16:44:21,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +19: [2023-05-09 16:44:21,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +27: [2023-05-09 16:44:21,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +16: [2023-05-09 16:44:21,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +27: [2023-05-09 16:44:21,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 4: [2023-05-09 16:44:21,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +19: [2023-05-09 16:44:21,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 2: [2023-05-09 16:44:21,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +29: [2023-05-09 16:44:21,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +29: [2023-05-09 16:44:21,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 2: [2023-05-09 16:44:21,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +17: [2023-05-09 16:44:21,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +17: [2023-05-09 16:44:21,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +15: [2023-05-09 16:44:21,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +17: [2023-05-09 16:44:21,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +17: [2023-05-09 16:44:21,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +31: [2023-05-09 16:44:21,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +17: [2023-05-09 16:44:21,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +17: [2023-05-09 16:44:21,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +12: [2023-05-09 16:44:21,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +31: [2023-05-09 16:44:21,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +31: [2023-05-09 16:44:21,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +15: [2023-05-09 16:44:21,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +28: [2023-05-09 16:44:21,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +31: [2023-05-09 16:44:21,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 2: [2023-05-09 16:44:21,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 3: [2023-05-09 16:44:21,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 3: [2023-05-09 16:44:21,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +28: [2023-05-09 16:44:21,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +28: [2023-05-09 16:44:21,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 3: [2023-05-09 16:44:21,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +28: [2023-05-09 16:44:21,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 8: [2023-05-09 16:44:21,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +19: [2023-05-09 16:44:21,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +25: [2023-05-09 16:44:21,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 2: [2023-05-09 16:44:21,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +12: [2023-05-09 16:44:21,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 8: [2023-05-09 16:44:21,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +25: [2023-05-09 16:44:21,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +25: [2023-05-09 16:44:21,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +19: [2023-05-09 16:44:21,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 8: [2023-05-09 16:44:21,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +25: [2023-05-09 16:44:21,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 8: [2023-05-09 16:44:21,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +15: [2023-05-09 16:44:21,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +15: [2023-05-09 16:44:21,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +17: [2023-05-09 16:44:21,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +30: [2023-05-09 16:44:21,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +12: [2023-05-09 16:44:21,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +27: [2023-05-09 16:44:21,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +27: [2023-05-09 16:44:21,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +20: [2023-05-09 16:44:21,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +20: [2023-05-09 16:44:21,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +20: [2023-05-09 16:44:21,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +20: [2023-05-09 16:44:21,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +17: [2023-05-09 16:44:21,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +26: [2023-05-09 16:44:21,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +16: [2023-05-09 16:44:21,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +17: [2023-05-09 16:44:21,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +21: [2023-05-09 16:44:21,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +21: [2023-05-09 16:44:21,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +16: [2023-05-09 16:44:21,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +26: [2023-05-09 16:44:21,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +26: [2023-05-09 16:44:21,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +12: [2023-05-09 16:44:21,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +27: [2023-05-09 16:44:21,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +24: [2023-05-09 16:44:21,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +26: [2023-05-09 16:44:21,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +15: [2023-05-09 16:44:21,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +27: [2023-05-09 16:44:21,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +15: [2023-05-09 16:44:21,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +30: [2023-05-09 16:44:21,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +30: [2023-05-09 16:44:21,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +15: [2023-05-09 16:44:21,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +24: [2023-05-09 16:44:21,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +21: [2023-05-09 16:44:21,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +23: [2023-05-09 16:44:21,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +23: [2023-05-09 16:44:21,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +23: [2023-05-09 16:44:21,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +21: [2023-05-09 16:44:21,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +16: [2023-05-09 16:44:21,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +30: [2023-05-09 16:44:21,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +16: [2023-05-09 16:44:21,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +18: [2023-05-09 16:44:21,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +24: [2023-05-09 16:44:21,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +17: [2023-05-09 16:44:21,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +15: [2023-05-09 16:44:21,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +15: [2023-05-09 16:44:21,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +31: [2023-05-09 16:44:21,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +21: [2023-05-09 16:44:21,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +21: [2023-05-09 16:44:21,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +31: [2023-05-09 16:44:21,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +23: [2023-05-09 16:44:21,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +28: [2023-05-09 16:44:21,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +21: [2023-05-09 16:44:21,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +21: [2023-05-09 16:44:21,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +21: [2023-05-09 16:44:21,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +21: [2023-05-09 16:44:21,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +31: [2023-05-09 16:44:21,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +21: [2023-05-09 16:44:21,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +28: [2023-05-09 16:44:21,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +28: [2023-05-09 16:44:21,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +25: [2023-05-09 16:44:21,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +18: [2023-05-09 16:44:21,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +17: [2023-05-09 16:44:21,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +25: [2023-05-09 16:44:21,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +31: [2023-05-09 16:44:21,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +25: [2023-05-09 16:44:21,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +21: [2023-05-09 16:44:21,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +28: [2023-05-09 16:44:21,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +25: [2023-05-09 16:44:21,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +17: [2023-05-09 16:44:21,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +24: [2023-05-09 16:44:21,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +20: [2023-05-09 16:44:21,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +21: [2023-05-09 16:44:21,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +20: [2023-05-09 16:44:21,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +17: [2023-05-09 16:44:21,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +15: [2023-05-09 16:44:21,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +21: [2023-05-09 16:44:21,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +24: [2023-05-09 16:44:21,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +20: [2023-05-09 16:44:21,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +15: [2023-05-09 16:44:21,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +20: [2023-05-09 16:44:21,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +24: [2023-05-09 16:44:21,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +24: [2023-05-09 16:44:21,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +16: [2023-05-09 16:44:21,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +16: [2023-05-09 16:44:21,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +16: [2023-05-09 16:44:21,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +18: [2023-05-09 16:44:21,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +26: [2023-05-09 16:44:21,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +26: [2023-05-09 16:44:21,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +16: [2023-05-09 16:44:21,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +30: [2023-05-09 16:44:21,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +18: [2023-05-09 16:44:21,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +30: [2023-05-09 16:44:21,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +30: [2023-05-09 16:44:21,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +27: [2023-05-09 16:44:21,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +18: [2023-05-09 16:44:21,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +27: [2023-05-09 16:44:21,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +21: [2023-05-09 16:44:21,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +21: [2023-05-09 16:44:21,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +16: [2023-05-09 16:44:21,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +27: [2023-05-09 16:44:21,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +16: [2023-05-09 16:44:21,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +27: [2023-05-09 16:44:21,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +17: [2023-05-09 16:44:21,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +16: [2023-05-09 16:44:21,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +30: [2023-05-09 16:44:21,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +23: [2023-05-09 16:44:21,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +23: [2023-05-09 16:44:21,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +23: [2023-05-09 16:44:21,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +23: [2023-05-09 16:44:21,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +16: [2023-05-09 16:44:21,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +21: [2023-05-09 16:44:21,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +16: [2023-05-09 16:44:21,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +16: [2023-05-09 16:44:21,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +16: [2023-05-09 16:44:21,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +16: [2023-05-09 16:44:21,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +17: [2023-05-09 16:44:21,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +24: [2023-05-09 16:44:21,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +18: [2023-05-09 16:44:21,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +26: [2023-05-09 16:44:21,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +26: [2023-05-09 16:44:21,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +26: [2023-05-09 16:44:21,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +26: [2023-05-09 16:44:21,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +26: [2023-05-09 16:44:21,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +26: [2023-05-09 16:44:21,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +17: [2023-05-09 16:44:21,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 6: [2023-05-09 16:44:21,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 6: [2023-05-09 16:44:21,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 6: [2023-05-09 16:44:21,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 6: [2023-05-09 16:44:21,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +18: [2023-05-09 16:44:21,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 0: [2023-05-09 16:44:21,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 0: [2023-05-09 16:44:21,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. + 0: [2023-05-09 16:44:21,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt. +27: [2023-05-09 16:44:21,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +27: [2023-05-09 16:44:21,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +16: [2023-05-09 16:44:21,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +16: [2023-05-09 16:44:21,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +27: [2023-05-09 16:44:21,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +27: [2023-05-09 16:44:21,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +28: [2023-05-09 16:44:21,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +28: [2023-05-09 16:44:21,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +28: [2023-05-09 16:44:21,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +25: [2023-05-09 16:44:21,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +28: [2023-05-09 16:44:21,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +28: [2023-05-09 16:44:21,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +28: [2023-05-09 16:44:21,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +20: [2023-05-09 16:44:21,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +23: [2023-05-09 16:44:21,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +25: [2023-05-09 16:44:21,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +20: [2023-05-09 16:44:21,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +16: [2023-05-09 16:44:21,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +16: [2023-05-09 16:44:21,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +23: [2023-05-09 16:44:21,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +21: [2023-05-09 16:44:21,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +21: [2023-05-09 16:44:21,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +17: [2023-05-09 16:44:21,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +31: [2023-05-09 16:44:21,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +20: [2023-05-09 16:44:21,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 8: [2023-05-09 16:44:21,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +18: [2023-05-09 16:44:21,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +21: [2023-05-09 16:44:21,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 8: [2023-05-09 16:44:21,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +20: [2023-05-09 16:44:21,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +31: [2023-05-09 16:44:21,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +31: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +21: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +21: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +28: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +31: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +31: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +31: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +31: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +29: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +29: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +29: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +31: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +29: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +29: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +29: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +31: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +29: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +29: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +14: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +14: [2023-05-09 16:44:21,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +23: [2023-05-09 16:44:21,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +22: [2023-05-09 16:44:21,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +28: [2023-05-09 16:44:21,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +22: [2023-05-09 16:44:21,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +22: [2023-05-09 16:44:21,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 6: [2023-05-09 16:44:21,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 6: [2023-05-09 16:44:21,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 6: [2023-05-09 16:44:21,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +22: [2023-05-09 16:44:21,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +31: [2023-05-09 16:44:21,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +25: [2023-05-09 16:44:21,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +22: [2023-05-09 16:44:21,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +25: [2023-05-09 16:44:21,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 6: [2023-05-09 16:44:21,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +25: [2023-05-09 16:44:21,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +23: [2023-05-09 16:44:21,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 6: [2023-05-09 16:44:21,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +22: [2023-05-09 16:44:21,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +22: [2023-05-09 16:44:21,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +22: [2023-05-09 16:44:21,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +25: [2023-05-09 16:44:21,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +25: [2023-05-09 16:44:21,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +25: [2023-05-09 16:44:21,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +25: [2023-05-09 16:44:21,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +25: [2023-05-09 16:44:21,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +25: [2023-05-09 16:44:21,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +25: [2023-05-09 16:44:21,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +25: [2023-05-09 16:44:21,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +14: [2023-05-09 16:44:21,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +31: [2023-05-09 16:44:21,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +31: [2023-05-09 16:44:21,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +20: [2023-05-09 16:44:21,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +25: [2023-05-09 16:44:21,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +20: [2023-05-09 16:44:21,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +20: [2023-05-09 16:44:21,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +31: [2023-05-09 16:44:21,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +20: [2023-05-09 16:44:21,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +14: [2023-05-09 16:44:21,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +14: [2023-05-09 16:44:21,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +21: [2023-05-09 16:44:21,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +21: [2023-05-09 16:44:21,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +14: [2023-05-09 16:44:21,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +25: [2023-05-09 16:44:21,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 6: [2023-05-09 16:44:21,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +25: [2023-05-09 16:44:21,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +20: [2023-05-09 16:44:21,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 6: [2023-05-09 16:44:21,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +16: [2023-05-09 16:44:21,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +20: [2023-05-09 16:44:21,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +31: [2023-05-09 16:44:21,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +16: [2023-05-09 16:44:21,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +20: [2023-05-09 16:44:21,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +20: [2023-05-09 16:44:21,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +14: [2023-05-09 16:44:21,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt... +16: [2023-05-09 16:44:21,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +14: [2023-05-09 16:44:21,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +16: [2023-05-09 16:44:21,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +17: [2023-05-09 16:44:21,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +20: [2023-05-09 16:44:21,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +20: [2023-05-09 16:44:21,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 0: [2023-05-09 16:44:21,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +16: [2023-05-09 16:44:21,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +31: [2023-05-09 16:44:21,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +20: [2023-05-09 16:44:21,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +16: [2023-05-09 16:44:21,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +31: [2023-05-09 16:44:21,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 8: [2023-05-09 16:44:21,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +14: [2023-05-09 16:44:21,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +31: [2023-05-09 16:44:21,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +20: [2023-05-09 16:44:21,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +29: [2023-05-09 16:44:21,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +29: [2023-05-09 16:44:21,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +31: [2023-05-09 16:44:21,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +31: [2023-05-09 16:44:21,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 9: [2023-05-09 16:44:21,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +29: [2023-05-09 16:44:21,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +29: [2023-05-09 16:44:21,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +16: [2023-05-09 16:44:21,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +16: [2023-05-09 16:44:21,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +20: [2023-05-09 16:44:21,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +21: [2023-05-09 16:44:21,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +21: [2023-05-09 16:44:21,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +21: [2023-05-09 16:44:21,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 6: [2023-05-09 16:44:21,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +14: [2023-05-09 16:44:21,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +21: [2023-05-09 16:44:21,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 6: [2023-05-09 16:44:21,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +21: [2023-05-09 16:44:21,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 6: [2023-05-09 16:44:21,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +29: [2023-05-09 16:44:21,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +29: [2023-05-09 16:44:21,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 6: [2023-05-09 16:44:21,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 6: [2023-05-09 16:44:21,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +31: [2023-05-09 16:44:21,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +31: [2023-05-09 16:44:21,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +25: [2023-05-09 16:44:21,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +25: [2023-05-09 16:44:21,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +25: [2023-05-09 16:44:21,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +31: [2023-05-09 16:44:21,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +20: [2023-05-09 16:44:21,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +20: [2023-05-09 16:44:21,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +22: [2023-05-09 16:44:21,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 3: [2023-05-09 16:44:21,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +29: [2023-05-09 16:44:21,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +22: [2023-05-09 16:44:21,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +20: [2023-05-09 16:44:21,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +29: [2023-05-09 16:44:21,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +14: [2023-05-09 16:44:21,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +22: [2023-05-09 16:44:21,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +21: [2023-05-09 16:44:21,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +31: [2023-05-09 16:44:21,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 3: [2023-05-09 16:44:21,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +21: [2023-05-09 16:44:21,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +25: [2023-05-09 16:44:21,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +21: [2023-05-09 16:44:21,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +14: [2023-05-09 16:44:21,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +14: [2023-05-09 16:44:21,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +25: [2023-05-09 16:44:21,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +25: [2023-05-09 16:44:21,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +22: [2023-05-09 16:44:21,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +25: [2023-05-09 16:44:21,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +22: [2023-05-09 16:44:21,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 6: [2023-05-09 16:44:21,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +20: [2023-05-09 16:44:21,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +20: [2023-05-09 16:44:21,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +31: [2023-05-09 16:44:21,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +22: [2023-05-09 16:44:21,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +14: [2023-05-09 16:44:21,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 0: [2023-05-09 16:44:21,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 0: [2023-05-09 16:44:21,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 8: [2023-05-09 16:44:21,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 6: [2023-05-09 16:44:21,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. +20: [2023-05-09 16:44:21,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +25: [2023-05-09 16:44:21,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +22: [2023-05-09 16:44:21,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 6: [2023-05-09 16:44:21,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... +12: [2023-05-09 16:44:21,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +12: [2023-05-09 16:44:21,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +12: [2023-05-09 16:44:21,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +12: [2023-05-09 16:44:21,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +12: [2023-05-09 16:44:21,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +12: [2023-05-09 16:44:21,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +12: [2023-05-09 16:44:21,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +14: [2023-05-09 16:44:21,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +14: [2023-05-09 16:44:21,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +12: [2023-05-09 16:44:21,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt... + 7: [2023-05-09 16:44:21,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 6: [2023-05-09 16:44:21,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +22: [2023-05-09 16:44:21,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +20: [2023-05-09 16:44:21,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +12: [2023-05-09 16:44:21,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +12: [2023-05-09 16:44:21,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +12: [2023-05-09 16:44:21,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +19: [2023-05-09 16:44:21,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +19: [2023-05-09 16:44:21,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +19: [2023-05-09 16:44:21,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +19: [2023-05-09 16:44:21,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +19: [2023-05-09 16:44:21,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +19: [2023-05-09 16:44:21,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +19: [2023-05-09 16:44:21,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +19: [2023-05-09 16:44:21,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +12: [2023-05-09 16:44:21,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +13: [2023-05-09 16:44:21,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +13: [2023-05-09 16:44:21,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +13: [2023-05-09 16:44:21,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +13: [2023-05-09 16:44:21,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +13: [2023-05-09 16:44:21,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +13: [2023-05-09 16:44:21,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +13: [2023-05-09 16:44:21,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +13: [2023-05-09 16:44:21,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +27: [2023-05-09 16:44:21,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +27: [2023-05-09 16:44:21,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +27: [2023-05-09 16:44:21,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +27: [2023-05-09 16:44:21,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +27: [2023-05-09 16:44:21,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +27: [2023-05-09 16:44:21,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +27: [2023-05-09 16:44:21,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +27: [2023-05-09 16:44:21,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +12: [2023-05-09 16:44:21,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +12: [2023-05-09 16:44:21,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +12: [2023-05-09 16:44:21,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 7: [2023-05-09 16:44:21,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +13: [2023-05-09 16:44:21,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +19: [2023-05-09 16:44:21,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +27: [2023-05-09 16:44:21,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +27: [2023-05-09 16:44:21,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +27: [2023-05-09 16:44:21,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +19: [2023-05-09 16:44:21,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +19: [2023-05-09 16:44:21,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +19: [2023-05-09 16:44:21,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +13: [2023-05-09 16:44:21,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +13: [2023-05-09 16:44:21,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +12: [2023-05-09 16:44:21,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 7: [2023-05-09 16:44:21,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +13: [2023-05-09 16:44:21,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +23: [2023-05-09 16:44:21,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +23: [2023-05-09 16:44:21,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +23: [2023-05-09 16:44:21,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +23: [2023-05-09 16:44:21,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +23: [2023-05-09 16:44:21,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +23: [2023-05-09 16:44:21,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +23: [2023-05-09 16:44:21,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +27: [2023-05-09 16:44:21,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +10: [2023-05-09 16:44:21,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +10: [2023-05-09 16:44:21,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +10: [2023-05-09 16:44:21,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +10: [2023-05-09 16:44:21,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +10: [2023-05-09 16:44:21,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +27: [2023-05-09 16:44:21,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +19: [2023-05-09 16:44:21,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +23: [2023-05-09 16:44:21,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +13: [2023-05-09 16:44:21,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +10: [2023-05-09 16:44:21,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +13: [2023-05-09 16:44:21,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +19: [2023-05-09 16:44:21,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +10: [2023-05-09 16:44:21,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +19: [2023-05-09 16:44:21,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +10: [2023-05-09 16:44:21,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +27: [2023-05-09 16:44:21,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +19: [2023-05-09 16:44:21,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +13: [2023-05-09 16:44:21,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +27: [2023-05-09 16:44:21,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +28: [2023-05-09 16:44:21,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +28: [2023-05-09 16:44:21,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +28: [2023-05-09 16:44:21,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +28: [2023-05-09 16:44:21,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +28: [2023-05-09 16:44:21,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +28: [2023-05-09 16:44:21,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +13: [2023-05-09 16:44:21,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +23: [2023-05-09 16:44:21,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +28: [2023-05-09 16:44:21,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +28: [2023-05-09 16:44:21,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +27: [2023-05-09 16:44:21,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +23: [2023-05-09 16:44:21,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +10: [2023-05-09 16:44:21,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +28: [2023-05-09 16:44:21,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +23: [2023-05-09 16:44:21,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +23: [2023-05-09 16:44:21,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +23: [2023-05-09 16:44:21,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 4: [2023-05-09 16:44:21,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 4: [2023-05-09 16:44:21,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +10: [2023-05-09 16:44:21,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +23: [2023-05-09 16:44:21,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +10: [2023-05-09 16:44:21,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +28: [2023-05-09 16:44:21,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +10: [2023-05-09 16:44:21,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +10: [2023-05-09 16:44:21,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +30: [2023-05-09 16:44:21,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +30: [2023-05-09 16:44:21,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +23: [2023-05-09 16:44:21,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +23: [2023-05-09 16:44:21,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +28: [2023-05-09 16:44:21,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +30: [2023-05-09 16:44:21,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +28: [2023-05-09 16:44:21,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +30: [2023-05-09 16:44:21,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +10: [2023-05-09 16:44:21,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +30: [2023-05-09 16:44:21,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +10: [2023-05-09 16:44:21,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +30: [2023-05-09 16:44:21,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +30: [2023-05-09 16:44:21,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +30: [2023-05-09 16:44:21,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +10: [2023-05-09 16:44:21,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +28: [2023-05-09 16:44:21,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +26: [2023-05-09 16:44:21,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +26: [2023-05-09 16:44:21,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +26: [2023-05-09 16:44:21,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +26: [2023-05-09 16:44:21,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +26: [2023-05-09 16:44:21,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +26: [2023-05-09 16:44:21,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +26: [2023-05-09 16:44:21,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +24: [2023-05-09 16:44:21,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +24: [2023-05-09 16:44:21,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +24: [2023-05-09 16:44:21,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +26: [2023-05-09 16:44:21,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +24: [2023-05-09 16:44:21,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +24: [2023-05-09 16:44:21,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +24: [2023-05-09 16:44:21,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +24: [2023-05-09 16:44:21,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +30: [2023-05-09 16:44:21,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +24: [2023-05-09 16:44:21,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +28: [2023-05-09 16:44:21,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +28: [2023-05-09 16:44:21,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +28: [2023-05-09 16:44:21,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +24: [2023-05-09 16:44:21,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +30: [2023-05-09 16:44:21,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +15: [2023-05-09 16:44:21,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +11: [2023-05-09 16:44:21,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +11: [2023-05-09 16:44:21,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +11: [2023-05-09 16:44:21,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +26: [2023-05-09 16:44:21,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +15: [2023-05-09 16:44:21,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +11: [2023-05-09 16:44:21,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +11: [2023-05-09 16:44:21,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +11: [2023-05-09 16:44:21,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +26: [2023-05-09 16:44:21,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +26: [2023-05-09 16:44:21,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +17: [2023-05-09 16:44:21,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +17: [2023-05-09 16:44:21,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +11: [2023-05-09 16:44:21,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +11: [2023-05-09 16:44:21,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +15: [2023-05-09 16:44:21,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +15: [2023-05-09 16:44:21,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +17: [2023-05-09 16:44:21,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +18: [2023-05-09 16:44:21,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +18: [2023-05-09 16:44:21,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +18: [2023-05-09 16:44:21,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +18: [2023-05-09 16:44:21,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +18: [2023-05-09 16:44:21,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +18: [2023-05-09 16:44:21,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +18: [2023-05-09 16:44:21,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +24: [2023-05-09 16:44:21,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +17: [2023-05-09 16:44:21,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +17: [2023-05-09 16:44:21,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +17: [2023-05-09 16:44:21,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +30: [2023-05-09 16:44:21,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +17: [2023-05-09 16:44:21,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +18: [2023-05-09 16:44:21,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +15: [2023-05-09 16:44:21,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +15: [2023-05-09 16:44:21,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +17: [2023-05-09 16:44:21,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +30: [2023-05-09 16:44:21,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +24: [2023-05-09 16:44:21,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +15: [2023-05-09 16:44:21,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +15: [2023-05-09 16:44:21,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +24: [2023-05-09 16:44:21,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +30: [2023-05-09 16:44:21,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +26: [2023-05-09 16:44:21,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +26: [2023-05-09 16:44:21,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +26: [2023-05-09 16:44:21,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +26: [2023-05-09 16:44:21,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +15: [2023-05-09 16:44:21,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +30: [2023-05-09 16:44:21,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +15: [2023-05-09 16:44:21,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +17: [2023-05-09 16:44:21,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +18: [2023-05-09 16:44:21,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +18: [2023-05-09 16:44:21,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +24: [2023-05-09 16:44:21,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +17: [2023-05-09 16:44:21,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +30: [2023-05-09 16:44:21,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +18: [2023-05-09 16:44:21,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +24: [2023-05-09 16:44:21,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +30: [2023-05-09 16:44:21,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +26: [2023-05-09 16:44:21,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +24: [2023-05-09 16:44:21,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +11: [2023-05-09 16:44:21,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +11: [2023-05-09 16:44:21,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +11: [2023-05-09 16:44:21,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +17: [2023-05-09 16:44:21,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +15: [2023-05-09 16:44:21,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +18: [2023-05-09 16:44:21,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +11: [2023-05-09 16:44:21,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +18: [2023-05-09 16:44:21,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +15: [2023-05-09 16:44:21,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +17: [2023-05-09 16:44:21,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +17: [2023-05-09 16:44:21,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +24: [2023-05-09 16:44:21,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +17: [2023-05-09 16:44:21,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +18: [2023-05-09 16:44:21,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +15: [2023-05-09 16:44:21,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +11: [2023-05-09 16:44:21,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +15: [2023-05-09 16:44:21,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +11: [2023-05-09 16:44:21,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +11: [2023-05-09 16:44:21,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +18: [2023-05-09 16:44:21,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +17: [2023-05-09 16:44:21,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +11: [2023-05-09 16:44:21,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +17: [2023-05-09 16:44:21,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +15: [2023-05-09 16:44:21,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +15: [2023-05-09 16:44:21,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +18: [2023-05-09 16:44:21,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +14: [2023-05-09 16:44:21,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +14: [2023-05-09 16:44:21,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +14: [2023-05-09 16:44:21,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +14: [2023-05-09 16:44:21,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +29: [2023-05-09 16:44:21,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +29: [2023-05-09 16:44:21,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +29: [2023-05-09 16:44:21,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +29: [2023-05-09 16:44:21,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +14: [2023-05-09 16:44:21,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +14: [2023-05-09 16:44:21,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +14: [2023-05-09 16:44:21,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +14: [2023-05-09 16:44:21,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +29: [2023-05-09 16:44:21,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +29: [2023-05-09 16:44:21,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +29: [2023-05-09 16:44:21,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +29: [2023-05-09 16:44:21,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +14: [2023-05-09 16:44:21,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +14: [2023-05-09 16:44:21,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 1: [2023-05-09 16:44:21,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 1: [2023-05-09 16:44:21,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 1: [2023-05-09 16:44:21,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 1: [2023-05-09 16:44:21,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +29: [2023-05-09 16:44:21,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +29: [2023-05-09 16:44:21,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +14: [2023-05-09 16:44:21,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +14: [2023-05-09 16:44:21,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +14: [2023-05-09 16:44:21,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +14: [2023-05-09 16:44:21,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +29: [2023-05-09 16:44:21,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +29: [2023-05-09 16:44:21,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +29: [2023-05-09 16:44:21,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +29: [2023-05-09 16:44:21,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +29: [2023-05-09 16:44:21,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +29: [2023-05-09 16:44:21,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 1: [2023-05-09 16:44:21,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +14: [2023-05-09 16:44:21,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +14: [2023-05-09 16:44:21,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 9: [2023-05-09 16:44:21,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 9: [2023-05-09 16:44:21,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 9: [2023-05-09 16:44:21,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 9: [2023-05-09 16:44:21,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 1: [2023-05-09 16:44:21,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 9: [2023-05-09 16:44:21,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 1: [2023-05-09 16:44:21,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 1: [2023-05-09 16:44:21,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +11: [2023-05-09 16:44:21,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +11: [2023-05-09 16:44:21,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +11: [2023-05-09 16:44:21,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +11: [2023-05-09 16:44:21,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 1: [2023-05-09 16:44:21,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 1: [2023-05-09 16:44:21,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 1: [2023-05-09 16:44:21,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 8: [2023-05-09 16:44:21,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 9: [2023-05-09 16:44:21,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 9: [2023-05-09 16:44:21,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 9: [2023-05-09 16:44:21,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 8: [2023-05-09 16:44:21,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +11: [2023-05-09 16:44:21,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +11: [2023-05-09 16:44:21,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +22: [2023-05-09 16:44:21,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 9: [2023-05-09 16:44:21,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 9: [2023-05-09 16:44:21,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +22: [2023-05-09 16:44:21,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +11: [2023-05-09 16:44:21,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +11: [2023-05-09 16:44:21,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +22: [2023-05-09 16:44:21,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +22: [2023-05-09 16:44:21,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 8: [2023-05-09 16:44:21,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 8: [2023-05-09 16:44:21,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +22: [2023-05-09 16:44:21,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +11: [2023-05-09 16:44:21,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +11: [2023-05-09 16:44:21,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +22: [2023-05-09 16:44:21,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +20: [2023-05-09 16:44:21,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +20: [2023-05-09 16:44:21,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +11: [2023-05-09 16:44:21,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +11: [2023-05-09 16:44:21,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +20: [2023-05-09 16:44:21,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +20: [2023-05-09 16:44:21,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +20: [2023-05-09 16:44:21,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +20: [2023-05-09 16:44:21,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +20: [2023-05-09 16:44:21,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +20: [2023-05-09 16:44:21,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +22: [2023-05-09 16:44:21,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +22: [2023-05-09 16:44:21,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +11: [2023-05-09 16:44:21,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +11: [2023-05-09 16:44:21,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +11: [2023-05-09 16:44:21,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +11: [2023-05-09 16:44:21,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +24: [2023-05-09 16:44:21,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +24: [2023-05-09 16:44:21,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +24: [2023-05-09 16:44:21,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 8: [2023-05-09 16:44:21,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +24: [2023-05-09 16:44:21,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 8: [2023-05-09 16:44:21,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +20: [2023-05-09 16:44:21,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +20: [2023-05-09 16:44:21,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +20: [2023-05-09 16:44:21,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +20: [2023-05-09 16:44:21,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +22: [2023-05-09 16:44:21,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +22: [2023-05-09 16:44:21,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +22: [2023-05-09 16:44:21,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +22: [2023-05-09 16:44:21,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +24: [2023-05-09 16:44:21,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +22: [2023-05-09 16:44:21,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +22: [2023-05-09 16:44:21,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +24: [2023-05-09 16:44:21,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +24: [2023-05-09 16:44:21,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 8: [2023-05-09 16:44:21,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +24: [2023-05-09 16:44:21,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 5: [2023-05-09 16:44:21,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 5: [2023-05-09 16:44:21,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 5: [2023-05-09 16:44:21,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +22: [2023-05-09 16:44:21,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +22: [2023-05-09 16:44:21,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +20: [2023-05-09 16:44:21,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +20: [2023-05-09 16:44:21,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +20: [2023-05-09 16:44:21,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +20: [2023-05-09 16:44:21,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +24: [2023-05-09 16:44:21,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +24: [2023-05-09 16:44:21,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 5: [2023-05-09 16:44:21,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 5: [2023-05-09 16:44:21,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +24: [2023-05-09 16:44:21,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +24: [2023-05-09 16:44:21,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 2: [2023-05-09 16:44:21,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 2: [2023-05-09 16:44:21,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 2: [2023-05-09 16:44:21,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 2: [2023-05-09 16:44:21,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +12: [2023-05-09 16:44:21,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +12: [2023-05-09 16:44:21,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +12: [2023-05-09 16:44:21,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +12: [2023-05-09 16:44:21,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +24: [2023-05-09 16:44:21,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +24: [2023-05-09 16:44:21,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +24: [2023-05-09 16:44:21,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +24: [2023-05-09 16:44:21,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 5: [2023-05-09 16:44:21,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 2: [2023-05-09 16:44:21,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +18: [2023-05-09 16:44:21,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +18: [2023-05-09 16:44:21,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +12: [2023-05-09 16:44:21,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +12: [2023-05-09 16:44:21,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +12: [2023-05-09 16:44:21,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +12: [2023-05-09 16:44:21,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 0: [2023-05-09 16:44:21,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +18: [2023-05-09 16:44:21,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 5: [2023-05-09 16:44:21,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +18: [2023-05-09 16:44:21,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 0: [2023-05-09 16:44:21,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +25: [2023-05-09 16:44:21,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +25: [2023-05-09 16:44:21,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +25: [2023-05-09 16:44:21,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +25: [2023-05-09 16:44:21,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +19: [2023-05-09 16:44:21,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +19: [2023-05-09 16:44:21,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +19: [2023-05-09 16:44:21,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +19: [2023-05-09 16:44:21,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +19: [2023-05-09 16:44:21,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +19: [2023-05-09 16:44:21,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +19: [2023-05-09 16:44:21,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +19: [2023-05-09 16:44:21,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +25: [2023-05-09 16:44:21,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +25: [2023-05-09 16:44:21,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +17: [2023-05-09 16:44:21,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +17: [2023-05-09 16:44:21,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +25: [2023-05-09 16:44:21,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +25: [2023-05-09 16:44:21,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +18: [2023-05-09 16:44:21,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 2: [2023-05-09 16:44:21,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 5: [2023-05-09 16:44:21,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +10: [2023-05-09 16:44:21,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 2: [2023-05-09 16:44:21,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +10: [2023-05-09 16:44:21,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +17: [2023-05-09 16:44:21,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 2: [2023-05-09 16:44:21,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +18: [2023-05-09 16:44:21,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +10: [2023-05-09 16:44:21,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +17: [2023-05-09 16:44:21,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +10: [2023-05-09 16:44:21,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +19: [2023-05-09 16:44:21,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 5: [2023-05-09 16:44:21,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 0: [2023-05-09 16:44:21,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +19: [2023-05-09 16:44:21,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +12: [2023-05-09 16:44:21,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +12: [2023-05-09 16:44:21,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +12: [2023-05-09 16:44:21,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +12: [2023-05-09 16:44:21,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 2: [2023-05-09 16:44:21,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +18: [2023-05-09 16:44:21,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +18: [2023-05-09 16:44:21,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +25: [2023-05-09 16:44:21,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +19: [2023-05-09 16:44:21,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +13: [2023-05-09 16:44:21,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +13: [2023-05-09 16:44:21,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +13: [2023-05-09 16:44:21,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +13: [2023-05-09 16:44:21,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +17: [2023-05-09 16:44:21,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +17: [2023-05-09 16:44:21,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +19: [2023-05-09 16:44:21,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +19: [2023-05-09 16:44:21,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +12: [2023-05-09 16:44:21,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +12: [2023-05-09 16:44:21,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +12: [2023-05-09 16:44:21,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +12: [2023-05-09 16:44:21,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +18: [2023-05-09 16:44:21,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +17: [2023-05-09 16:44:21,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +18: [2023-05-09 16:44:21,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +19: [2023-05-09 16:44:21,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +19: [2023-05-09 16:44:21,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 0: [2023-05-09 16:44:21,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +17: [2023-05-09 16:44:21,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +25: [2023-05-09 16:44:21,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 2: [2023-05-09 16:44:21,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +10: [2023-05-09 16:44:21,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +10: [2023-05-09 16:44:21,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +10: [2023-05-09 16:44:21,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +25: [2023-05-09 16:44:22,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 0: [2023-05-09 16:44:22,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +25: [2023-05-09 16:44:22,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +18: [2023-05-09 16:44:22,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 2: [2023-05-09 16:44:22,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +18: [2023-05-09 16:44:22,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 0: [2023-05-09 16:44:22,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +10: [2023-05-09 16:44:22,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +13: [2023-05-09 16:44:22,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +13: [2023-05-09 16:44:22,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +19: [2023-05-09 16:44:22,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +13: [2023-05-09 16:44:22,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +17: [2023-05-09 16:44:22,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +13: [2023-05-09 16:44:22,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +17: [2023-05-09 16:44:22,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +18: [2023-05-09 16:44:22,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +18: [2023-05-09 16:44:22,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +25: [2023-05-09 16:44:22,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +19: [2023-05-09 16:44:22,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +19: [2023-05-09 16:44:22,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +25: [2023-05-09 16:44:22,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +18: [2023-05-09 16:44:22,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 0: [2023-05-09 16:44:22,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +19: [2023-05-09 16:44:22,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +19: [2023-05-09 16:44:22,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +18: [2023-05-09 16:44:22,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +25: [2023-05-09 16:44:22,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +17: [2023-05-09 16:44:22,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +17: [2023-05-09 16:44:22,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +13: [2023-05-09 16:44:22,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +17: [2023-05-09 16:44:22,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +17: [2023-05-09 16:44:22,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +17: [2023-05-09 16:44:22,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +17: [2023-05-09 16:44:22,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +13: [2023-05-09 16:44:22,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +25: [2023-05-09 16:44:22,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +13: [2023-05-09 16:44:22,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +13: [2023-05-09 16:44:22,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +10: [2023-05-09 16:44:22,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +10: [2023-05-09 16:44:22,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +10: [2023-05-09 16:44:22,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +10: [2023-05-09 16:44:22,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +19: [2023-05-09 16:44:22,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +19: [2023-05-09 16:44:22,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +19: [2023-05-09 16:44:22,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +10: [2023-05-09 16:44:22,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +10: [2023-05-09 16:44:22,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +19: [2023-05-09 16:44:22,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +10: [2023-05-09 16:44:22,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +10: [2023-05-09 16:44:22,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +19: [2023-05-09 16:44:22,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +19: [2023-05-09 16:44:22,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +19: [2023-05-09 16:44:22,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 7: [2023-05-09 16:44:22,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +13: [2023-05-09 16:44:22,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +13: [2023-05-09 16:44:22,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 5: [2023-05-09 16:44:22,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +10: [2023-05-09 16:44:22,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +10: [2023-05-09 16:44:22,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +10: [2023-05-09 16:44:22,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +10: [2023-05-09 16:44:22,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +10: [2023-05-09 16:44:22,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +10: [2023-05-09 16:44:22,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +10: [2023-05-09 16:44:22,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +10: [2023-05-09 16:44:22,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +19: [2023-05-09 16:44:22,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +19: [2023-05-09 16:44:22,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +19: [2023-05-09 16:44:22,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +13: [2023-05-09 16:44:22,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +17: [2023-05-09 16:44:22,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +17: [2023-05-09 16:44:22,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +29: [2023-05-09 16:44:22,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +29: [2023-05-09 16:44:22,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +17: [2023-05-09 16:44:22,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +17: [2023-05-09 16:44:22,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +29: [2023-05-09 16:44:22,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +29: [2023-05-09 16:44:22,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +17: [2023-05-09 16:44:22,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +17: [2023-05-09 16:44:22,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +17: [2023-05-09 16:44:22,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +17: [2023-05-09 16:44:22,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +29: [2023-05-09 16:44:22,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +29: [2023-05-09 16:44:22,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +29: [2023-05-09 16:44:22,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +13: [2023-05-09 16:44:22,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +19: [2023-05-09 16:44:22,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +29: [2023-05-09 16:44:22,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +19: [2023-05-09 16:44:22,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 7: [2023-05-09 16:44:22,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +12: [2023-05-09 16:44:22,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +12: [2023-05-09 16:44:22,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +12: [2023-05-09 16:44:22,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +12: [2023-05-09 16:44:22,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +12: [2023-05-09 16:44:22,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +12: [2023-05-09 16:44:22,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +12: [2023-05-09 16:44:22,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +10: [2023-05-09 16:44:22,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +12: [2023-05-09 16:44:22,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +17: [2023-05-09 16:44:22,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +17: [2023-05-09 16:44:22,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +10: [2023-05-09 16:44:22,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +29: [2023-05-09 16:44:22,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +17: [2023-05-09 16:44:22,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +29: [2023-05-09 16:44:22,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +17: [2023-05-09 16:44:22,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +29: [2023-05-09 16:44:22,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +10: [2023-05-09 16:44:22,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +10: [2023-05-09 16:44:22,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +10: [2023-05-09 16:44:22,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +14: [2023-05-09 16:44:22,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +14: [2023-05-09 16:44:22,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +14: [2023-05-09 16:44:22,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +14: [2023-05-09 16:44:22,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +14: [2023-05-09 16:44:22,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +14: [2023-05-09 16:44:22,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +14: [2023-05-09 16:44:22,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +14: [2023-05-09 16:44:22,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +10: [2023-05-09 16:44:22,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +29: [2023-05-09 16:44:22,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +17: [2023-05-09 16:44:22,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +29: [2023-05-09 16:44:22,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +29: [2023-05-09 16:44:22,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 6: [2023-05-09 16:44:22,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +10: [2023-05-09 16:44:22,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +10: [2023-05-09 16:44:22,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +12: [2023-05-09 16:44:22,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +12: [2023-05-09 16:44:22,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +12: [2023-05-09 16:44:22,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +17: [2023-05-09 16:44:22,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +17: [2023-05-09 16:44:22,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +21: [2023-05-09 16:44:22,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +21: [2023-05-09 16:44:22,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +21: [2023-05-09 16:44:22,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +29: [2023-05-09 16:44:22,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +21: [2023-05-09 16:44:22,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +28: [2023-05-09 16:44:22,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +28: [2023-05-09 16:44:22,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +28: [2023-05-09 16:44:22,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +28: [2023-05-09 16:44:22,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +28: [2023-05-09 16:44:22,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +28: [2023-05-09 16:44:22,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +17: [2023-05-09 16:44:22,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +28: [2023-05-09 16:44:22,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +28: [2023-05-09 16:44:22,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +31: [2023-05-09 16:44:22,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +31: [2023-05-09 16:44:22,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +31: [2023-05-09 16:44:22,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +31: [2023-05-09 16:44:22,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +31: [2023-05-09 16:44:22,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +31: [2023-05-09 16:44:22,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +31: [2023-05-09 16:44:22,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +23: [2023-05-09 16:44:22,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +23: [2023-05-09 16:44:22,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +23: [2023-05-09 16:44:22,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +23: [2023-05-09 16:44:22,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +14: [2023-05-09 16:44:22,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +31: [2023-05-09 16:44:22,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +23: [2023-05-09 16:44:22,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +23: [2023-05-09 16:44:22,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +23: [2023-05-09 16:44:22,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +29: [2023-05-09 16:44:22,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +23: [2023-05-09 16:44:22,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +14: [2023-05-09 16:44:22,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +14: [2023-05-09 16:44:22,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +12: [2023-05-09 16:44:22,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +12: [2023-05-09 16:44:22,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +12: [2023-05-09 16:44:22,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +21: [2023-05-09 16:44:22,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +21: [2023-05-09 16:44:22,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +21: [2023-05-09 16:44:22,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +21: [2023-05-09 16:44:22,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +12: [2023-05-09 16:44:22,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +28: [2023-05-09 16:44:22,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +14: [2023-05-09 16:44:22,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +14: [2023-05-09 16:44:22,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +31: [2023-05-09 16:44:22,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +23: [2023-05-09 16:44:22,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +14: [2023-05-09 16:44:22,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +28: [2023-05-09 16:44:22,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +23: [2023-05-09 16:44:22,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +12: [2023-05-09 16:44:22,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +31: [2023-05-09 16:44:22,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +31: [2023-05-09 16:44:22,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +14: [2023-05-09 16:44:22,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +28: [2023-05-09 16:44:22,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +28: [2023-05-09 16:44:22,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +28: [2023-05-09 16:44:22,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +25: [2023-05-09 16:44:22,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +25: [2023-05-09 16:44:22,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +21: [2023-05-09 16:44:22,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +21: [2023-05-09 16:44:22,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +21: [2023-05-09 16:44:22,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +21: [2023-05-09 16:44:22,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +21: [2023-05-09 16:44:22,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +21: [2023-05-09 16:44:22,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +21: [2023-05-09 16:44:22,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +21: [2023-05-09 16:44:22,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +31: [2023-05-09 16:44:22,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +14: [2023-05-09 16:44:22,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +25: [2023-05-09 16:44:22,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +25: [2023-05-09 16:44:22,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 6: [2023-05-09 16:44:22,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 6: [2023-05-09 16:44:22,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +23: [2023-05-09 16:44:22,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +21: [2023-05-09 16:44:22,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +21: [2023-05-09 16:44:22,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +21: [2023-05-09 16:44:22,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +21: [2023-05-09 16:44:22,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +21: [2023-05-09 16:44:22,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +21: [2023-05-09 16:44:22,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +21: [2023-05-09 16:44:22,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +21: [2023-05-09 16:44:22,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +28: [2023-05-09 16:44:22,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +16: [2023-05-09 16:44:22,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +16: [2023-05-09 16:44:22,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 3: [2023-05-09 16:44:22,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +16: [2023-05-09 16:44:22,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 6: [2023-05-09 16:44:22,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +23: [2023-05-09 16:44:22,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +31: [2023-05-09 16:44:22,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +31: [2023-05-09 16:44:22,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +31: [2023-05-09 16:44:22,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +28: [2023-05-09 16:44:22,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +16: [2023-05-09 16:44:22,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +28: [2023-05-09 16:44:22,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +23: [2023-05-09 16:44:22,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +26: [2023-05-09 16:44:22,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 4: [2023-05-09 16:44:22,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +23: [2023-05-09 16:44:22,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +30: [2023-05-09 16:44:22,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +30: [2023-05-09 16:44:22,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +30: [2023-05-09 16:44:22,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +26: [2023-05-09 16:44:22,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +26: [2023-05-09 16:44:22,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +26: [2023-05-09 16:44:22,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 4: [2023-05-09 16:44:22,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +30: [2023-05-09 16:44:22,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 6: [2023-05-09 16:44:22,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 8: [2023-05-09 16:44:22,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 8: [2023-05-09 16:44:22,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 8: [2023-05-09 16:44:22,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 4: [2023-05-09 16:44:22,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 0: [2023-05-09 16:44:22,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +25: [2023-05-09 16:44:22,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 3: [2023-05-09 16:44:22,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +25: [2023-05-09 16:44:22,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 3: [2023-05-09 16:44:22,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 6: [2023-05-09 16:44:22,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 0: [2023-05-09 16:44:22,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +25: [2023-05-09 16:44:22,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +23: [2023-05-09 16:44:22,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +23: [2023-05-09 16:44:22,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +16: [2023-05-09 16:44:22,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +25: [2023-05-09 16:44:22,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +31: [2023-05-09 16:44:22,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +20: [2023-05-09 16:44:22,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +20: [2023-05-09 16:44:22,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +20: [2023-05-09 16:44:22,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 0: [2023-05-09 16:44:22,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +20: [2023-05-09 16:44:22,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +31: [2023-05-09 16:44:22,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +16: [2023-05-09 16:44:22,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +16: [2023-05-09 16:44:22,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +31: [2023-05-09 16:44:22,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 7: [2023-05-09 16:44:22,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 7: [2023-05-09 16:44:22,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 7: [2023-05-09 16:44:22,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +14: [2023-05-09 16:44:22,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +14: [2023-05-09 16:44:22,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +14: [2023-05-09 16:44:22,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +14: [2023-05-09 16:44:22,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +23: [2023-05-09 16:44:22,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +23: [2023-05-09 16:44:22,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +16: [2023-05-09 16:44:22,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +30: [2023-05-09 16:44:22,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +26: [2023-05-09 16:44:22,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +30: [2023-05-09 16:44:22,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +26: [2023-05-09 16:44:22,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +31: [2023-05-09 16:44:22,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +26: [2023-05-09 16:44:22,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +23: [2023-05-09 16:44:22,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 4: [2023-05-09 16:44:22,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +26: [2023-05-09 16:44:22,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +23: [2023-05-09 16:44:22,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +30: [2023-05-09 16:44:22,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +31: [2023-05-09 16:44:22,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +30: [2023-05-09 16:44:22,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +21: [2023-05-09 16:44:22,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +21: [2023-05-09 16:44:22,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +21: [2023-05-09 16:44:22,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 0: [2023-05-09 16:44:22,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +18: [2023-05-09 16:44:22,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +18: [2023-05-09 16:44:22,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +18: [2023-05-09 16:44:22,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +18: [2023-05-09 16:44:22,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +18: [2023-05-09 16:44:22,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +18: [2023-05-09 16:44:22,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +18: [2023-05-09 16:44:22,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +20: [2023-05-09 16:44:22,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +18: [2023-05-09 16:44:22,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +20: [2023-05-09 16:44:22,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +20: [2023-05-09 16:44:22,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +20: [2023-05-09 16:44:22,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +25: [2023-05-09 16:44:22,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +26: [2023-05-09 16:44:22,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +26: [2023-05-09 16:44:22,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +26: [2023-05-09 16:44:22,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +26: [2023-05-09 16:44:22,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +26: [2023-05-09 16:44:22,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +26: [2023-05-09 16:44:22,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +26: [2023-05-09 16:44:22,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +25: [2023-05-09 16:44:22,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +26: [2023-05-09 16:44:22,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +21: [2023-05-09 16:44:22,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +23: [2023-05-09 16:44:22,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +23: [2023-05-09 16:44:22,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +31: [2023-05-09 16:44:22,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +31: [2023-05-09 16:44:22,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +31: [2023-05-09 16:44:22,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +23: [2023-05-09 16:44:22,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +28: [2023-05-09 16:44:22,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +28: [2023-05-09 16:44:22,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 7: [2023-05-09 16:44:22,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +28: [2023-05-09 16:44:22,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +14: [2023-05-09 16:44:22,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +14: [2023-05-09 16:44:22,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +14: [2023-05-09 16:44:22,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +23: [2023-05-09 16:44:22,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +28: [2023-05-09 16:44:22,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +16: [2023-05-09 16:44:22,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +31: [2023-05-09 16:44:22,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +16: [2023-05-09 16:44:22,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +15: [2023-05-09 16:44:22,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +15: [2023-05-09 16:44:22,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +15: [2023-05-09 16:44:22,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +21: [2023-05-09 16:44:22,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +14: [2023-05-09 16:44:22,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +16: [2023-05-09 16:44:22,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +16: [2023-05-09 16:44:22,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +16: [2023-05-09 16:44:22,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +27: [2023-05-09 16:44:22,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +27: [2023-05-09 16:44:22,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +27: [2023-05-09 16:44:22,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +27: [2023-05-09 16:44:22,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. +16: [2023-05-09 16:44:22,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +22: [2023-05-09 16:44:22,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +22: [2023-05-09 16:44:22,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +26: [2023-05-09 16:44:22,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +22: [2023-05-09 16:44:22,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +22: [2023-05-09 16:44:22,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +26: [2023-05-09 16:44:22,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +21: [2023-05-09 16:44:22,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +21: [2023-05-09 16:44:22,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +30: [2023-05-09 16:44:22,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +30: [2023-05-09 16:44:22,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +30: [2023-05-09 16:44:22,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +15: [2023-05-09 16:44:22,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt. + 9: [2023-05-09 16:44:22,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +30: [2023-05-09 16:44:22,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +30: [2023-05-09 16:44:22,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +30: [2023-05-09 16:44:22,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +25: [2023-05-09 16:44:22,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +16: [2023-05-09 16:44:22,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +16: [2023-05-09 16:44:22,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +25: [2023-05-09 16:44:22,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 9: [2023-05-09 16:44:22,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +18: [2023-05-09 16:44:22,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +21: [2023-05-09 16:44:22,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +20: [2023-05-09 16:44:22,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +26: [2023-05-09 16:44:22,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +26: [2023-05-09 16:44:22,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +26: [2023-05-09 16:44:22,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +20: [2023-05-09 16:44:22,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 4: [2023-05-09 16:44:22,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +31: [2023-05-09 16:44:22,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +26: [2023-05-09 16:44:22,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +31: [2023-05-09 16:44:22,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 3: [2023-05-09 16:44:22,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 3: [2023-05-09 16:44:22,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +28: [2023-05-09 16:44:22,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +20: [2023-05-09 16:44:22,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +20: [2023-05-09 16:44:22,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +28: [2023-05-09 16:44:22,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +28: [2023-05-09 16:44:22,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +20: [2023-05-09 16:44:22,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +20: [2023-05-09 16:44:22,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +20: [2023-05-09 16:44:22,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +20: [2023-05-09 16:44:22,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +25: [2023-05-09 16:44:22,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +25: [2023-05-09 16:44:22,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +23: [2023-05-09 16:44:22,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +26: [2023-05-09 16:44:22,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +26: [2023-05-09 16:44:22,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +27: [2023-05-09 16:44:22,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +23: [2023-05-09 16:44:22,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 8: [2023-05-09 16:44:22,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 6: [2023-05-09 16:44:22,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +15: [2023-05-09 16:44:22,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +27: [2023-05-09 16:44:22,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +23: [2023-05-09 16:44:22,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +23: [2023-05-09 16:44:22,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +18: [2023-05-09 16:44:22,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +22: [2023-05-09 16:44:22,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +26: [2023-05-09 16:44:22,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +26: [2023-05-09 16:44:22,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +22: [2023-05-09 16:44:22,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +26: [2023-05-09 16:44:22,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +30: [2023-05-09 16:44:22,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +26: [2023-05-09 16:44:22,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +15: [2023-05-09 16:44:22,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +27: [2023-05-09 16:44:22,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +30: [2023-05-09 16:44:22,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +14: [2023-05-09 16:44:22,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +28: [2023-05-09 16:44:22,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +27: [2023-05-09 16:44:22,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +18: [2023-05-09 16:44:22,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +18: [2023-05-09 16:44:22,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 6: [2023-05-09 16:44:22,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +22: [2023-05-09 16:44:22,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +15: [2023-05-09 16:44:22,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... +22: [2023-05-09 16:44:22,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +25: [2023-05-09 16:44:22,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 0: [2023-05-09 16:44:22,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 0: [2023-05-09 16:44:22,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 0: [2023-05-09 16:44:22,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +14: [2023-05-09 16:44:22,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 0: [2023-05-09 16:44:22,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 0: [2023-05-09 16:44:22,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 0: [2023-05-09 16:44:22,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +25: [2023-05-09 16:44:22,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 8: [2023-05-09 16:44:22,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 4: [2023-05-09 16:44:22,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +15: [2023-05-09 16:44:22,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 3: [2023-05-09 16:44:22,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +31: [2023-05-09 16:44:22,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +31: [2023-05-09 16:44:22,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +31: [2023-05-09 16:44:22,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +31: [2023-05-09 16:44:22,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 4: [2023-05-09 16:44:22,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 3: [2023-05-09 16:44:22,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 9: [2023-05-09 16:44:22,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 9: [2023-05-09 16:44:22,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 9: [2023-05-09 16:44:22,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +29: [2023-05-09 16:44:22,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +29: [2023-05-09 16:44:22,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +29: [2023-05-09 16:44:22,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +29: [2023-05-09 16:44:22,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 7: [2023-05-09 16:44:22,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +14: [2023-05-09 16:44:22,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +14: [2023-05-09 16:44:22,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +31: [2023-05-09 16:44:22,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +31: [2023-05-09 16:44:22,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 9: [2023-05-09 16:44:22,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 0: [2023-05-09 16:44:22,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 0: [2023-05-09 16:44:22,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 7: [2023-05-09 16:44:22,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +23: [2023-05-09 16:44:22,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +23: [2023-05-09 16:44:22,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +26: [2023-05-09 16:44:22,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 9: [2023-05-09 16:44:22,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +26: [2023-05-09 16:44:22,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +23: [2023-05-09 16:44:22,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +23: [2023-05-09 16:44:22,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +18: [2023-05-09 16:44:22,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +26: [2023-05-09 16:44:22,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +26: [2023-05-09 16:44:22,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +18: [2023-05-09 16:44:22,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +18: [2023-05-09 16:44:22,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +13: [2023-05-09 16:44:22,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +13: [2023-05-09 16:44:22,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +14: [2023-05-09 16:44:22,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +14: [2023-05-09 16:44:22,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +13: [2023-05-09 16:44:22,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +13: [2023-05-09 16:44:22,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +27: [2023-05-09 16:44:22,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +13: [2023-05-09 16:44:22,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +27: [2023-05-09 16:44:22,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +13: [2023-05-09 16:44:22,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +13: [2023-05-09 16:44:22,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +29: [2023-05-09 16:44:22,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +22: [2023-05-09 16:44:22,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +22: [2023-05-09 16:44:22,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +22: [2023-05-09 16:44:22,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +22: [2023-05-09 16:44:22,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +29: [2023-05-09 16:44:22,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +28: [2023-05-09 16:44:22,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +15: [2023-05-09 16:44:22,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +13: [2023-05-09 16:44:22,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +15: [2023-05-09 16:44:22,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 9: [2023-05-09 16:44:22,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +28: [2023-05-09 16:44:22,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +28: [2023-05-09 16:44:22,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +29: [2023-05-09 16:44:22,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +28: [2023-05-09 16:44:22,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +28: [2023-05-09 16:44:22,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +28: [2023-05-09 16:44:22,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +15: [2023-05-09 16:44:22,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +27: [2023-05-09 16:44:22,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +29: [2023-05-09 16:44:22,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +27: [2023-05-09 16:44:22,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +15: [2023-05-09 16:44:22,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +15: [2023-05-09 16:44:22,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +15: [2023-05-09 16:44:22,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +18: [2023-05-09 16:44:22,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +15: [2023-05-09 16:44:22,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +15: [2023-05-09 16:44:22,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... + 9: [2023-05-09 16:44:22,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +14: [2023-05-09 16:44:22,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +28: [2023-05-09 16:44:22,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +28: [2023-05-09 16:44:22,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +14: [2023-05-09 16:44:22,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 9: [2023-05-09 16:44:22,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +13: [2023-05-09 16:44:22,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +27: [2023-05-09 16:44:22,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +22: [2023-05-09 16:44:22,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +29: [2023-05-09 16:44:22,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +29: [2023-05-09 16:44:22,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +27: [2023-05-09 16:44:22,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt. +27: [2023-05-09 16:44:22,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +22: [2023-05-09 16:44:22,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 6: [2023-05-09 16:44:22,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +22: [2023-05-09 16:44:22,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +27: [2023-05-09 16:44:22,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt... +13: [2023-05-09 16:44:22,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +13: [2023-05-09 16:44:22,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +22: [2023-05-09 16:44:22,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 6: [2023-05-09 16:44:22,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +29: [2023-05-09 16:44:22,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +29: [2023-05-09 16:44:22,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 9: [2023-05-09 16:44:22,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +13: [2023-05-09 16:44:22,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +29: [2023-05-09 16:44:22,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +29: [2023-05-09 16:44:22,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 9: [2023-05-09 16:44:22,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 9: [2023-05-09 16:44:22,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 9: [2023-05-09 16:44:22,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 6: [2023-05-09 16:44:22,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +13: [2023-05-09 16:44:22,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +13: [2023-05-09 16:44:22,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +29: [2023-05-09 16:44:22,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +29: [2023-05-09 16:44:22,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 6: [2023-05-09 16:44:22,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +13: [2023-05-09 16:44:22,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +30: [2023-05-09 16:44:22,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +30: [2023-05-09 16:44:22,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +30: [2023-05-09 16:44:22,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +30: [2023-05-09 16:44:22,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +30: [2023-05-09 16:44:22,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 2: [2023-05-09 16:44:22,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +30: [2023-05-09 16:44:22,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +30: [2023-05-09 16:44:22,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +30: [2023-05-09 16:44:22,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 2: [2023-05-09 16:44:22,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 2: [2023-05-09 16:44:22,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +15: [2023-05-09 16:44:22,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +15: [2023-05-09 16:44:22,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +15: [2023-05-09 16:44:22,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +15: [2023-05-09 16:44:22,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +15: [2023-05-09 16:44:22,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +15: [2023-05-09 16:44:22,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +15: [2023-05-09 16:44:22,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +13: [2023-05-09 16:44:22,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +11: [2023-05-09 16:44:22,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +11: [2023-05-09 16:44:22,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +11: [2023-05-09 16:44:22,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +11: [2023-05-09 16:44:22,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +11: [2023-05-09 16:44:22,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +15: [2023-05-09 16:44:22,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +11: [2023-05-09 16:44:22,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +11: [2023-05-09 16:44:22,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +11: [2023-05-09 16:44:22,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +22: [2023-05-09 16:44:22,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +22: [2023-05-09 16:44:22,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +22: [2023-05-09 16:44:22,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +22: [2023-05-09 16:44:22,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +22: [2023-05-09 16:44:22,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +27: [2023-05-09 16:44:22,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +27: [2023-05-09 16:44:22,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +22: [2023-05-09 16:44:22,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +27: [2023-05-09 16:44:22,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +27: [2023-05-09 16:44:22,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +22: [2023-05-09 16:44:22,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +22: [2023-05-09 16:44:22,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +16: [2023-05-09 16:44:22,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +16: [2023-05-09 16:44:22,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +16: [2023-05-09 16:44:22,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +16: [2023-05-09 16:44:22,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +27: [2023-05-09 16:44:22,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +27: [2023-05-09 16:44:22,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +24: [2023-05-09 16:44:22,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +24: [2023-05-09 16:44:22,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +16: [2023-05-09 16:44:22,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +16: [2023-05-09 16:44:22,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +16: [2023-05-09 16:44:22,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +24: [2023-05-09 16:44:22,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +24: [2023-05-09 16:44:22,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +27: [2023-05-09 16:44:22,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +27: [2023-05-09 16:44:22,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +24: [2023-05-09 16:44:22,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +24: [2023-05-09 16:44:22,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +24: [2023-05-09 16:44:22,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +16: [2023-05-09 16:44:22,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +24: [2023-05-09 16:44:22,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +30: [2023-05-09 16:44:22,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +24: [2023-05-09 16:44:22,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +24: [2023-05-09 16:44:22,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +24: [2023-05-09 16:44:22,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 2: [2023-05-09 16:44:22,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +15: [2023-05-09 16:44:22,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +24: [2023-05-09 16:44:22,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +11: [2023-05-09 16:44:22,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 2: [2023-05-09 16:44:22,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +11: [2023-05-09 16:44:22,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 1: [2023-05-09 16:44:22,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 1: [2023-05-09 16:44:22,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 1: [2023-05-09 16:44:22,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 2: [2023-05-09 16:44:22,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 2: [2023-05-09 16:44:22,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 2: [2023-05-09 16:44:22,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +30: [2023-05-09 16:44:22,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +30: [2023-05-09 16:44:22,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +15: [2023-05-09 16:44:22,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +15: [2023-05-09 16:44:22,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +11: [2023-05-09 16:44:22,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +22: [2023-05-09 16:44:22,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +22: [2023-05-09 16:44:22,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +16: [2023-05-09 16:44:22,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +11: [2023-05-09 16:44:22,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +27: [2023-05-09 16:44:22,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +27: [2023-05-09 16:44:22,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +16: [2023-05-09 16:44:22,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +16: [2023-05-09 16:44:22,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 0: [2023-05-09 16:44:22,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 0: [2023-05-09 16:44:22,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 0: [2023-05-09 16:44:22,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 0: [2023-05-09 16:44:22,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +30: [2023-05-09 16:44:22,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +22: [2023-05-09 16:44:22,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +15: [2023-05-09 16:44:22,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +24: [2023-05-09 16:44:22,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +22: [2023-05-09 16:44:22,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +24: [2023-05-09 16:44:22,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +30: [2023-05-09 16:44:22,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +20: [2023-05-09 16:44:22,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +20: [2023-05-09 16:44:22,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +20: [2023-05-09 16:44:22,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +20: [2023-05-09 16:44:22,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +11: [2023-05-09 16:44:22,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +30: [2023-05-09 16:44:22,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 2: [2023-05-09 16:44:22,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +22: [2023-05-09 16:44:22,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +16: [2023-05-09 16:44:22,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +11: [2023-05-09 16:44:22,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +30: [2023-05-09 16:44:22,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +20: [2023-05-09 16:44:22,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +20: [2023-05-09 16:44:22,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +15: [2023-05-09 16:44:22,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +20: [2023-05-09 16:44:22,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +24: [2023-05-09 16:44:22,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +20: [2023-05-09 16:44:22,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +12: [2023-05-09 16:44:22,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +12: [2023-05-09 16:44:22,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +12: [2023-05-09 16:44:22,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +12: [2023-05-09 16:44:22,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +12: [2023-05-09 16:44:22,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +12: [2023-05-09 16:44:22,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +16: [2023-05-09 16:44:22,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +12: [2023-05-09 16:44:22,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +24: [2023-05-09 16:44:22,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +27: [2023-05-09 16:44:22,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +27: [2023-05-09 16:44:22,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +11: [2023-05-09 16:44:22,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +11: [2023-05-09 16:44:22,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +11: [2023-05-09 16:44:22,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +27: [2023-05-09 16:44:22,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +22: [2023-05-09 16:44:22,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +11: [2023-05-09 16:44:22,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +16: [2023-05-09 16:44:22,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +15: [2023-05-09 16:44:22,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +16: [2023-05-09 16:44:22,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +12: [2023-05-09 16:44:22,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +15: [2023-05-09 16:44:22,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +24: [2023-05-09 16:44:22,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +24: [2023-05-09 16:44:22,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +24: [2023-05-09 16:44:22,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +27: [2023-05-09 16:44:22,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +22: [2023-05-09 16:44:22,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +22: [2023-05-09 16:44:22,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +30: [2023-05-09 16:44:22,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +18: [2023-05-09 16:44:22,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +18: [2023-05-09 16:44:22,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +18: [2023-05-09 16:44:22,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +16: [2023-05-09 16:44:22,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +18: [2023-05-09 16:44:22,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 0: [2023-05-09 16:44:22,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +24: [2023-05-09 16:44:22,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +13: [2023-05-09 16:44:22,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +15: [2023-05-09 16:44:22,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +13: [2023-05-09 16:44:22,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +13: [2023-05-09 16:44:22,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +12: [2023-05-09 16:44:22,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +12: [2023-05-09 16:44:22,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +12: [2023-05-09 16:44:22,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +13: [2023-05-09 16:44:22,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 5: [2023-05-09 16:44:22,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 5: [2023-05-09 16:44:22,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 5: [2023-05-09 16:44:22,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 5: [2023-05-09 16:44:22,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 0: [2023-05-09 16:44:22,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 0: [2023-05-09 16:44:22,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +12: [2023-05-09 16:44:22,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +27: [2023-05-09 16:44:22,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +24: [2023-05-09 16:44:22,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +20: [2023-05-09 16:44:22,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +20: [2023-05-09 16:44:22,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +24: [2023-05-09 16:44:22,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +24: [2023-05-09 16:44:22,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +24: [2023-05-09 16:44:22,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 1: [2023-05-09 16:44:22,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 2: [2023-05-09 16:44:22,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +12: [2023-05-09 16:44:22,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +11: [2023-05-09 16:44:22,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +27: [2023-05-09 16:44:22,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +10: [2023-05-09 16:44:22,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +10: [2023-05-09 16:44:22,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +10: [2023-05-09 16:44:22,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +10: [2023-05-09 16:44:22,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +11: [2023-05-09 16:44:22,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +11: [2023-05-09 16:44:22,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 2: [2023-05-09 16:44:22,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +24: [2023-05-09 16:44:22,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +17: [2023-05-09 16:44:22,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +17: [2023-05-09 16:44:22,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +17: [2023-05-09 16:44:22,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +17: [2023-05-09 16:44:22,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 8: [2023-05-09 16:44:22,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +19: [2023-05-09 16:44:22,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +19: [2023-05-09 16:44:22,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +19: [2023-05-09 16:44:22,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +19: [2023-05-09 16:44:22,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 2: [2023-05-09 16:44:22,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +24: [2023-05-09 16:44:22,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +11: [2023-05-09 16:44:22,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +11: [2023-05-09 16:44:22,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +24: [2023-05-09 16:44:22,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +11: [2023-05-09 16:44:22,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +13: [2023-05-09 16:44:22,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +24: [2023-05-09 16:44:22,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +12: [2023-05-09 16:44:22,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +12: [2023-05-09 16:44:22,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +13: [2023-05-09 16:44:22,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +13: [2023-05-09 16:44:22,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 2: [2023-05-09 16:44:22,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 8: [2023-05-09 16:44:22,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +13: [2023-05-09 16:44:22,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +18: [2023-05-09 16:44:22,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +14: [2023-05-09 16:44:22,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +18: [2023-05-09 16:44:22,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +14: [2023-05-09 16:44:22,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +14: [2023-05-09 16:44:22,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +20: [2023-05-09 16:44:22,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +14: [2023-05-09 16:44:22,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +12: [2023-05-09 16:44:22,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +14: [2023-05-09 16:44:22,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +14: [2023-05-09 16:44:22,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +24: [2023-05-09 16:44:22,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +24: [2023-05-09 16:44:22,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 1: [2023-05-09 16:44:22,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +18: [2023-05-09 16:44:22,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +18: [2023-05-09 16:44:22,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 1: [2023-05-09 16:44:22,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 0: [2023-05-09 16:44:22,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +12: [2023-05-09 16:44:22,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +14: [2023-05-09 16:44:22,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +14: [2023-05-09 16:44:22,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +24: [2023-05-09 16:44:22,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +23: [2023-05-09 16:44:22,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +24: [2023-05-09 16:44:22,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +23: [2023-05-09 16:44:22,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +23: [2023-05-09 16:44:22,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +23: [2023-05-09 16:44:22,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 2: [2023-05-09 16:44:22,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +12: [2023-05-09 16:44:22,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +10: [2023-05-09 16:44:22,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +20: [2023-05-09 16:44:22,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +20: [2023-05-09 16:44:22,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +23: [2023-05-09 16:44:22,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +23: [2023-05-09 16:44:22,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +23: [2023-05-09 16:44:22,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +23: [2023-05-09 16:44:22,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +12: [2023-05-09 16:44:22,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +11: [2023-05-09 16:44:22,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +11: [2023-05-09 16:44:22,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +10: [2023-05-09 16:44:22,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +10: [2023-05-09 16:44:22,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 2: [2023-05-09 16:44:22,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +17: [2023-05-09 16:44:22,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +12: [2023-05-09 16:44:22,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +19: [2023-05-09 16:44:22,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +11: [2023-05-09 16:44:22,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +11: [2023-05-09 16:44:22,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +19: [2023-05-09 16:44:22,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +20: [2023-05-09 16:44:22,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +10: [2023-05-09 16:44:22,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +17: [2023-05-09 16:44:22,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +17: [2023-05-09 16:44:22,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +17: [2023-05-09 16:44:22,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +19: [2023-05-09 16:44:22,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +19: [2023-05-09 16:44:22,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +14: [2023-05-09 16:44:22,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +20: [2023-05-09 16:44:22,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +13: [2023-05-09 16:44:22,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 0: [2023-05-09 16:44:22,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +13: [2023-05-09 16:44:22,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +11: [2023-05-09 16:44:22,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +12: [2023-05-09 16:44:22,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +13: [2023-05-09 16:44:22,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +13: [2023-05-09 16:44:22,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +12: [2023-05-09 16:44:22,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +14: [2023-05-09 16:44:22,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +11: [2023-05-09 16:44:22,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +12: [2023-05-09 16:44:22,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +12: [2023-05-09 16:44:22,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +11: [2023-05-09 16:44:22,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +11: [2023-05-09 16:44:22,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +12: [2023-05-09 16:44:22,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +12: [2023-05-09 16:44:22,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +20: [2023-05-09 16:44:22,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +18: [2023-05-09 16:44:22,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +18: [2023-05-09 16:44:22,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +18: [2023-05-09 16:44:22,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +18: [2023-05-09 16:44:22,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +23: [2023-05-09 16:44:22,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 1: [2023-05-09 16:44:22,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +13: [2023-05-09 16:44:22,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +13: [2023-05-09 16:44:22,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +13: [2023-05-09 16:44:22,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +13: [2023-05-09 16:44:22,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +12: [2023-05-09 16:44:22,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +28: [2023-05-09 16:44:22,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +28: [2023-05-09 16:44:22,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +28: [2023-05-09 16:44:22,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +28: [2023-05-09 16:44:22,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +28: [2023-05-09 16:44:22,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +28: [2023-05-09 16:44:22,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 0: [2023-05-09 16:44:22,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +10: [2023-05-09 16:44:22,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +14: [2023-05-09 16:44:22,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +23: [2023-05-09 16:44:22,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +10: [2023-05-09 16:44:22,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +28: [2023-05-09 16:44:22,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +28: [2023-05-09 16:44:22,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +19: [2023-05-09 16:44:22,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +19: [2023-05-09 16:44:22,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +17: [2023-05-09 16:44:22,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +14: [2023-05-09 16:44:22,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +17: [2023-05-09 16:44:22,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +18: [2023-05-09 16:44:22,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +19: [2023-05-09 16:44:22,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +19: [2023-05-09 16:44:22,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +12: [2023-05-09 16:44:22,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +18: [2023-05-09 16:44:22,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +18: [2023-05-09 16:44:22,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +23: [2023-05-09 16:44:22,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +17: [2023-05-09 16:44:22,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +17: [2023-05-09 16:44:22,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +18: [2023-05-09 16:44:22,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +12: [2023-05-09 16:44:22,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +10: [2023-05-09 16:44:22,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +10: [2023-05-09 16:44:22,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +31: [2023-05-09 16:44:22,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +31: [2023-05-09 16:44:22,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +31: [2023-05-09 16:44:22,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +31: [2023-05-09 16:44:22,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +31: [2023-05-09 16:44:22,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +23: [2023-05-09 16:44:22,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +31: [2023-05-09 16:44:22,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +12: [2023-05-09 16:44:22,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +31: [2023-05-09 16:44:22,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +31: [2023-05-09 16:44:22,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +23: [2023-05-09 16:44:22,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +23: [2023-05-09 16:44:22,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +17: [2023-05-09 16:44:22,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +17: [2023-05-09 16:44:22,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +14: [2023-05-09 16:44:22,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +10: [2023-05-09 16:44:22,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +10: [2023-05-09 16:44:22,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +25: [2023-05-09 16:44:22,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +25: [2023-05-09 16:44:22,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +25: [2023-05-09 16:44:22,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +25: [2023-05-09 16:44:22,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +25: [2023-05-09 16:44:22,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +25: [2023-05-09 16:44:22,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +14: [2023-05-09 16:44:22,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +25: [2023-05-09 16:44:22,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +25: [2023-05-09 16:44:22,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +19: [2023-05-09 16:44:22,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +23: [2023-05-09 16:44:22,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +28: [2023-05-09 16:44:22,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +17: [2023-05-09 16:44:22,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +17: [2023-05-09 16:44:22,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +17: [2023-05-09 16:44:22,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +17: [2023-05-09 16:44:22,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +17: [2023-05-09 16:44:22,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +17: [2023-05-09 16:44:22,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +28: [2023-05-09 16:44:22,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +19: [2023-05-09 16:44:22,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +12: [2023-05-09 16:44:22,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +17: [2023-05-09 16:44:22,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +19: [2023-05-09 16:44:22,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +17: [2023-05-09 16:44:22,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +12: [2023-05-09 16:44:22,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +10: [2023-05-09 16:44:22,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +23: [2023-05-09 16:44:22,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +19: [2023-05-09 16:44:22,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +14: [2023-05-09 16:44:22,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +10: [2023-05-09 16:44:22,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +17: [2023-05-09 16:44:22,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +14: [2023-05-09 16:44:22,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +31: [2023-05-09 16:44:22,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +17: [2023-05-09 16:44:22,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +31: [2023-05-09 16:44:22,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +28: [2023-05-09 16:44:22,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +28: [2023-05-09 16:44:22,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +28: [2023-05-09 16:44:22,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +17: [2023-05-09 16:44:22,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +17: [2023-05-09 16:44:22,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +25: [2023-05-09 16:44:22,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +25: [2023-05-09 16:44:22,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +31: [2023-05-09 16:44:22,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +21: [2023-05-09 16:44:22,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +21: [2023-05-09 16:44:22,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +21: [2023-05-09 16:44:22,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +21: [2023-05-09 16:44:22,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +31: [2023-05-09 16:44:22,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +31: [2023-05-09 16:44:22,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +28: [2023-05-09 16:44:22,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +28: [2023-05-09 16:44:22,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +19: [2023-05-09 16:44:22,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +19: [2023-05-09 16:44:22,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +19: [2023-05-09 16:44:22,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +31: [2023-05-09 16:44:22,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +10: [2023-05-09 16:44:22,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +10: [2023-05-09 16:44:22,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +10: [2023-05-09 16:44:22,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +10: [2023-05-09 16:44:22,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +10: [2023-05-09 16:44:22,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +10: [2023-05-09 16:44:22,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +19: [2023-05-09 16:44:22,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +10: [2023-05-09 16:44:22,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +19: [2023-05-09 16:44:22,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +19: [2023-05-09 16:44:22,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +19: [2023-05-09 16:44:22,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +19: [2023-05-09 16:44:22,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +10: [2023-05-09 16:44:22,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +25: [2023-05-09 16:44:22,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +17: [2023-05-09 16:44:22,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +17: [2023-05-09 16:44:22,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +29: [2023-05-09 16:44:22,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +29: [2023-05-09 16:44:22,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +29: [2023-05-09 16:44:22,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +29: [2023-05-09 16:44:22,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +29: [2023-05-09 16:44:22,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +29: [2023-05-09 16:44:22,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +29: [2023-05-09 16:44:22,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +25: [2023-05-09 16:44:22,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +25: [2023-05-09 16:44:22,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +25: [2023-05-09 16:44:22,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +21: [2023-05-09 16:44:22,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +21: [2023-05-09 16:44:22,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +17: [2023-05-09 16:44:22,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +31: [2023-05-09 16:44:22,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +29: [2023-05-09 16:44:22,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +18: [2023-05-09 16:44:22,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +28: [2023-05-09 16:44:22,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +18: [2023-05-09 16:44:22,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +18: [2023-05-09 16:44:22,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +18: [2023-05-09 16:44:22,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +18: [2023-05-09 16:44:22,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +18: [2023-05-09 16:44:22,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +18: [2023-05-09 16:44:22,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +31: [2023-05-09 16:44:22,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +21: [2023-05-09 16:44:22,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +21: [2023-05-09 16:44:22,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +17: [2023-05-09 16:44:22,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +18: [2023-05-09 16:44:22,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +25: [2023-05-09 16:44:22,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +25: [2023-05-09 16:44:22,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +19: [2023-05-09 16:44:22,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +17: [2023-05-09 16:44:22,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +17: [2023-05-09 16:44:22,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +10: [2023-05-09 16:44:22,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +29: [2023-05-09 16:44:22,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +10: [2023-05-09 16:44:22,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +10: [2023-05-09 16:44:22,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +18: [2023-05-09 16:44:22,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +18: [2023-05-09 16:44:22,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +19: [2023-05-09 16:44:22,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +19: [2023-05-09 16:44:22,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +21: [2023-05-09 16:44:22,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +21: [2023-05-09 16:44:22,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +18: [2023-05-09 16:44:22,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +29: [2023-05-09 16:44:22,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +21: [2023-05-09 16:44:22,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 6: [2023-05-09 16:44:22,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +21: [2023-05-09 16:44:22,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 3: [2023-05-09 16:44:22,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +29: [2023-05-09 16:44:22,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +19: [2023-05-09 16:44:22,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +10: [2023-05-09 16:44:22,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +18: [2023-05-09 16:44:22,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +19: [2023-05-09 16:44:22,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +10: [2023-05-09 16:44:22,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +19: [2023-05-09 16:44:22,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +10: [2023-05-09 16:44:22,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +10: [2023-05-09 16:44:22,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +29: [2023-05-09 16:44:22,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +29: [2023-05-09 16:44:22,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +18: [2023-05-09 16:44:22,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +18: [2023-05-09 16:44:22,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +29: [2023-05-09 16:44:22,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +19: [2023-05-09 16:44:22,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +10: [2023-05-09 16:44:22,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +21: [2023-05-09 16:44:22,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +21: [2023-05-09 16:44:22,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +21: [2023-05-09 16:44:22,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +29: [2023-05-09 16:44:22,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +21: [2023-05-09 16:44:22,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 6: [2023-05-09 16:44:22,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +19: [2023-05-09 16:44:22,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +18: [2023-05-09 16:44:22,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +26: [2023-05-09 16:44:22,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +26: [2023-05-09 16:44:22,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +26: [2023-05-09 16:44:22,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +26: [2023-05-09 16:44:22,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +26: [2023-05-09 16:44:22,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +26: [2023-05-09 16:44:22,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +26: [2023-05-09 16:44:22,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +26: [2023-05-09 16:44:22,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +29: [2023-05-09 16:44:22,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +21: [2023-05-09 16:44:22,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +21: [2023-05-09 16:44:22,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +21: [2023-05-09 16:44:22,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +21: [2023-05-09 16:44:22,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +18: [2023-05-09 16:44:22,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +21: [2023-05-09 16:44:22,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +21: [2023-05-09 16:44:22,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +21: [2023-05-09 16:44:22,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +21: [2023-05-09 16:44:22,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +26: [2023-05-09 16:44:22,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +26: [2023-05-09 16:44:22,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +13: [2023-05-09 16:44:22,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +13: [2023-05-09 16:44:22,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +13: [2023-05-09 16:44:22,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +13: [2023-05-09 16:44:22,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +13: [2023-05-09 16:44:22,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +13: [2023-05-09 16:44:22,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +13: [2023-05-09 16:44:22,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +15: [2023-05-09 16:44:22,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +15: [2023-05-09 16:44:22,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +15: [2023-05-09 16:44:22,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +15: [2023-05-09 16:44:22,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +15: [2023-05-09 16:44:22,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +15: [2023-05-09 16:44:22,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +21: [2023-05-09 16:44:22,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +13: [2023-05-09 16:44:22,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +16: [2023-05-09 16:44:22,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +16: [2023-05-09 16:44:22,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +27: [2023-05-09 16:44:22,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 2: [2023-05-09 16:44:22,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 2: [2023-05-09 16:44:22,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +15: [2023-05-09 16:44:22,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 2: [2023-05-09 16:44:22,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +27: [2023-05-09 16:44:22,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +27: [2023-05-09 16:44:22,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +27: [2023-05-09 16:44:22,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +27: [2023-05-09 16:44:22,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +27: [2023-05-09 16:44:22,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +27: [2023-05-09 16:44:22,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +15: [2023-05-09 16:44:22,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +16: [2023-05-09 16:44:22,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +27: [2023-05-09 16:44:22,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +22: [2023-05-09 16:44:22,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +22: [2023-05-09 16:44:22,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +22: [2023-05-09 16:44:22,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +11: [2023-05-09 16:44:22,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +11: [2023-05-09 16:44:22,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +22: [2023-05-09 16:44:22,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +11: [2023-05-09 16:44:22,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +11: [2023-05-09 16:44:22,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +11: [2023-05-09 16:44:22,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +11: [2023-05-09 16:44:22,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +22: [2023-05-09 16:44:22,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +16: [2023-05-09 16:44:22,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +22: [2023-05-09 16:44:22,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +22: [2023-05-09 16:44:22,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +26: [2023-05-09 16:44:22,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +16: [2023-05-09 16:44:22,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +16: [2023-05-09 16:44:22,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +16: [2023-05-09 16:44:22,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +16: [2023-05-09 16:44:22,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +11: [2023-05-09 16:44:22,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +26: [2023-05-09 16:44:22,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +21: [2023-05-09 16:44:22,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +22: [2023-05-09 16:44:22,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +21: [2023-05-09 16:44:22,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +11: [2023-05-09 16:44:22,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +30: [2023-05-09 16:44:22,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +30: [2023-05-09 16:44:22,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +30: [2023-05-09 16:44:22,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +30: [2023-05-09 16:44:22,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +26: [2023-05-09 16:44:22,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +30: [2023-05-09 16:44:22,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +30: [2023-05-09 16:44:22,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +30: [2023-05-09 16:44:22,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +30: [2023-05-09 16:44:22,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +24: [2023-05-09 16:44:22,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +24: [2023-05-09 16:44:22,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +26: [2023-05-09 16:44:22,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +15: [2023-05-09 16:44:22,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +13: [2023-05-09 16:44:22,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +24: [2023-05-09 16:44:22,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +24: [2023-05-09 16:44:22,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +21: [2023-05-09 16:44:22,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +15: [2023-05-09 16:44:22,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 2: [2023-05-09 16:44:22,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +16: [2023-05-09 16:44:22,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +24: [2023-05-09 16:44:22,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +24: [2023-05-09 16:44:22,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +21: [2023-05-09 16:44:22,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +21: [2023-05-09 16:44:22,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +24: [2023-05-09 16:44:22,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +24: [2023-05-09 16:44:22,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +22: [2023-05-09 16:44:22,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +27: [2023-05-09 16:44:22,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +27: [2023-05-09 16:44:22,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +26: [2023-05-09 16:44:22,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +13: [2023-05-09 16:44:22,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +13: [2023-05-09 16:44:22,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +27: [2023-05-09 16:44:22,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +22: [2023-05-09 16:44:22,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +16: [2023-05-09 16:44:22,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +16: [2023-05-09 16:44:22,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 2: [2023-05-09 16:44:22,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +26: [2023-05-09 16:44:22,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +13: [2023-05-09 16:44:22,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +22: [2023-05-09 16:44:22,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +24: [2023-05-09 16:44:22,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +15: [2023-05-09 16:44:22,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +16: [2023-05-09 16:44:22,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +16: [2023-05-09 16:44:22,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +30: [2023-05-09 16:44:22,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +16: [2023-05-09 16:44:22,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 2: [2023-05-09 16:44:22,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +11: [2023-05-09 16:44:22,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +11: [2023-05-09 16:44:22,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +11: [2023-05-09 16:44:22,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +16: [2023-05-09 16:44:22,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 4: [2023-05-09 16:44:22,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +15: [2023-05-09 16:44:22,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +15: [2023-05-09 16:44:22,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +27: [2023-05-09 16:44:22,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +22: [2023-05-09 16:44:22,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +16: [2023-05-09 16:44:22,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +30: [2023-05-09 16:44:22,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +27: [2023-05-09 16:44:22,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +21: [2023-05-09 16:44:22,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +13: [2023-05-09 16:44:22,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +15: [2023-05-09 16:44:22,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +13: [2023-05-09 16:44:22,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +22: [2023-05-09 16:44:22,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +27: [2023-05-09 16:44:22,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +22: [2023-05-09 16:44:22,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +24: [2023-05-09 16:44:22,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +16: [2023-05-09 16:44:22,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +16: [2023-05-09 16:44:22,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +27: [2023-05-09 16:44:22,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 3: [2023-05-09 16:44:22,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 3: [2023-05-09 16:44:22,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +20: [2023-05-09 16:44:22,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +20: [2023-05-09 16:44:22,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 1: [2023-05-09 16:44:22,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +20: [2023-05-09 16:44:22,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +20: [2023-05-09 16:44:22,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +24: [2023-05-09 16:44:22,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +13: [2023-05-09 16:44:22,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +16: [2023-05-09 16:44:22,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +22: [2023-05-09 16:44:22,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +30: [2023-05-09 16:44:22,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +28: [2023-05-09 16:44:22,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +16: [2023-05-09 16:44:22,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +15: [2023-05-09 16:44:22,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +28: [2023-05-09 16:44:22,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +28: [2023-05-09 16:44:22,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +16: [2023-05-09 16:44:22,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +28: [2023-05-09 16:44:22,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 2: [2023-05-09 16:44:22,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +11: [2023-05-09 16:44:22,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +30: [2023-05-09 16:44:22,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 6: [2023-05-09 16:44:22,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 6: [2023-05-09 16:44:22,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +16: [2023-05-09 16:44:22,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +21: [2023-05-09 16:44:22,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +27: [2023-05-09 16:44:22,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +11: [2023-05-09 16:44:22,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +11: [2023-05-09 16:44:22,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +30: [2023-05-09 16:44:22,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +26: [2023-05-09 16:44:22,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +26: [2023-05-09 16:44:22,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +26: [2023-05-09 16:44:22,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +26: [2023-05-09 16:44:22,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +15: [2023-05-09 16:44:22,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +16: [2023-05-09 16:44:22,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +24: [2023-05-09 16:44:22,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +20: [2023-05-09 16:44:22,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +30: [2023-05-09 16:44:22,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +13: [2023-05-09 16:44:22,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +22: [2023-05-09 16:44:22,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +20: [2023-05-09 16:44:22,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +20: [2023-05-09 16:44:22,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +28: [2023-05-09 16:44:22,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +11: [2023-05-09 16:44:22,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +30: [2023-05-09 16:44:22,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +30: [2023-05-09 16:44:22,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 2: [2023-05-09 16:44:22,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +30: [2023-05-09 16:44:22,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +20: [2023-05-09 16:44:22,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +28: [2023-05-09 16:44:22,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +28: [2023-05-09 16:44:22,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 0: [2023-05-09 16:44:22,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 0: [2023-05-09 16:44:22,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 0: [2023-05-09 16:44:22,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 6: [2023-05-09 16:44:22,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +27: [2023-05-09 16:44:22,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +27: [2023-05-09 16:44:22,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +31: [2023-05-09 16:44:22,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +24: [2023-05-09 16:44:22,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +31: [2023-05-09 16:44:22,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +31: [2023-05-09 16:44:22,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +25: [2023-05-09 16:44:22,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +24: [2023-05-09 16:44:22,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +31: [2023-05-09 16:44:22,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 4: [2023-05-09 16:44:22,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 4: [2023-05-09 16:44:22,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 4: [2023-05-09 16:44:22,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 4: [2023-05-09 16:44:22,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +16: [2023-05-09 16:44:22,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +28: [2023-05-09 16:44:22,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +25: [2023-05-09 16:44:22,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +25: [2023-05-09 16:44:22,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 6: [2023-05-09 16:44:22,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +25: [2023-05-09 16:44:22,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +30: [2023-05-09 16:44:22,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +26: [2023-05-09 16:44:22,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +26: [2023-05-09 16:44:22,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +30: [2023-05-09 16:44:22,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 4: [2023-05-09 16:44:22,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +26: [2023-05-09 16:44:22,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +26: [2023-05-09 16:44:22,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 7: [2023-05-09 16:44:22,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 7: [2023-05-09 16:44:22,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 7: [2023-05-09 16:44:22,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +16: [2023-05-09 16:44:22,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +15: [2023-05-09 16:44:22,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +15: [2023-05-09 16:44:22,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +15: [2023-05-09 16:44:22,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +15: [2023-05-09 16:44:22,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +16: [2023-05-09 16:44:22,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +27: [2023-05-09 16:44:22,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +16: [2023-05-09 16:44:22,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +16: [2023-05-09 16:44:22,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +27: [2023-05-09 16:44:22,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +30: [2023-05-09 16:44:22,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +23: [2023-05-09 16:44:22,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 8: [2023-05-09 16:44:22,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +23: [2023-05-09 16:44:22,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +23: [2023-05-09 16:44:22,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +23: [2023-05-09 16:44:22,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +11: [2023-05-09 16:44:22,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. +27: [2023-05-09 16:44:22,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +30: [2023-05-09 16:44:22,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +30: [2023-05-09 16:44:22,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +30: [2023-05-09 16:44:22,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt. + 0: [2023-05-09 16:44:22,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +20: [2023-05-09 16:44:22,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +31: [2023-05-09 16:44:22,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +24: [2023-05-09 16:44:22,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +20: [2023-05-09 16:44:22,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +24: [2023-05-09 16:44:22,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +31: [2023-05-09 16:44:22,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +27: [2023-05-09 16:44:22,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +31: [2023-05-09 16:44:22,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +28: [2023-05-09 16:44:22,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +25: [2023-05-09 16:44:22,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +28: [2023-05-09 16:44:22,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +25: [2023-05-09 16:44:22,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +31: [2023-05-09 16:44:22,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +25: [2023-05-09 16:44:22,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 4: [2023-05-09 16:44:22,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +25: [2023-05-09 16:44:22,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +16: [2023-05-09 16:44:22,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +16: [2023-05-09 16:44:22,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +16: [2023-05-09 16:44:22,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 7: [2023-05-09 16:44:22,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +16: [2023-05-09 16:44:22,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +15: [2023-05-09 16:44:22,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +15: [2023-05-09 16:44:22,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +15: [2023-05-09 16:44:22,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +30: [2023-05-09 16:44:22,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +27: [2023-05-09 16:44:22,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +27: [2023-05-09 16:44:22,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 3: [2023-05-09 16:44:22,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 8: [2023-05-09 16:44:22,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 6: [2023-05-09 16:44:22,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +15: [2023-05-09 16:44:22,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 3: [2023-05-09 16:44:22,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 8: [2023-05-09 16:44:22,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +28: [2023-05-09 16:44:22,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +28: [2023-05-09 16:44:22,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +28: [2023-05-09 16:44:22,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +28: [2023-05-09 16:44:22,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +26: [2023-05-09 16:44:22,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +23: [2023-05-09 16:44:22,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +23: [2023-05-09 16:44:22,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +26: [2023-05-09 16:44:22,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +20: [2023-05-09 16:44:22,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +30: [2023-05-09 16:44:22,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +30: [2023-05-09 16:44:22,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +20: [2023-05-09 16:44:22,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +26: [2023-05-09 16:44:22,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +26: [2023-05-09 16:44:22,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +27: [2023-05-09 16:44:22,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +20: [2023-05-09 16:44:22,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +23: [2023-05-09 16:44:22,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +26: [2023-05-09 16:44:22,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +27: [2023-05-09 16:44:22,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +26: [2023-05-09 16:44:22,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +23: [2023-05-09 16:44:22,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt... +20: [2023-05-09 16:44:22,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +30: [2023-05-09 16:44:22,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +30: [2023-05-09 16:44:22,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +20: [2023-05-09 16:44:22,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +30: [2023-05-09 16:44:22,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +30: [2023-05-09 16:44:22,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +31: [2023-05-09 16:44:22,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +20: [2023-05-09 16:44:22,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 6: [2023-05-09 16:44:22,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +30: [2023-05-09 16:44:22,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +30: [2023-05-09 16:44:22,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 6: [2023-05-09 16:44:22,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +26: [2023-05-09 16:44:22,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +26: [2023-05-09 16:44:22,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +31: [2023-05-09 16:44:22,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +27: [2023-05-09 16:44:22,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +27: [2023-05-09 16:44:22,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +28: [2023-05-09 16:44:22,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 0: [2023-05-09 16:44:22,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +25: [2023-05-09 16:44:22,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 7: [2023-05-09 16:44:22,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 7: [2023-05-09 16:44:22,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +25: [2023-05-09 16:44:22,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 0: [2023-05-09 16:44:22,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +28: [2023-05-09 16:44:22,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 0: [2023-05-09 16:44:22,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 0: [2023-05-09 16:44:22,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +15: [2023-05-09 16:44:22,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +15: [2023-05-09 16:44:22,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +31: [2023-05-09 16:44:22,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +15: [2023-05-09 16:44:22,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +15: [2023-05-09 16:44:22,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +15: [2023-05-09 16:44:22,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +15: [2023-05-09 16:44:22,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +31: [2023-05-09 16:44:22,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +31: [2023-05-09 16:44:22,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +31: [2023-05-09 16:44:22,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +25: [2023-05-09 16:44:22,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +25: [2023-05-09 16:44:22,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +27: [2023-05-09 16:44:22,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +27: [2023-05-09 16:44:22,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 8: [2023-05-09 16:44:22,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 8: [2023-05-09 16:44:22,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 8: [2023-05-09 16:44:22,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +27: [2023-05-09 16:44:22,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +27: [2023-05-09 16:44:22,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +23: [2023-05-09 16:44:22,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +23: [2023-05-09 16:44:22,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +25: [2023-05-09 16:44:22,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +23: [2023-05-09 16:44:22,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +23: [2023-05-09 16:44:22,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 7: [2023-05-09 16:44:22,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +25: [2023-05-09 16:44:22,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +31: [2023-05-09 16:44:22,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +31: [2023-05-09 16:44:22,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +25: [2023-05-09 16:44:22,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +25: [2023-05-09 16:44:22,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +15: [2023-05-09 16:44:22,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +15: [2023-05-09 16:44:22,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 4: [2023-05-09 16:44:22,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 0: [2023-05-09 16:44:22,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 0: [2023-05-09 16:44:22,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 0: [2023-05-09 16:44:22,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +23: [2023-05-09 16:44:22,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 0: [2023-05-09 16:44:22,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 4: [2023-05-09 16:44:22,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +23: [2023-05-09 16:44:22,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 4: [2023-05-09 16:44:22,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... + 4: [2023-05-09 16:44:22,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +23: [2023-05-09 16:44:22,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt. +23: [2023-05-09 16:44:22,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt... +14: [2023-05-09 16:44:22,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +14: [2023-05-09 16:44:22,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +14: [2023-05-09 16:44:22,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +14: [2023-05-09 16:44:22,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +14: [2023-05-09 16:44:22,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +14: [2023-05-09 16:44:22,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +14: [2023-05-09 16:44:22,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +14: [2023-05-09 16:44:22,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +20: [2023-05-09 16:44:22,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +20: [2023-05-09 16:44:22,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +20: [2023-05-09 16:44:22,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +20: [2023-05-09 16:44:22,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +20: [2023-05-09 16:44:22,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +20: [2023-05-09 16:44:22,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +20: [2023-05-09 16:44:22,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +20: [2023-05-09 16:44:22,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +12: [2023-05-09 16:44:22,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +12: [2023-05-09 16:44:22,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +12: [2023-05-09 16:44:22,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +12: [2023-05-09 16:44:22,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +12: [2023-05-09 16:44:22,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +12: [2023-05-09 16:44:22,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +12: [2023-05-09 16:44:22,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +19: [2023-05-09 16:44:22,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +19: [2023-05-09 16:44:22,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +14: [2023-05-09 16:44:22,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +19: [2023-05-09 16:44:22,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +19: [2023-05-09 16:44:22,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +19: [2023-05-09 16:44:22,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +12: [2023-05-09 16:44:22,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +19: [2023-05-09 16:44:22,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +19: [2023-05-09 16:44:22,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +19: [2023-05-09 16:44:22,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +14: [2023-05-09 16:44:22,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +14: [2023-05-09 16:44:22,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +14: [2023-05-09 16:44:22,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +20: [2023-05-09 16:44:22,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +20: [2023-05-09 16:44:22,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +20: [2023-05-09 16:44:22,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +14: [2023-05-09 16:44:22,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +14: [2023-05-09 16:44:22,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +19: [2023-05-09 16:44:22,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +14: [2023-05-09 16:44:22,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +14: [2023-05-09 16:44:22,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +12: [2023-05-09 16:44:22,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +12: [2023-05-09 16:44:22,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +12: [2023-05-09 16:44:22,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +20: [2023-05-09 16:44:22,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +20: [2023-05-09 16:44:22,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +19: [2023-05-09 16:44:22,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +19: [2023-05-09 16:44:22,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +20: [2023-05-09 16:44:22,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +19: [2023-05-09 16:44:22,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +20: [2023-05-09 16:44:22,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +19: [2023-05-09 16:44:22,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +12: [2023-05-09 16:44:22,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +12: [2023-05-09 16:44:22,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +12: [2023-05-09 16:44:22,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +12: [2023-05-09 16:44:22,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +20: [2023-05-09 16:44:22,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +19: [2023-05-09 16:44:22,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +19: [2023-05-09 16:44:22,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +19: [2023-05-09 16:44:22,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +23: [2023-05-09 16:44:22,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +23: [2023-05-09 16:44:22,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +23: [2023-05-09 16:44:22,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +23: [2023-05-09 16:44:22,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 0: [2023-05-09 16:44:22,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 0: [2023-05-09 16:44:22,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +23: [2023-05-09 16:44:22,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +23: [2023-05-09 16:44:22,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +23: [2023-05-09 16:44:22,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +12: [2023-05-09 16:44:22,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 0: [2023-05-09 16:44:22,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 0: [2023-05-09 16:44:22,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +23: [2023-05-09 16:44:22,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +18: [2023-05-09 16:44:22,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +23: [2023-05-09 16:44:22,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +23: [2023-05-09 16:44:22,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +18: [2023-05-09 16:44:22,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +18: [2023-05-09 16:44:22,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +18: [2023-05-09 16:44:22,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +18: [2023-05-09 16:44:22,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +18: [2023-05-09 16:44:22,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +18: [2023-05-09 16:44:22,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +21: [2023-05-09 16:44:22,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +21: [2023-05-09 16:44:22,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +21: [2023-05-09 16:44:22,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +21: [2023-05-09 16:44:22,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +18: [2023-05-09 16:44:22,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +21: [2023-05-09 16:44:22,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +21: [2023-05-09 16:44:22,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +21: [2023-05-09 16:44:22,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +23: [2023-05-09 16:44:22,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +26: [2023-05-09 16:44:22,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +26: [2023-05-09 16:44:22,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +26: [2023-05-09 16:44:22,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +21: [2023-05-09 16:44:22,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +26: [2023-05-09 16:44:22,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +26: [2023-05-09 16:44:22,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +26: [2023-05-09 16:44:22,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +26: [2023-05-09 16:44:22,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +26: [2023-05-09 16:44:22,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 0: [2023-05-09 16:44:22,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +23: [2023-05-09 16:44:22,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +23: [2023-05-09 16:44:22,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +23: [2023-05-09 16:44:22,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +18: [2023-05-09 16:44:22,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +18: [2023-05-09 16:44:22,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +18: [2023-05-09 16:44:22,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +23: [2023-05-09 16:44:22,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 0: [2023-05-09 16:44:22,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +26: [2023-05-09 16:44:22,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +21: [2023-05-09 16:44:22,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +26: [2023-05-09 16:44:22,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +21: [2023-05-09 16:44:22,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +21: [2023-05-09 16:44:22,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +26: [2023-05-09 16:44:22,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +26: [2023-05-09 16:44:22,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +18: [2023-05-09 16:44:22,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +25: [2023-05-09 16:44:22,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +25: [2023-05-09 16:44:22,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +25: [2023-05-09 16:44:22,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +25: [2023-05-09 16:44:22,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +25: [2023-05-09 16:44:22,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +25: [2023-05-09 16:44:22,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +25: [2023-05-09 16:44:22,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +15: [2023-05-09 16:44:22,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +15: [2023-05-09 16:44:22,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +25: [2023-05-09 16:44:22,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +15: [2023-05-09 16:44:22,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +15: [2023-05-09 16:44:22,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +15: [2023-05-09 16:44:22,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +18: [2023-05-09 16:44:22,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +15: [2023-05-09 16:44:22,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +18: [2023-05-09 16:44:22,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +15: [2023-05-09 16:44:22,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +23: [2023-05-09 16:44:22,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +15: [2023-05-09 16:44:22,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +18: [2023-05-09 16:44:22,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +26: [2023-05-09 16:44:22,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +26: [2023-05-09 16:44:22,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 0: [2023-05-09 16:44:22,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +21: [2023-05-09 16:44:22,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +26: [2023-05-09 16:44:22,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +21: [2023-05-09 16:44:22,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +26: [2023-05-09 16:44:22,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +21: [2023-05-09 16:44:22,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +21: [2023-05-09 16:44:22,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +25: [2023-05-09 16:44:22,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +31: [2023-05-09 16:44:22,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +31: [2023-05-09 16:44:22,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +31: [2023-05-09 16:44:22,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +31: [2023-05-09 16:44:22,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +18: [2023-05-09 16:44:22,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +31: [2023-05-09 16:44:22,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +31: [2023-05-09 16:44:22,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +31: [2023-05-09 16:44:22,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +31: [2023-05-09 16:44:22,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +15: [2023-05-09 16:44:22,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 8: [2023-05-09 16:44:22,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +15: [2023-05-09 16:44:22,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +15: [2023-05-09 16:44:22,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +21: [2023-05-09 16:44:22,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +25: [2023-05-09 16:44:22,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +25: [2023-05-09 16:44:22,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +25: [2023-05-09 16:44:22,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +31: [2023-05-09 16:44:22,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +15: [2023-05-09 16:44:22,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +31: [2023-05-09 16:44:22,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +28: [2023-05-09 16:44:22,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +28: [2023-05-09 16:44:22,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +28: [2023-05-09 16:44:22,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +28: [2023-05-09 16:44:22,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +28: [2023-05-09 16:44:22,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +28: [2023-05-09 16:44:22,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +28: [2023-05-09 16:44:22,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +28: [2023-05-09 16:44:22,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +31: [2023-05-09 16:44:22,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +15: [2023-05-09 16:44:22,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +17: [2023-05-09 16:44:22,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +17: [2023-05-09 16:44:22,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +17: [2023-05-09 16:44:22,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +17: [2023-05-09 16:44:22,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +17: [2023-05-09 16:44:22,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +17: [2023-05-09 16:44:22,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +17: [2023-05-09 16:44:22,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +15: [2023-05-09 16:44:22,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +17: [2023-05-09 16:44:22,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +15: [2023-05-09 16:44:22,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 8: [2023-05-09 16:44:22,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +25: [2023-05-09 16:44:22,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +25: [2023-05-09 16:44:22,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +25: [2023-05-09 16:44:22,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +31: [2023-05-09 16:44:22,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +31: [2023-05-09 16:44:22,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +31: [2023-05-09 16:44:22,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +15: [2023-05-09 16:44:22,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +28: [2023-05-09 16:44:22,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +31: [2023-05-09 16:44:22,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +17: [2023-05-09 16:44:22,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +17: [2023-05-09 16:44:22,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +28: [2023-05-09 16:44:22,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +25: [2023-05-09 16:44:22,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +17: [2023-05-09 16:44:22,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +17: [2023-05-09 16:44:22,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +28: [2023-05-09 16:44:22,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +28: [2023-05-09 16:44:22,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +29: [2023-05-09 16:44:22,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +29: [2023-05-09 16:44:22,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +29: [2023-05-09 16:44:22,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +29: [2023-05-09 16:44:22,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 2: [2023-05-09 16:44:22,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 2: [2023-05-09 16:44:22,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +29: [2023-05-09 16:44:22,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +29: [2023-05-09 16:44:22,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 2: [2023-05-09 16:44:22,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +29: [2023-05-09 16:44:22,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +31: [2023-05-09 16:44:22,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +29: [2023-05-09 16:44:22,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +10: [2023-05-09 16:44:22,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +10: [2023-05-09 16:44:22,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +10: [2023-05-09 16:44:22,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +10: [2023-05-09 16:44:22,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +10: [2023-05-09 16:44:22,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +10: [2023-05-09 16:44:22,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +10: [2023-05-09 16:44:22,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +28: [2023-05-09 16:44:22,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 2: [2023-05-09 16:44:22,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +10: [2023-05-09 16:44:22,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +17: [2023-05-09 16:44:22,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +17: [2023-05-09 16:44:22,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +24: [2023-05-09 16:44:22,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +24: [2023-05-09 16:44:22,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +24: [2023-05-09 16:44:22,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +24: [2023-05-09 16:44:22,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +24: [2023-05-09 16:44:22,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +22: [2023-05-09 16:44:22,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +22: [2023-05-09 16:44:22,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +22: [2023-05-09 16:44:22,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +22: [2023-05-09 16:44:22,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +24: [2023-05-09 16:44:22,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +22: [2023-05-09 16:44:22,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +22: [2023-05-09 16:44:22,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +24: [2023-05-09 16:44:22,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +22: [2023-05-09 16:44:22,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +13: [2023-05-09 16:44:22,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +13: [2023-05-09 16:44:22,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +13: [2023-05-09 16:44:22,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +13: [2023-05-09 16:44:22,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +11: [2023-05-09 16:44:22,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +11: [2023-05-09 16:44:22,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +22: [2023-05-09 16:44:22,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +24: [2023-05-09 16:44:22,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +17: [2023-05-09 16:44:22,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +17: [2023-05-09 16:44:22,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +11: [2023-05-09 16:44:22,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +11: [2023-05-09 16:44:22,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +11: [2023-05-09 16:44:22,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +11: [2023-05-09 16:44:22,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +13: [2023-05-09 16:44:22,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +13: [2023-05-09 16:44:22,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +28: [2023-05-09 16:44:22,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +28: [2023-05-09 16:44:22,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +13: [2023-05-09 16:44:22,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +13: [2023-05-09 16:44:22,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +27: [2023-05-09 16:44:22,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +11: [2023-05-09 16:44:22,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +11: [2023-05-09 16:44:22,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +27: [2023-05-09 16:44:22,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +27: [2023-05-09 16:44:22,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 2: [2023-05-09 16:44:22,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +28: [2023-05-09 16:44:22,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +30: [2023-05-09 16:44:22,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +27: [2023-05-09 16:44:22,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +30: [2023-05-09 16:44:22,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +30: [2023-05-09 16:44:22,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +27: [2023-05-09 16:44:22,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +30: [2023-05-09 16:44:22,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +27: [2023-05-09 16:44:22,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +16: [2023-05-09 16:44:22,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +16: [2023-05-09 16:44:22,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +30: [2023-05-09 16:44:22,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +16: [2023-05-09 16:44:22,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +16: [2023-05-09 16:44:22,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +16: [2023-05-09 16:44:22,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +30: [2023-05-09 16:44:22,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +30: [2023-05-09 16:44:22,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +16: [2023-05-09 16:44:22,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +16: [2023-05-09 16:44:22,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +16: [2023-05-09 16:44:22,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +27: [2023-05-09 16:44:22,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +27: [2023-05-09 16:44:22,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +30: [2023-05-09 16:44:22,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +10: [2023-05-09 16:44:22,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 2: [2023-05-09 16:44:22,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +29: [2023-05-09 16:44:22,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +29: [2023-05-09 16:44:22,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +22: [2023-05-09 16:44:22,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +22: [2023-05-09 16:44:22,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +10: [2023-05-09 16:44:22,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +10: [2023-05-09 16:44:22,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 9: [2023-05-09 16:44:22,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +22: [2023-05-09 16:44:22,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 1: [2023-05-09 16:44:22,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 1: [2023-05-09 16:44:22,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +10: [2023-05-09 16:44:22,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +27: [2023-05-09 16:44:22,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +30: [2023-05-09 16:44:22,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +29: [2023-05-09 16:44:22,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +30: [2023-05-09 16:44:22,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 5: [2023-05-09 16:44:22,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +16: [2023-05-09 16:44:22,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +16: [2023-05-09 16:44:22,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +13: [2023-05-09 16:44:22,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +13: [2023-05-09 16:44:22,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +27: [2023-05-09 16:44:22,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +24: [2023-05-09 16:44:22,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +24: [2023-05-09 16:44:22,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +24: [2023-05-09 16:44:22,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +11: [2023-05-09 16:44:22,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +29: [2023-05-09 16:44:22,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 3: [2023-05-09 16:44:22,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +30: [2023-05-09 16:44:22,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 5: [2023-05-09 16:44:22,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +29: [2023-05-09 16:44:22,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +16: [2023-05-09 16:44:22,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +29: [2023-05-09 16:44:22,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +22: [2023-05-09 16:44:22,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +16: [2023-05-09 16:44:22,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +11: [2023-05-09 16:44:22,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 3: [2023-05-09 16:44:22,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +13: [2023-05-09 16:44:22,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +22: [2023-05-09 16:44:22,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 4: [2023-05-09 16:44:22,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +10: [2023-05-09 16:44:22,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +10: [2023-05-09 16:44:22,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 9: [2023-05-09 16:44:22,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 4: [2023-05-09 16:44:22,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +10: [2023-05-09 16:44:22,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +30: [2023-05-09 16:44:22,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +30: [2023-05-09 16:44:22,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +16: [2023-05-09 16:44:22,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +22: [2023-05-09 16:44:22,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +13: [2023-05-09 16:44:22,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +27: [2023-05-09 16:44:22,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +27: [2023-05-09 16:44:22,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +24: [2023-05-09 16:44:22,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +22: [2023-05-09 16:44:22,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +24: [2023-05-09 16:44:22,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +13: [2023-05-09 16:44:22,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +30: [2023-05-09 16:44:22,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +27: [2023-05-09 16:44:22,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +29: [2023-05-09 16:44:22,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +24: [2023-05-09 16:44:22,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +24: [2023-05-09 16:44:22,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +29: [2023-05-09 16:44:22,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +11: [2023-05-09 16:44:22,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +11: [2023-05-09 16:44:22,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +16: [2023-05-09 16:44:22,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +27: [2023-05-09 16:44:22,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +13: [2023-05-09 16:44:22,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +30: [2023-05-09 16:44:22,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +10: [2023-05-09 16:44:22,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +16: [2023-05-09 16:44:22,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 2: [2023-05-09 16:44:22,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +16: [2023-05-09 16:44:22,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +11: [2023-05-09 16:44:22,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +22: [2023-05-09 16:44:22,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +24: [2023-05-09 16:44:22,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +27: [2023-05-09 16:44:22,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +13: [2023-05-09 16:44:22,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +11: [2023-05-09 16:44:22,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +27: [2023-05-09 16:44:22,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +30: [2023-05-09 16:44:22,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +11: [2023-05-09 16:44:22,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +13: [2023-05-09 16:44:22,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +11: [2023-05-09 16:44:22,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +14: [2023-05-09 16:44:22,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +14: [2023-05-09 16:44:22,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +14: [2023-05-09 16:44:22,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +14: [2023-05-09 16:44:22,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +14: [2023-05-09 16:44:22,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +14: [2023-05-09 16:44:22,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +14: [2023-05-09 16:44:22,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +14: [2023-05-09 16:44:22,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +14: [2023-05-09 16:44:22,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +14: [2023-05-09 16:44:22,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +14: [2023-05-09 16:44:22,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +14: [2023-05-09 16:44:22,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +14: [2023-05-09 16:44:22,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +14: [2023-05-09 16:44:22,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +14: [2023-05-09 16:44:22,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +14: [2023-05-09 16:44:22,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +19: [2023-05-09 16:44:22,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +19: [2023-05-09 16:44:22,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +19: [2023-05-09 16:44:22,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +19: [2023-05-09 16:44:22,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +19: [2023-05-09 16:44:22,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +19: [2023-05-09 16:44:22,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +19: [2023-05-09 16:44:22,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +19: [2023-05-09 16:44:22,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +12: [2023-05-09 16:44:22,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +12: [2023-05-09 16:44:22,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +12: [2023-05-09 16:44:22,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +12: [2023-05-09 16:44:22,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +12: [2023-05-09 16:44:22,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +12: [2023-05-09 16:44:22,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +12: [2023-05-09 16:44:22,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +12: [2023-05-09 16:44:22,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +19: [2023-05-09 16:44:22,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +12: [2023-05-09 16:44:22,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +19: [2023-05-09 16:44:22,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +19: [2023-05-09 16:44:22,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +12: [2023-05-09 16:44:22,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +12: [2023-05-09 16:44:22,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +12: [2023-05-09 16:44:22,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +12: [2023-05-09 16:44:22,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +19: [2023-05-09 16:44:22,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +19: [2023-05-09 16:44:22,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +19: [2023-05-09 16:44:22,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +12: [2023-05-09 16:44:22,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +12: [2023-05-09 16:44:22,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +12: [2023-05-09 16:44:22,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +19: [2023-05-09 16:44:22,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +19: [2023-05-09 16:44:22,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 7: [2023-05-09 16:44:22,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 6: [2023-05-09 16:44:22,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 7: [2023-05-09 16:44:22,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 6: [2023-05-09 16:44:22,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +20: [2023-05-09 16:44:22,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +20: [2023-05-09 16:44:22,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +20: [2023-05-09 16:44:22,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +20: [2023-05-09 16:44:22,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +20: [2023-05-09 16:44:22,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +20: [2023-05-09 16:44:22,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +20: [2023-05-09 16:44:22,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +20: [2023-05-09 16:44:23,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +20: [2023-05-09 16:44:23,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +20: [2023-05-09 16:44:23,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +20: [2023-05-09 16:44:23,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +20: [2023-05-09 16:44:23,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +20: [2023-05-09 16:44:23,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +20: [2023-05-09 16:44:23,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +23: [2023-05-09 16:44:23,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +23: [2023-05-09 16:44:23,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +23: [2023-05-09 16:44:23,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +23: [2023-05-09 16:44:23,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +23: [2023-05-09 16:44:23,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +23: [2023-05-09 16:44:23,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +23: [2023-05-09 16:44:23,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +23: [2023-05-09 16:44:23,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +20: [2023-05-09 16:44:23,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +20: [2023-05-09 16:44:23,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +23: [2023-05-09 16:44:23,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +23: [2023-05-09 16:44:23,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +23: [2023-05-09 16:44:23,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +28: [2023-05-09 16:44:23,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +28: [2023-05-09 16:44:23,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +28: [2023-05-09 16:44:23,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +28: [2023-05-09 16:44:23,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +28: [2023-05-09 16:44:23,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +28: [2023-05-09 16:44:23,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +28: [2023-05-09 16:44:23,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +28: [2023-05-09 16:44:23,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +23: [2023-05-09 16:44:23,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +23: [2023-05-09 16:44:23,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +23: [2023-05-09 16:44:23,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +29: [2023-05-09 16:44:23,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +29: [2023-05-09 16:44:23,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +23: [2023-05-09 16:44:23,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +29: [2023-05-09 16:44:23,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +29: [2023-05-09 16:44:23,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +29: [2023-05-09 16:44:23,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +29: [2023-05-09 16:44:23,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +29: [2023-05-09 16:44:23,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +29: [2023-05-09 16:44:23,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +23: [2023-05-09 16:44:23,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +28: [2023-05-09 16:44:23,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +28: [2023-05-09 16:44:23,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +29: [2023-05-09 16:44:23,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +29: [2023-05-09 16:44:23,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +28: [2023-05-09 16:44:23,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +29: [2023-05-09 16:44:23,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +28: [2023-05-09 16:44:23,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +28: [2023-05-09 16:44:23,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +28: [2023-05-09 16:44:23,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +29: [2023-05-09 16:44:23,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +29: [2023-05-09 16:44:23,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +29: [2023-05-09 16:44:23,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +28: [2023-05-09 16:44:23,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +29: [2023-05-09 16:44:23,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +28: [2023-05-09 16:44:23,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +29: [2023-05-09 16:44:23,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +10: [2023-05-09 16:44:23,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +10: [2023-05-09 16:44:23,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +10: [2023-05-09 16:44:23,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +10: [2023-05-09 16:44:23,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +10: [2023-05-09 16:44:23,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +10: [2023-05-09 16:44:23,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +10: [2023-05-09 16:44:23,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +10: [2023-05-09 16:44:23,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +21: [2023-05-09 16:44:23,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +21: [2023-05-09 16:44:23,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +21: [2023-05-09 16:44:23,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +21: [2023-05-09 16:44:23,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +21: [2023-05-09 16:44:23,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +21: [2023-05-09 16:44:23,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +14: [2023-05-09 16:44:23,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +14: [2023-05-09 16:44:23,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +14: [2023-05-09 16:44:23,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +21: [2023-05-09 16:44:23,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +14: [2023-05-09 16:44:23,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +14: [2023-05-09 16:44:23,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +14: [2023-05-09 16:44:23,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +14: [2023-05-09 16:44:23,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +21: [2023-05-09 16:44:23,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +14: [2023-05-09 16:44:23,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +15: [2023-05-09 16:44:23,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +15: [2023-05-09 16:44:23,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +15: [2023-05-09 16:44:23,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +10: [2023-05-09 16:44:23,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +15: [2023-05-09 16:44:23,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +15: [2023-05-09 16:44:23,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +15: [2023-05-09 16:44:23,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +25: [2023-05-09 16:44:23,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +25: [2023-05-09 16:44:23,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +25: [2023-05-09 16:44:23,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +25: [2023-05-09 16:44:23,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +25: [2023-05-09 16:44:23,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +25: [2023-05-09 16:44:23,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +25: [2023-05-09 16:44:23,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +21: [2023-05-09 16:44:23,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +10: [2023-05-09 16:44:23,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +10: [2023-05-09 16:44:23,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +15: [2023-05-09 16:44:23,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +15: [2023-05-09 16:44:23,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +25: [2023-05-09 16:44:23,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +18: [2023-05-09 16:44:23,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +18: [2023-05-09 16:44:23,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +18: [2023-05-09 16:44:23,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +14: [2023-05-09 16:44:23,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +18: [2023-05-09 16:44:23,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +18: [2023-05-09 16:44:23,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +18: [2023-05-09 16:44:23,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +18: [2023-05-09 16:44:23,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +26: [2023-05-09 16:44:23,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +26: [2023-05-09 16:44:23,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +26: [2023-05-09 16:44:23,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +26: [2023-05-09 16:44:23,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +26: [2023-05-09 16:44:23,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +26: [2023-05-09 16:44:23,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +26: [2023-05-09 16:44:23,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +14: [2023-05-09 16:44:23,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +18: [2023-05-09 16:44:23,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +21: [2023-05-09 16:44:23,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +26: [2023-05-09 16:44:23,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +10: [2023-05-09 16:44:23,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +14: [2023-05-09 16:44:23,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +10: [2023-05-09 16:44:23,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +10: [2023-05-09 16:44:23,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +10: [2023-05-09 16:44:23,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +21: [2023-05-09 16:44:23,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +14: [2023-05-09 16:44:23,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +15: [2023-05-09 16:44:23,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +14: [2023-05-09 16:44:23,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +18: [2023-05-09 16:44:23,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +21: [2023-05-09 16:44:23,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +10: [2023-05-09 16:44:23,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +18: [2023-05-09 16:44:23,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +14: [2023-05-09 16:44:23,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +26: [2023-05-09 16:44:23,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +25: [2023-05-09 16:44:23,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +25: [2023-05-09 16:44:23,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +25: [2023-05-09 16:44:23,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +26: [2023-05-09 16:44:23,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +26: [2023-05-09 16:44:23,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +15: [2023-05-09 16:44:23,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +21: [2023-05-09 16:44:23,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +27: [2023-05-09 16:44:23,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +27: [2023-05-09 16:44:23,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +27: [2023-05-09 16:44:23,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +27: [2023-05-09 16:44:23,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +15: [2023-05-09 16:44:23,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +25: [2023-05-09 16:44:23,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +27: [2023-05-09 16:44:23,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +27: [2023-05-09 16:44:23,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +15: [2023-05-09 16:44:23,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +27: [2023-05-09 16:44:23,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +14: [2023-05-09 16:44:23,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +18: [2023-05-09 16:44:23,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +27: [2023-05-09 16:44:23,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +18: [2023-05-09 16:44:23,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +21: [2023-05-09 16:44:23,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +18: [2023-05-09 16:44:23,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +25: [2023-05-09 16:44:23,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +25: [2023-05-09 16:44:23,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +25: [2023-05-09 16:44:23,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +26: [2023-05-09 16:44:23,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +26: [2023-05-09 16:44:23,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +21: [2023-05-09 16:44:23,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +15: [2023-05-09 16:44:23,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +15: [2023-05-09 16:44:23,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +26: [2023-05-09 16:44:23,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +26: [2023-05-09 16:44:23,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +15: [2023-05-09 16:44:23,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +25: [2023-05-09 16:44:23,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +18: [2023-05-09 16:44:23,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +27: [2023-05-09 16:44:23,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +14: [2023-05-09 16:44:23,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +27: [2023-05-09 16:44:23,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +27: [2023-05-09 16:44:23,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +18: [2023-05-09 16:44:23,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +21: [2023-05-09 16:44:23,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +15: [2023-05-09 16:44:23,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +26: [2023-05-09 16:44:23,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +18: [2023-05-09 16:44:23,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +27: [2023-05-09 16:44:23,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +27: [2023-05-09 16:44:23,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +27: [2023-05-09 16:44:23,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +27: [2023-05-09 16:44:23,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +27: [2023-05-09 16:44:23,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +13: [2023-05-09 16:44:23,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +13: [2023-05-09 16:44:23,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +13: [2023-05-09 16:44:23,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +13: [2023-05-09 16:44:23,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +13: [2023-05-09 16:44:23,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +13: [2023-05-09 16:44:23,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +16: [2023-05-09 16:44:23,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +13: [2023-05-09 16:44:23,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +16: [2023-05-09 16:44:23,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +16: [2023-05-09 16:44:23,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +13: [2023-05-09 16:44:23,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +16: [2023-05-09 16:44:23,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +24: [2023-05-09 16:44:23,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +24: [2023-05-09 16:44:23,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +24: [2023-05-09 16:44:23,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +16: [2023-05-09 16:44:23,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +16: [2023-05-09 16:44:23,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +24: [2023-05-09 16:44:23,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +16: [2023-05-09 16:44:23,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +24: [2023-05-09 16:44:23,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +24: [2023-05-09 16:44:23,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +30: [2023-05-09 16:44:23,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +30: [2023-05-09 16:44:23,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +30: [2023-05-09 16:44:23,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +30: [2023-05-09 16:44:23,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +30: [2023-05-09 16:44:23,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +30: [2023-05-09 16:44:23,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +30: [2023-05-09 16:44:23,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +16: [2023-05-09 16:44:23,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +24: [2023-05-09 16:44:23,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +24: [2023-05-09 16:44:23,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +30: [2023-05-09 16:44:23,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +11: [2023-05-09 16:44:23,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +11: [2023-05-09 16:44:23,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +11: [2023-05-09 16:44:23,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +11: [2023-05-09 16:44:23,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +11: [2023-05-09 16:44:23,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +11: [2023-05-09 16:44:23,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +22: [2023-05-09 16:44:23,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +17: [2023-05-09 16:44:23,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +17: [2023-05-09 16:44:23,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +22: [2023-05-09 16:44:23,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +11: [2023-05-09 16:44:23,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +22: [2023-05-09 16:44:23,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +22: [2023-05-09 16:44:23,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +22: [2023-05-09 16:44:23,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +22: [2023-05-09 16:44:23,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +22: [2023-05-09 16:44:23,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +17: [2023-05-09 16:44:23,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +22: [2023-05-09 16:44:23,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +11: [2023-05-09 16:44:23,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +17: [2023-05-09 16:44:23,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +17: [2023-05-09 16:44:23,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +17: [2023-05-09 16:44:23,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +17: [2023-05-09 16:44:23,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +17: [2023-05-09 16:44:23,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +16: [2023-05-09 16:44:23,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +24: [2023-05-09 16:44:23,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +16: [2023-05-09 16:44:23,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +30: [2023-05-09 16:44:23,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +31: [2023-05-09 16:44:23,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +30: [2023-05-09 16:44:23,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +31: [2023-05-09 16:44:23,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +31: [2023-05-09 16:44:23,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +31: [2023-05-09 16:44:23,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +31: [2023-05-09 16:44:23,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +31: [2023-05-09 16:44:23,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +31: [2023-05-09 16:44:23,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +13: [2023-05-09 16:44:23,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +13: [2023-05-09 16:44:23,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +22: [2023-05-09 16:44:23,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +31: [2023-05-09 16:44:23,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +30: [2023-05-09 16:44:23,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +17: [2023-05-09 16:44:23,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +22: [2023-05-09 16:44:23,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +24: [2023-05-09 16:44:23,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +22: [2023-05-09 16:44:23,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +24: [2023-05-09 16:44:23,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +30: [2023-05-09 16:44:23,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +13: [2023-05-09 16:44:23,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +16: [2023-05-09 16:44:23,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +11: [2023-05-09 16:44:23,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +11: [2023-05-09 16:44:23,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +11: [2023-05-09 16:44:23,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +16: [2023-05-09 16:44:23,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +13: [2023-05-09 16:44:23,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +30: [2023-05-09 16:44:23,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +17: [2023-05-09 16:44:23,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +31: [2023-05-09 16:44:23,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +13: [2023-05-09 16:44:23,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +22: [2023-05-09 16:44:23,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +22: [2023-05-09 16:44:23,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +30: [2023-05-09 16:44:23,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +16: [2023-05-09 16:44:23,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +24: [2023-05-09 16:44:23,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +30: [2023-05-09 16:44:23,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +31: [2023-05-09 16:44:23,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +31: [2023-05-09 16:44:23,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +16: [2023-05-09 16:44:23,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +22: [2023-05-09 16:44:23,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +17: [2023-05-09 16:44:23,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +17: [2023-05-09 16:44:23,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +13: [2023-05-09 16:44:23,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +13: [2023-05-09 16:44:23,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +16: [2023-05-09 16:44:23,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +24: [2023-05-09 16:44:23,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +31: [2023-05-09 16:44:23,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +11: [2023-05-09 16:44:23,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +17: [2023-05-09 16:44:23,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +11: [2023-05-09 16:44:23,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +11: [2023-05-09 16:44:23,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +24: [2023-05-09 16:44:23,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +22: [2023-05-09 16:44:23,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +31: [2023-05-09 16:44:23,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +31: [2023-05-09 16:44:23,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +31: [2023-05-09 16:44:23,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +30: [2023-05-09 16:44:23,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +16: [2023-05-09 16:44:23,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +17: [2023-05-09 16:44:23,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +13: [2023-05-09 16:44:23,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +17: [2023-05-09 16:44:23,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +24: [2023-05-09 16:44:23,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +12: [2023-05-09 16:44:23,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +12: [2023-05-09 16:44:23,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +12: [2023-05-09 16:44:23,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +12: [2023-05-09 16:44:23,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +12: [2023-05-09 16:44:23,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +12: [2023-05-09 16:44:23,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +11: [2023-05-09 16:44:23,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +12: [2023-05-09 16:44:23,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +12: [2023-05-09 16:44:23,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +17: [2023-05-09 16:44:23,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +22: [2023-05-09 16:44:23,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +31: [2023-05-09 16:44:23,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +24: [2023-05-09 16:44:23,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +19: [2023-05-09 16:44:23,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +19: [2023-05-09 16:44:23,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +19: [2023-05-09 16:44:23,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +19: [2023-05-09 16:44:23,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +19: [2023-05-09 16:44:23,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +19: [2023-05-09 16:44:23,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +19: [2023-05-09 16:44:23,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +19: [2023-05-09 16:44:23,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +20: [2023-05-09 16:44:23,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +20: [2023-05-09 16:44:23,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +20: [2023-05-09 16:44:23,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +20: [2023-05-09 16:44:23,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +20: [2023-05-09 16:44:23,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +20: [2023-05-09 16:44:23,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +20: [2023-05-09 16:44:23,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +20: [2023-05-09 16:44:23,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +29: [2023-05-09 16:44:23,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +29: [2023-05-09 16:44:23,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +29: [2023-05-09 16:44:23,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +29: [2023-05-09 16:44:23,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +29: [2023-05-09 16:44:23,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +29: [2023-05-09 16:44:23,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +29: [2023-05-09 16:44:23,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +11: [2023-05-09 16:44:23,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +29: [2023-05-09 16:44:23,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +12: [2023-05-09 16:44:23,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +19: [2023-05-09 16:44:23,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +20: [2023-05-09 16:44:23,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +20: [2023-05-09 16:44:23,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +12: [2023-05-09 16:44:23,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +12: [2023-05-09 16:44:23,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +29: [2023-05-09 16:44:23,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +29: [2023-05-09 16:44:23,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +29: [2023-05-09 16:44:23,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +19: [2023-05-09 16:44:23,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +19: [2023-05-09 16:44:23,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +19: [2023-05-09 16:44:23,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +20: [2023-05-09 16:44:23,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +12: [2023-05-09 16:44:23,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:23,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +19: [2023-05-09 16:44:23,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +20: [2023-05-09 16:44:23,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +20: [2023-05-09 16:44:23,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +29: [2023-05-09 16:44:23,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:23,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:23,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +20: [2023-05-09 16:44:23,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +20: [2023-05-09 16:44:23,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +19: [2023-05-09 16:44:23,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +29: [2023-05-09 16:44:23,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +29: [2023-05-09 16:44:23,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +29: [2023-05-09 16:44:23,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +19: [2023-05-09 16:44:23,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:23,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +19: [2023-05-09 16:44:23,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +20: [2023-05-09 16:44:23,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +29: [2023-05-09 16:44:23,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +23: [2023-05-09 16:44:23,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +23: [2023-05-09 16:44:23,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +23: [2023-05-09 16:44:23,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +23: [2023-05-09 16:44:23,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +23: [2023-05-09 16:44:23,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +23: [2023-05-09 16:44:23,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +23: [2023-05-09 16:44:23,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +23: [2023-05-09 16:44:23,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +23: [2023-05-09 16:44:23,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +23: [2023-05-09 16:44:23,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +23: [2023-05-09 16:44:23,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +23: [2023-05-09 16:44:23,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +23: [2023-05-09 16:44:23,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +23: [2023-05-09 16:44:23,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +23: [2023-05-09 16:44:23,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +23: [2023-05-09 16:44:23,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +28: [2023-05-09 16:44:23,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +28: [2023-05-09 16:44:23,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +28: [2023-05-09 16:44:23,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +28: [2023-05-09 16:44:23,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +28: [2023-05-09 16:44:23,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +28: [2023-05-09 16:44:23,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +28: [2023-05-09 16:44:23,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +28: [2023-05-09 16:44:23,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +28: [2023-05-09 16:44:23,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +26: [2023-05-09 16:44:23,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +26: [2023-05-09 16:44:23,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +26: [2023-05-09 16:44:23,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +26: [2023-05-09 16:44:23,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +26: [2023-05-09 16:44:23,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +26: [2023-05-09 16:44:23,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +26: [2023-05-09 16:44:23,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +26: [2023-05-09 16:44:23,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +28: [2023-05-09 16:44:23,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +10: [2023-05-09 16:44:23,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +10: [2023-05-09 16:44:23,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +10: [2023-05-09 16:44:23,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +10: [2023-05-09 16:44:23,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +10: [2023-05-09 16:44:23,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +10: [2023-05-09 16:44:23,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +10: [2023-05-09 16:44:23,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +10: [2023-05-09 16:44:23,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +27: [2023-05-09 16:44:23,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +21: [2023-05-09 16:44:23,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +27: [2023-05-09 16:44:23,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +27: [2023-05-09 16:44:23,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +21: [2023-05-09 16:44:23,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +21: [2023-05-09 16:44:23,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +27: [2023-05-09 16:44:23,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +21: [2023-05-09 16:44:23,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +27: [2023-05-09 16:44:23,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +24: [2023-05-09 16:44:23,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +24: [2023-05-09 16:44:23,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +27: [2023-05-09 16:44:23,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +24: [2023-05-09 16:44:23,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +24: [2023-05-09 16:44:23,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +24: [2023-05-09 16:44:23,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +24: [2023-05-09 16:44:23,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +24: [2023-05-09 16:44:23,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +21: [2023-05-09 16:44:23,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +21: [2023-05-09 16:44:23,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +25: [2023-05-09 16:44:23,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +25: [2023-05-09 16:44:23,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +25: [2023-05-09 16:44:23,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +25: [2023-05-09 16:44:23,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +25: [2023-05-09 16:44:23,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +25: [2023-05-09 16:44:23,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +25: [2023-05-09 16:44:23,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +25: [2023-05-09 16:44:23,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +27: [2023-05-09 16:44:23,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +27: [2023-05-09 16:44:23,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +24: [2023-05-09 16:44:23,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +21: [2023-05-09 16:44:23,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +21: [2023-05-09 16:44:23,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +28: [2023-05-09 16:44:23,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +16: [2023-05-09 16:44:23,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +16: [2023-05-09 16:44:23,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +16: [2023-05-09 16:44:23,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +16: [2023-05-09 16:44:23,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +16: [2023-05-09 16:44:23,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +16: [2023-05-09 16:44:23,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +16: [2023-05-09 16:44:23,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +16: [2023-05-09 16:44:23,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +15: [2023-05-09 16:44:23,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +17: [2023-05-09 16:44:23,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +17: [2023-05-09 16:44:23,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +17: [2023-05-09 16:44:23,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +17: [2023-05-09 16:44:23,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +17: [2023-05-09 16:44:23,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +17: [2023-05-09 16:44:23,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +17: [2023-05-09 16:44:23,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +15: [2023-05-09 16:44:23,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +15: [2023-05-09 16:44:23,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +15: [2023-05-09 16:44:23,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +15: [2023-05-09 16:44:23,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +15: [2023-05-09 16:44:23,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +15: [2023-05-09 16:44:23,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +17: [2023-05-09 16:44:23,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +15: [2023-05-09 16:44:23,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +28: [2023-05-09 16:44:23,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +13: [2023-05-09 16:44:23,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +13: [2023-05-09 16:44:23,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +13: [2023-05-09 16:44:23,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +13: [2023-05-09 16:44:23,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +13: [2023-05-09 16:44:23,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +13: [2023-05-09 16:44:23,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +14: [2023-05-09 16:44:23,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 0: [2023-05-09 16:44:23,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +10: [2023-05-09 16:44:23,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +14: [2023-05-09 16:44:23,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +14: [2023-05-09 16:44:23,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +14: [2023-05-09 16:44:23,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +13: [2023-05-09 16:44:23,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +10: [2023-05-09 16:44:23,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +28: [2023-05-09 16:44:23,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +10: [2023-05-09 16:44:23,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 7: [2023-05-09 16:44:23,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +29: [2023-05-09 16:44:23,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +29: [2023-05-09 16:44:23,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +11: [2023-05-09 16:44:23,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +29: [2023-05-09 16:44:23,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 2: [2023-05-09 16:44:23,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +11: [2023-05-09 16:44:23,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +10: [2023-05-09 16:44:23,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +29: [2023-05-09 16:44:23,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 1: [2023-05-09 16:44:23,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 1: [2023-05-09 16:44:23,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 1: [2023-05-09 16:44:23,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 9: [2023-05-09 16:44:23,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 2: [2023-05-09 16:44:23,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 2: [2023-05-09 16:44:23,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 9: [2023-05-09 16:44:23,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 9: [2023-05-09 16:44:23,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +28: [2023-05-09 16:44:23,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 2: [2023-05-09 16:44:23,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 8: [2023-05-09 16:44:23,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +26: [2023-05-09 16:44:23,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +26: [2023-05-09 16:44:23,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +11: [2023-05-09 16:44:23,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 9: [2023-05-09 16:44:23,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +10: [2023-05-09 16:44:23,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +11: [2023-05-09 16:44:23,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +26: [2023-05-09 16:44:23,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +26: [2023-05-09 16:44:23,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +13: [2023-05-09 16:44:23,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +25: [2023-05-09 16:44:23,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +27: [2023-05-09 16:44:23,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +22: [2023-05-09 16:44:23,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +22: [2023-05-09 16:44:23,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +22: [2023-05-09 16:44:23,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +22: [2023-05-09 16:44:23,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +22: [2023-05-09 16:44:23,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +22: [2023-05-09 16:44:23,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +22: [2023-05-09 16:44:23,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +22: [2023-05-09 16:44:23,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +22: [2023-05-09 16:44:23,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +22: [2023-05-09 16:44:23,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +22: [2023-05-09 16:44:23,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +22: [2023-05-09 16:44:23,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 7: [2023-05-09 16:44:23,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +16: [2023-05-09 16:44:23,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +15: [2023-05-09 16:44:23,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +15: [2023-05-09 16:44:23,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +14: [2023-05-09 16:44:23,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +21: [2023-05-09 16:44:23,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +17: [2023-05-09 16:44:23,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +21: [2023-05-09 16:44:23,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +16: [2023-05-09 16:44:23,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +14: [2023-05-09 16:44:23,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +27: [2023-05-09 16:44:23,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +17: [2023-05-09 16:44:23,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +15: [2023-05-09 16:44:23,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +11: [2023-05-09 16:44:23,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +14: [2023-05-09 16:44:23,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +16: [2023-05-09 16:44:23,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +13: [2023-05-09 16:44:23,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +11: [2023-05-09 16:44:23,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +10: [2023-05-09 16:44:23,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +29: [2023-05-09 16:44:23,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +26: [2023-05-09 16:44:23,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +29: [2023-05-09 16:44:23,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +24: [2023-05-09 16:44:23,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +24: [2023-05-09 16:44:23,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +15: [2023-05-09 16:44:23,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +29: [2023-05-09 16:44:23,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +26: [2023-05-09 16:44:23,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +26: [2023-05-09 16:44:23,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +29: [2023-05-09 16:44:23,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +17: [2023-05-09 16:44:23,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +17: [2023-05-09 16:44:23,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +26: [2023-05-09 16:44:23,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +28: [2023-05-09 16:44:23,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +14: [2023-05-09 16:44:23,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +25: [2023-05-09 16:44:23,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +25: [2023-05-09 16:44:23,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +25: [2023-05-09 16:44:23,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +28: [2023-05-09 16:44:23,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +11: [2023-05-09 16:44:23,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +11: [2023-05-09 16:44:23,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +10: [2023-05-09 16:44:23,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +25: [2023-05-09 16:44:23,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +27: [2023-05-09 16:44:23,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +10: [2023-05-09 16:44:23,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +10: [2023-05-09 16:44:23,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +10: [2023-05-09 16:44:23,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +20: [2023-05-09 16:44:23,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +20: [2023-05-09 16:44:23,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +20: [2023-05-09 16:44:23,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +20: [2023-05-09 16:44:23,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +20: [2023-05-09 16:44:23,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +20: [2023-05-09 16:44:23,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +20: [2023-05-09 16:44:23,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +20: [2023-05-09 16:44:23,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +10: [2023-05-09 16:44:23,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +10: [2023-05-09 16:44:23,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +13: [2023-05-09 16:44:23,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +13: [2023-05-09 16:44:23,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +24: [2023-05-09 16:44:23,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +22: [2023-05-09 16:44:23,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +16: [2023-05-09 16:44:23,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +16: [2023-05-09 16:44:23,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +22: [2023-05-09 16:44:23,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +15: [2023-05-09 16:44:23,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +27: [2023-05-09 16:44:23,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +21: [2023-05-09 16:44:23,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +21: [2023-05-09 16:44:23,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +24: [2023-05-09 16:44:23,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +22: [2023-05-09 16:44:23,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +22: [2023-05-09 16:44:23,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +27: [2023-05-09 16:44:23,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +15: [2023-05-09 16:44:23,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +16: [2023-05-09 16:44:23,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +22: [2023-05-09 16:44:23,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +27: [2023-05-09 16:44:23,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +16: [2023-05-09 16:44:23,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +17: [2023-05-09 16:44:23,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +24: [2023-05-09 16:44:23,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +21: [2023-05-09 16:44:23,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +17: [2023-05-09 16:44:23,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +13: [2023-05-09 16:44:23,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +14: [2023-05-09 16:44:23,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +25: [2023-05-09 16:44:23,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +14: [2023-05-09 16:44:23,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +17: [2023-05-09 16:44:23,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +15: [2023-05-09 16:44:23,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +17: [2023-05-09 16:44:23,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +25: [2023-05-09 16:44:23,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +15: [2023-05-09 16:44:23,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +25: [2023-05-09 16:44:23,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +11: [2023-05-09 16:44:23,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +24: [2023-05-09 16:44:23,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +29: [2023-05-09 16:44:23,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +29: [2023-05-09 16:44:23,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +29: [2023-05-09 16:44:23,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +10: [2023-05-09 16:44:23,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +29: [2023-05-09 16:44:23,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +29: [2023-05-09 16:44:23,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +29: [2023-05-09 16:44:23,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +29: [2023-05-09 16:44:23,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +29: [2023-05-09 16:44:23,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +29: [2023-05-09 16:44:23,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +21: [2023-05-09 16:44:23,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +11: [2023-05-09 16:44:23,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +22: [2023-05-09 16:44:23,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +14: [2023-05-09 16:44:23,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +14: [2023-05-09 16:44:23,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +14: [2023-05-09 16:44:23,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +14: [2023-05-09 16:44:23,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +14: [2023-05-09 16:44:23,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +14: [2023-05-09 16:44:23,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +10: [2023-05-09 16:44:23,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +29: [2023-05-09 16:44:23,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +11: [2023-05-09 16:44:23,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +11: [2023-05-09 16:44:23,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +16: [2023-05-09 16:44:23,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +24: [2023-05-09 16:44:23,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +24: [2023-05-09 16:44:23,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +13: [2023-05-09 16:44:23,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +13: [2023-05-09 16:44:23,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +10: [2023-05-09 16:44:23,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +20: [2023-05-09 16:44:23,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +27: [2023-05-09 16:44:23,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +29: [2023-05-09 16:44:23,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +29: [2023-05-09 16:44:23,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +10: [2023-05-09 16:44:23,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +10: [2023-05-09 16:44:23,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +27: [2023-05-09 16:44:23,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +20: [2023-05-09 16:44:23,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +20: [2023-05-09 16:44:23,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +22: [2023-05-09 16:44:23,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +21: [2023-05-09 16:44:23,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +14: [2023-05-09 16:44:23,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +20: [2023-05-09 16:44:23,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 9: [2023-05-09 16:44:23,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +22: [2023-05-09 16:44:23,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +14: [2023-05-09 16:44:23,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +14: [2023-05-09 16:44:23,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +14: [2023-05-09 16:44:23,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +14: [2023-05-09 16:44:23,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +14: [2023-05-09 16:44:23,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +11: [2023-05-09 16:44:23,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +11: [2023-05-09 16:44:23,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 2: [2023-05-09 16:44:23,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +21: [2023-05-09 16:44:23,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +22: [2023-05-09 16:44:23,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +13: [2023-05-09 16:44:23,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 9: [2023-05-09 16:44:23,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +10: [2023-05-09 16:44:23,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +11: [2023-05-09 16:44:23,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +11: [2023-05-09 16:44:23,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +22: [2023-05-09 16:44:23,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +29: [2023-05-09 16:44:23,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +29: [2023-05-09 16:44:23,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +20: [2023-05-09 16:44:23,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +14: [2023-05-09 16:44:23,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +22: [2023-05-09 16:44:23,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +29: [2023-05-09 16:44:23,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +29: [2023-05-09 16:44:23,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +29: [2023-05-09 16:44:23,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +20: [2023-05-09 16:44:23,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +20: [2023-05-09 16:44:23,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +29: [2023-05-09 16:44:23,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +29: [2023-05-09 16:44:23,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 1: [2023-05-09 16:44:23,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +20: [2023-05-09 16:44:23,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +14: [2023-05-09 16:44:23,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +14: [2023-05-09 16:44:23,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +14: [2023-05-09 16:44:23,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +14: [2023-05-09 16:44:23,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 4: [2023-05-09 16:44:23,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +22: [2023-05-09 16:44:23,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +22: [2023-05-09 16:44:23,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 2: [2023-05-09 16:44:23,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +13: [2023-05-09 16:44:23,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +22: [2023-05-09 16:44:23,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +22: [2023-05-09 16:44:23,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +22: [2023-05-09 16:44:23,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +22: [2023-05-09 16:44:23,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +22: [2023-05-09 16:44:23,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +10: [2023-05-09 16:44:23,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +10: [2023-05-09 16:44:23,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 1: [2023-05-09 16:44:23,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +14: [2023-05-09 16:44:23,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +29: [2023-05-09 16:44:23,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +29: [2023-05-09 16:44:23,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +10: [2023-05-09 16:44:23,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +10: [2023-05-09 16:44:23,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 1: [2023-05-09 16:44:23,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +29: [2023-05-09 16:44:23,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +31: [2023-05-09 16:44:23,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +31: [2023-05-09 16:44:23,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +31: [2023-05-09 16:44:23,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +31: [2023-05-09 16:44:23,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +31: [2023-05-09 16:44:23,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +31: [2023-05-09 16:44:23,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +10: [2023-05-09 16:44:23,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +10: [2023-05-09 16:44:23,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +31: [2023-05-09 16:44:23,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +11: [2023-05-09 16:44:23,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +11: [2023-05-09 16:44:23,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +11: [2023-05-09 16:44:23,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +11: [2023-05-09 16:44:23,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +11: [2023-05-09 16:44:23,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +12: [2023-05-09 16:44:23,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +12: [2023-05-09 16:44:23,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +12: [2023-05-09 16:44:23,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +12: [2023-05-09 16:44:23,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:23,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:23,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:23,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +12: [2023-05-09 16:44:23,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +31: [2023-05-09 16:44:23,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +14: [2023-05-09 16:44:23,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +14: [2023-05-09 16:44:23,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +22: [2023-05-09 16:44:23,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +29: [2023-05-09 16:44:23,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +11: [2023-05-09 16:44:23,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +11: [2023-05-09 16:44:23,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +11: [2023-05-09 16:44:23,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +14: [2023-05-09 16:44:23,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +22: [2023-05-09 16:44:23,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +29: [2023-05-09 16:44:23,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +31: [2023-05-09 16:44:23,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:23,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +31: [2023-05-09 16:44:23,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +31: [2023-05-09 16:44:23,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +14: [2023-05-09 16:44:23,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +11: [2023-05-09 16:44:23,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +12: [2023-05-09 16:44:23,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +12: [2023-05-09 16:44:23,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +12: [2023-05-09 16:44:23,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +31: [2023-05-09 16:44:23,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:23,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +31: [2023-05-09 16:44:23,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +31: [2023-05-09 16:44:23,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +31: [2023-05-09 16:44:23,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:23,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:23,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +12: [2023-05-09 16:44:23,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +11: [2023-05-09 16:44:23,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +11: [2023-05-09 16:44:23,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +23: [2023-05-09 16:44:23,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +23: [2023-05-09 16:44:23,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +23: [2023-05-09 16:44:23,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +23: [2023-05-09 16:44:23,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +23: [2023-05-09 16:44:23,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +23: [2023-05-09 16:44:23,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +23: [2023-05-09 16:44:23,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +23: [2023-05-09 16:44:23,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +11: [2023-05-09 16:44:23,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +31: [2023-05-09 16:44:23,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +19: [2023-05-09 16:44:23,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +19: [2023-05-09 16:44:23,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +19: [2023-05-09 16:44:23,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +19: [2023-05-09 16:44:23,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +19: [2023-05-09 16:44:23,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +19: [2023-05-09 16:44:23,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +19: [2023-05-09 16:44:23,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +19: [2023-05-09 16:44:23,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +11: [2023-05-09 16:44:23,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +11: [2023-05-09 16:44:23,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +24: [2023-05-09 16:44:23,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +12: [2023-05-09 16:44:23,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +30: [2023-05-09 16:44:23,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +30: [2023-05-09 16:44:23,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +24: [2023-05-09 16:44:23,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +24: [2023-05-09 16:44:23,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +12: [2023-05-09 16:44:23,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +12: [2023-05-09 16:44:23,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +30: [2023-05-09 16:44:23,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +12: [2023-05-09 16:44:23,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +30: [2023-05-09 16:44:23,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +24: [2023-05-09 16:44:23,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +18: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +18: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 5: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 0: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 0: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +18: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +20: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +20: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +20: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +28: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +16: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +16: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 0: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +11: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +18: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +21: [2023-05-09 16:44:23,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +21: [2023-05-09 16:44:23,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +21: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 5: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +16: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 0: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 5: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +21: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 3: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 3: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +28: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 6: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +28: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +20: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +17: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 5: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +16: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 3: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 6: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 6: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +17: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +28: [2023-05-09 16:44:23,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 3: [2023-05-09 16:44:23,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +17: [2023-05-09 16:44:23,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 7: [2023-05-09 16:44:23,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 6: [2023-05-09 16:44:23,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +23: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +23: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +23: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 7: [2023-05-09 16:44:23,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 7: [2023-05-09 16:44:23,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +17: [2023-05-09 16:44:23,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +26: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +26: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +26: [2023-05-09 16:44:23,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +23: [2023-05-09 16:44:23,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 7: [2023-05-09 16:44:23,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +26: [2023-05-09 16:44:23,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +18: [2023-05-09 16:44:23,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +30: [2023-05-09 16:44:23,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +18: [2023-05-09 16:44:23,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +18: [2023-05-09 16:44:23,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +30: [2023-05-09 16:44:23,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +18: [2023-05-09 16:44:23,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +30: [2023-05-09 16:44:23,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +30: [2023-05-09 16:44:23,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +30: [2023-05-09 16:44:23,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +23: [2023-05-09 16:44:23,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +23: [2023-05-09 16:44:23,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +30: [2023-05-09 16:44:23,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +30: [2023-05-09 16:44:23,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +30: [2023-05-09 16:44:23,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +23: [2023-05-09 16:44:23,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +18: [2023-05-09 16:44:23,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +18: [2023-05-09 16:44:23,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +18: [2023-05-09 16:44:23,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +18: [2023-05-09 16:44:23,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +11: [2023-05-09 16:44:23,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:23,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +12: [2023-05-09 16:44:23,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +24: [2023-05-09 16:44:23,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +12: [2023-05-09 16:44:23,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +30: [2023-05-09 16:44:23,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +21: [2023-05-09 16:44:23,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +21: [2023-05-09 16:44:23,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +24: [2023-05-09 16:44:23,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +30: [2023-05-09 16:44:23,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +21: [2023-05-09 16:44:23,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +21: [2023-05-09 16:44:23,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +26: [2023-05-09 16:44:23,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +30: [2023-05-09 16:44:23,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +20: [2023-05-09 16:44:23,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +20: [2023-05-09 16:44:23,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +18: [2023-05-09 16:44:23,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +24: [2023-05-09 16:44:23,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +16: [2023-05-09 16:44:23,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +16: [2023-05-09 16:44:23,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +20: [2023-05-09 16:44:23,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +18: [2023-05-09 16:44:23,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +16: [2023-05-09 16:44:23,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +23: [2023-05-09 16:44:23,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +12: [2023-05-09 16:44:23,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +17: [2023-05-09 16:44:23,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +28: [2023-05-09 16:44:23,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +17: [2023-05-09 16:44:23,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +23: [2023-05-09 16:44:23,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +17: [2023-05-09 16:44:23,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +28: [2023-05-09 16:44:23,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +16: [2023-05-09 16:44:23,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +24: [2023-05-09 16:44:23,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +30: [2023-05-09 16:44:23,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +23: [2023-05-09 16:44:23,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +23: [2023-05-09 16:44:23,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +17: [2023-05-09 16:44:23,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +20: [2023-05-09 16:44:23,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +26: [2023-05-09 16:44:23,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +18: [2023-05-09 16:44:23,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +23: [2023-05-09 16:44:23,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +28: [2023-05-09 16:44:23,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +18: [2023-05-09 16:44:23,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +19: [2023-05-09 16:44:23,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +28: [2023-05-09 16:44:23,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +23: [2023-05-09 16:44:23,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +26: [2023-05-09 16:44:23,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +26: [2023-05-09 16:44:23,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +23: [2023-05-09 16:44:23,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +23: [2023-05-09 16:44:23,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +30: [2023-05-09 16:44:23,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +24: [2023-05-09 16:44:23,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +12: [2023-05-09 16:44:23,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +12: [2023-05-09 16:44:23,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +19: [2023-05-09 16:44:23,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +19: [2023-05-09 16:44:23,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +19: [2023-05-09 16:44:23,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +24: [2023-05-09 16:44:23,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +24: [2023-05-09 16:44:23,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +24: [2023-05-09 16:44:23,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +24: [2023-05-09 16:44:23,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +24: [2023-05-09 16:44:23,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +17: [2023-05-09 16:44:23,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +23: [2023-05-09 16:44:23,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +18: [2023-05-09 16:44:23,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +18: [2023-05-09 16:44:23,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +16: [2023-05-09 16:44:23,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +26: [2023-05-09 16:44:23,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +26: [2023-05-09 16:44:23,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +16: [2023-05-09 16:44:23,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +30: [2023-05-09 16:44:23,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +19: [2023-05-09 16:44:23,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +16: [2023-05-09 16:44:23,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +16: [2023-05-09 16:44:23,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +21: [2023-05-09 16:44:23,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +21: [2023-05-09 16:44:23,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +17: [2023-05-09 16:44:23,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +21: [2023-05-09 16:44:23,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +12: [2023-05-09 16:44:23,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +12: [2023-05-09 16:44:23,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +12: [2023-05-09 16:44:23,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +12: [2023-05-09 16:44:23,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +21: [2023-05-09 16:44:23,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +21: [2023-05-09 16:44:23,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +21: [2023-05-09 16:44:23,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +20: [2023-05-09 16:44:23,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +16: [2023-05-09 16:44:23,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +30: [2023-05-09 16:44:23,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +20: [2023-05-09 16:44:23,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +20: [2023-05-09 16:44:23,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +20: [2023-05-09 16:44:23,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 5: [2023-05-09 16:44:23,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 5: [2023-05-09 16:44:23,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +18: [2023-05-09 16:44:23,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +20: [2023-05-09 16:44:23,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +20: [2023-05-09 16:44:23,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +16: [2023-05-09 16:44:23,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 7: [2023-05-09 16:44:23,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 7: [2023-05-09 16:44:23,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 6: [2023-05-09 16:44:23,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +18: [2023-05-09 16:44:23,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 6: [2023-05-09 16:44:23,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 6: [2023-05-09 16:44:23,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 3: [2023-05-09 16:44:23,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +17: [2023-05-09 16:44:23,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +30: [2023-05-09 16:44:23,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +17: [2023-05-09 16:44:23,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +17: [2023-05-09 16:44:23,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +17: [2023-05-09 16:44:23,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +30: [2023-05-09 16:44:23,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 7: [2023-05-09 16:44:23,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 3: [2023-05-09 16:44:23,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +30: [2023-05-09 16:44:23,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +19: [2023-05-09 16:44:23,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +21: [2023-05-09 16:44:23,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +20: [2023-05-09 16:44:23,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +20: [2023-05-09 16:44:23,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +28: [2023-05-09 16:44:23,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +28: [2023-05-09 16:44:23,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +28: [2023-05-09 16:44:23,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +18: [2023-05-09 16:44:23,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +21: [2023-05-09 16:44:23,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +19: [2023-05-09 16:44:23,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +18: [2023-05-09 16:44:23,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +19: [2023-05-09 16:44:23,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +23: [2023-05-09 16:44:23,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +23: [2023-05-09 16:44:23,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +23: [2023-05-09 16:44:23,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +28: [2023-05-09 16:44:23,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +28: [2023-05-09 16:44:23,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +23: [2023-05-09 16:44:23,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +28: [2023-05-09 16:44:23,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 3: [2023-05-09 16:44:23,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +23: [2023-05-09 16:44:23,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +23: [2023-05-09 16:44:23,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +23: [2023-05-09 16:44:23,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +23: [2023-05-09 16:44:23,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +30: [2023-05-09 16:44:23,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +30: [2023-05-09 16:44:23,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +30: [2023-05-09 16:44:23,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +17: [2023-05-09 16:44:23,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +18: [2023-05-09 16:44:23,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +17: [2023-05-09 16:44:23,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +18: [2023-05-09 16:44:23,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +18: [2023-05-09 16:44:23,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +26: [2023-05-09 16:44:23,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +26: [2023-05-09 16:44:23,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +12: [2023-05-09 16:44:23,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 0: [2023-05-09 16:44:23,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +30: [2023-05-09 16:44:23,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:23,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 5: [2023-05-09 16:44:23,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +24: [2023-05-09 16:44:23,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +30: [2023-05-09 16:44:23,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +30: [2023-05-09 16:44:23,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +18: [2023-05-09 16:44:23,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +30: [2023-05-09 16:44:23,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +24: [2023-05-09 16:44:23,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +16: [2023-05-09 16:44:23,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +30: [2023-05-09 16:44:23,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +28: [2023-05-09 16:44:23,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +28: [2023-05-09 16:44:23,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 3: [2023-05-09 16:44:23,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +16: [2023-05-09 16:44:23,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 0: [2023-05-09 16:44:23,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +30: [2023-05-09 16:44:23,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 7: [2023-05-09 16:44:23,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +26: [2023-05-09 16:44:23,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +26: [2023-05-09 16:44:23,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +26: [2023-05-09 16:44:23,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +26: [2023-05-09 16:44:23,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 6: [2023-05-09 16:44:23,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +18: [2023-05-09 16:44:23,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +30: [2023-05-09 16:44:23,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +18: [2023-05-09 16:44:23,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +18: [2023-05-09 16:44:23,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +18: [2023-05-09 16:44:23,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 0: [2023-05-09 16:44:23,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +28: [2023-05-09 16:44:23,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +28: [2023-05-09 16:44:23,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +28: [2023-05-09 16:44:23,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +28: [2023-05-09 16:44:23,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +18: [2023-05-09 16:44:23,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +28: [2023-05-09 16:44:23,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +28: [2023-05-09 16:44:23,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +18: [2023-05-09 16:44:23,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 8: [2023-05-09 16:44:23,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +28: [2023-05-09 16:44:23,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +28: [2023-05-09 16:44:23,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +28: [2023-05-09 16:44:23,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +28: [2023-05-09 16:44:23,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +28: [2023-05-09 16:44:23,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +28: [2023-05-09 16:44:23,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +28: [2023-05-09 16:44:23,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +28: [2023-05-09 16:44:23,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +15: [2023-05-09 16:44:23,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +15: [2023-05-09 16:44:23,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +15: [2023-05-09 16:44:23,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +15: [2023-05-09 16:44:23,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +15: [2023-05-09 16:44:23,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +15: [2023-05-09 16:44:23,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +15: [2023-05-09 16:44:23,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +28: [2023-05-09 16:44:23,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +15: [2023-05-09 16:44:23,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +25: [2023-05-09 16:44:23,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +25: [2023-05-09 16:44:23,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +25: [2023-05-09 16:44:23,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +25: [2023-05-09 16:44:23,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +25: [2023-05-09 16:44:23,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +25: [2023-05-09 16:44:23,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +25: [2023-05-09 16:44:23,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +25: [2023-05-09 16:44:23,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +26: [2023-05-09 16:44:23,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +26: [2023-05-09 16:44:23,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +26: [2023-05-09 16:44:23,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +26: [2023-05-09 16:44:23,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +26: [2023-05-09 16:44:23,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +26: [2023-05-09 16:44:23,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +13: [2023-05-09 16:44:23,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +26: [2023-05-09 16:44:23,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +13: [2023-05-09 16:44:23,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +13: [2023-05-09 16:44:23,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +13: [2023-05-09 16:44:23,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +13: [2023-05-09 16:44:23,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +13: [2023-05-09 16:44:23,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +26: [2023-05-09 16:44:23,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +10: [2023-05-09 16:44:23,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +13: [2023-05-09 16:44:23,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +10: [2023-05-09 16:44:23,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +10: [2023-05-09 16:44:23,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +13: [2023-05-09 16:44:23,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +10: [2023-05-09 16:44:23,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +10: [2023-05-09 16:44:23,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +10: [2023-05-09 16:44:23,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +10: [2023-05-09 16:44:23,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +10: [2023-05-09 16:44:23,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +21: [2023-05-09 16:44:23,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +21: [2023-05-09 16:44:23,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +21: [2023-05-09 16:44:23,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +21: [2023-05-09 16:44:23,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +21: [2023-05-09 16:44:23,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +21: [2023-05-09 16:44:23,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +21: [2023-05-09 16:44:23,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +21: [2023-05-09 16:44:23,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +15: [2023-05-09 16:44:23,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +17: [2023-05-09 16:44:23,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +17: [2023-05-09 16:44:23,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +17: [2023-05-09 16:44:23,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +17: [2023-05-09 16:44:23,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +17: [2023-05-09 16:44:23,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +17: [2023-05-09 16:44:23,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +28: [2023-05-09 16:44:23,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +17: [2023-05-09 16:44:23,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +17: [2023-05-09 16:44:23,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +25: [2023-05-09 16:44:23,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +16: [2023-05-09 16:44:23,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +16: [2023-05-09 16:44:23,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +16: [2023-05-09 16:44:23,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +16: [2023-05-09 16:44:23,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +16: [2023-05-09 16:44:23,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +16: [2023-05-09 16:44:23,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +13: [2023-05-09 16:44:23,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +16: [2023-05-09 16:44:23,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +16: [2023-05-09 16:44:23,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +31: [2023-05-09 16:44:23,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +15: [2023-05-09 16:44:23,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +31: [2023-05-09 16:44:23,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +31: [2023-05-09 16:44:23,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +31: [2023-05-09 16:44:23,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +31: [2023-05-09 16:44:23,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +27: [2023-05-09 16:44:23,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +27: [2023-05-09 16:44:23,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +31: [2023-05-09 16:44:23,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +31: [2023-05-09 16:44:23,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +31: [2023-05-09 16:44:23,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +27: [2023-05-09 16:44:23,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +27: [2023-05-09 16:44:23,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +27: [2023-05-09 16:44:23,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +27: [2023-05-09 16:44:23,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +27: [2023-05-09 16:44:23,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +27: [2023-05-09 16:44:23,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +26: [2023-05-09 16:44:23,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +21: [2023-05-09 16:44:23,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +26: [2023-05-09 16:44:23,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +26: [2023-05-09 16:44:23,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +25: [2023-05-09 16:44:23,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +25: [2023-05-09 16:44:23,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +25: [2023-05-09 16:44:23,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +25: [2023-05-09 16:44:23,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +13: [2023-05-09 16:44:23,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +24: [2023-05-09 16:44:23,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +24: [2023-05-09 16:44:23,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +24: [2023-05-09 16:44:23,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +24: [2023-05-09 16:44:23,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +15: [2023-05-09 16:44:23,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +24: [2023-05-09 16:44:23,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +11: [2023-05-09 16:44:23,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +11: [2023-05-09 16:44:23,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +11: [2023-05-09 16:44:23,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +24: [2023-05-09 16:44:23,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +11: [2023-05-09 16:44:23,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +11: [2023-05-09 16:44:23,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +11: [2023-05-09 16:44:23,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +24: [2023-05-09 16:44:23,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +22: [2023-05-09 16:44:23,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +22: [2023-05-09 16:44:23,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +10: [2023-05-09 16:44:23,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +22: [2023-05-09 16:44:23,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +11: [2023-05-09 16:44:23,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +22: [2023-05-09 16:44:23,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +15: [2023-05-09 16:44:23,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +22: [2023-05-09 16:44:23,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +22: [2023-05-09 16:44:23,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +17: [2023-05-09 16:44:23,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +22: [2023-05-09 16:44:23,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +15: [2023-05-09 16:44:23,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +13: [2023-05-09 16:44:23,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +21: [2023-05-09 16:44:23,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +21: [2023-05-09 16:44:23,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +11: [2023-05-09 16:44:23,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +22: [2023-05-09 16:44:23,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +17: [2023-05-09 16:44:23,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +24: [2023-05-09 16:44:23,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +26: [2023-05-09 16:44:23,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +21: [2023-05-09 16:44:23,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +10: [2023-05-09 16:44:23,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +10: [2023-05-09 16:44:23,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +26: [2023-05-09 16:44:23,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +26: [2023-05-09 16:44:23,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +27: [2023-05-09 16:44:23,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +16: [2023-05-09 16:44:23,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +31: [2023-05-09 16:44:23,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +31: [2023-05-09 16:44:23,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +25: [2023-05-09 16:44:23,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +25: [2023-05-09 16:44:23,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +27: [2023-05-09 16:44:23,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +16: [2023-05-09 16:44:23,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +27: [2023-05-09 16:44:23,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +25: [2023-05-09 16:44:23,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +27: [2023-05-09 16:44:23,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +26: [2023-05-09 16:44:23,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +21: [2023-05-09 16:44:23,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +31: [2023-05-09 16:44:23,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +31: [2023-05-09 16:44:23,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +13: [2023-05-09 16:44:23,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +15: [2023-05-09 16:44:23,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +15: [2023-05-09 16:44:23,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +17: [2023-05-09 16:44:23,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +10: [2023-05-09 16:44:23,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +15: [2023-05-09 16:44:23,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +10: [2023-05-09 16:44:23,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +13: [2023-05-09 16:44:23,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +22: [2023-05-09 16:44:23,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +17: [2023-05-09 16:44:23,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +22: [2023-05-09 16:44:23,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +24: [2023-05-09 16:44:23,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +16: [2023-05-09 16:44:23,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +13: [2023-05-09 16:44:23,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +31: [2023-05-09 16:44:23,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +17: [2023-05-09 16:44:23,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +21: [2023-05-09 16:44:23,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +16: [2023-05-09 16:44:23,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +21: [2023-05-09 16:44:23,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +27: [2023-05-09 16:44:23,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +21: [2023-05-09 16:44:23,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +22: [2023-05-09 16:44:23,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +31: [2023-05-09 16:44:23,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +26: [2023-05-09 16:44:23,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +16: [2023-05-09 16:44:23,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +27: [2023-05-09 16:44:23,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +10: [2023-05-09 16:44:23,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +16: [2023-05-09 16:44:23,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +10: [2023-05-09 16:44:23,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +27: [2023-05-09 16:44:23,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +31: [2023-05-09 16:44:23,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +31: [2023-05-09 16:44:23,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +17: [2023-05-09 16:44:23,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +24: [2023-05-09 16:44:23,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +27: [2023-05-09 16:44:23,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +22: [2023-05-09 16:44:23,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +13: [2023-05-09 16:44:23,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +11: [2023-05-09 16:44:23,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +11: [2023-05-09 16:44:23,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +11: [2023-05-09 16:44:23,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +10: [2023-05-09 16:44:23,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +22: [2023-05-09 16:44:23,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +24: [2023-05-09 16:44:23,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +24: [2023-05-09 16:44:23,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +17: [2023-05-09 16:44:23,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +22: [2023-05-09 16:44:23,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +13: [2023-05-09 16:44:23,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +17: [2023-05-09 16:44:23,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +16: [2023-05-09 16:44:23,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +16: [2023-05-09 16:44:23,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +22: [2023-05-09 16:44:23,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +30: [2023-05-09 16:44:23,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +24: [2023-05-09 16:44:23,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +30: [2023-05-09 16:44:23,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +30: [2023-05-09 16:44:23,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +30: [2023-05-09 16:44:23,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +30: [2023-05-09 16:44:23,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +30: [2023-05-09 16:44:23,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +30: [2023-05-09 16:44:23,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +30: [2023-05-09 16:44:23,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +11: [2023-05-09 16:44:23,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +24: [2023-05-09 16:44:23,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +11: [2023-05-09 16:44:23,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +11: [2023-05-09 16:44:23,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +18: [2023-05-09 16:44:23,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +18: [2023-05-09 16:44:23,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +18: [2023-05-09 16:44:23,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +18: [2023-05-09 16:44:23,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +18: [2023-05-09 16:44:23,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +18: [2023-05-09 16:44:23,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +18: [2023-05-09 16:44:23,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +18: [2023-05-09 16:44:23,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +22: [2023-05-09 16:44:23,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +24: [2023-05-09 16:44:23,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +29: [2023-05-09 16:44:23,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +29: [2023-05-09 16:44:23,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +29: [2023-05-09 16:44:23,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +29: [2023-05-09 16:44:23,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +29: [2023-05-09 16:44:23,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +20: [2023-05-09 16:44:23,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +20: [2023-05-09 16:44:23,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +30: [2023-05-09 16:44:23,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +20: [2023-05-09 16:44:23,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +20: [2023-05-09 16:44:23,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +20: [2023-05-09 16:44:23,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +20: [2023-05-09 16:44:23,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +30: [2023-05-09 16:44:23,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +20: [2023-05-09 16:44:23,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +20: [2023-05-09 16:44:23,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +29: [2023-05-09 16:44:23,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +29: [2023-05-09 16:44:23,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +29: [2023-05-09 16:44:23,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +11: [2023-05-09 16:44:23,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 8: [2023-05-09 16:44:23,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 8: [2023-05-09 16:44:23,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +27: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 8: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +19: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +19: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +15: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +31: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +15: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +15: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +19: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 4: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 4: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +31: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +31: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 4: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +27: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +13: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +13: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +13: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +25: [2023-05-09 16:44:23,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +25: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +25: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +15: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +27: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +31: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +13: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +25: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +27: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. + 4: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +19: [2023-05-09 16:44:23,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt. +14: [2023-05-09 16:44:23,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +10: [2023-05-09 16:44:23,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +14: [2023-05-09 16:44:23,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +14: [2023-05-09 16:44:23,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +14: [2023-05-09 16:44:23,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +10: [2023-05-09 16:44:23,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +10: [2023-05-09 16:44:23,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +10: [2023-05-09 16:44:23,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +30: [2023-05-09 16:44:23,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +30: [2023-05-09 16:44:23,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +24: [2023-05-09 16:44:23,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +18: [2023-05-09 16:44:23,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +30: [2023-05-09 16:44:23,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +18: [2023-05-09 16:44:23,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +30: [2023-05-09 16:44:23,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +25: [2023-05-09 16:44:23,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +27: [2023-05-09 16:44:23,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +11: [2023-05-09 16:44:23,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +31: [2023-05-09 16:44:23,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +27: [2023-05-09 16:44:23,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +13: [2023-05-09 16:44:23,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +13: [2023-05-09 16:44:23,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +31: [2023-05-09 16:44:23,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +25: [2023-05-09 16:44:23,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +13: [2023-05-09 16:44:23,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +25: [2023-05-09 16:44:23,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +31: [2023-05-09 16:44:23,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +31: [2023-05-09 16:44:23,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +19: [2023-05-09 16:44:23,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +13: [2023-05-09 16:44:23,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +18: [2023-05-09 16:44:23,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +15: [2023-05-09 16:44:23,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +25: [2023-05-09 16:44:23,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +27: [2023-05-09 16:44:23,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +10: [2023-05-09 16:44:23,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +30: [2023-05-09 16:44:23,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +30: [2023-05-09 16:44:23,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +14: [2023-05-09 16:44:23,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +29: [2023-05-09 16:44:23,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +20: [2023-05-09 16:44:23,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +20: [2023-05-09 16:44:23,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +14: [2023-05-09 16:44:23,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +15: [2023-05-09 16:44:23,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +15: [2023-05-09 16:44:23,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +10: [2023-05-09 16:44:23,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +10: [2023-05-09 16:44:23,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +27: [2023-05-09 16:44:23,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +14: [2023-05-09 16:44:23,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +18: [2023-05-09 16:44:23,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +15: [2023-05-09 16:44:23,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +19: [2023-05-09 16:44:23,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +19: [2023-05-09 16:44:23,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +19: [2023-05-09 16:44:23,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt... +20: [2023-05-09 16:44:23,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +14: [2023-05-09 16:44:23,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +20: [2023-05-09 16:44:23,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +10: [2023-05-09 16:44:23,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +18: [2023-05-09 16:44:23,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +25: [2023-05-09 16:44:23,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +18: [2023-05-09 16:44:23,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +25: [2023-05-09 16:44:23,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +20: [2023-05-09 16:44:23,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +18: [2023-05-09 16:44:23,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +29: [2023-05-09 16:44:23,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +27: [2023-05-09 16:44:23,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +27: [2023-05-09 16:44:23,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +20: [2023-05-09 16:44:23,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +20: [2023-05-09 16:44:23,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +27: [2023-05-09 16:44:23,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +20: [2023-05-09 16:44:23,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +29: [2023-05-09 16:44:23,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +27: [2023-05-09 16:44:23,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +31: [2023-05-09 16:44:23,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +13: [2023-05-09 16:44:23,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +13: [2023-05-09 16:44:23,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +19: [2023-05-09 16:44:23,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +31: [2023-05-09 16:44:23,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +13: [2023-05-09 16:44:23,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +13: [2023-05-09 16:44:23,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +15: [2023-05-09 16:44:23,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +15: [2023-05-09 16:44:23,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +15: [2023-05-09 16:44:23,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +15: [2023-05-09 16:44:23,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 8: [2023-05-09 16:44:23,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +13: [2023-05-09 16:44:23,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +19: [2023-05-09 16:44:23,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +25: [2023-05-09 16:44:23,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +25: [2023-05-09 16:44:23,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +29: [2023-05-09 16:44:23,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +29: [2023-05-09 16:44:23,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +14: [2023-05-09 16:44:23,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +13: [2023-05-09 16:44:23,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +13: [2023-05-09 16:44:23,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +13: [2023-05-09 16:44:23,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +14: [2023-05-09 16:44:23,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +18: [2023-05-09 16:44:23,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +31: [2023-05-09 16:44:23,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +31: [2023-05-09 16:44:23,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +31: [2023-05-09 16:44:23,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +31: [2023-05-09 16:44:23,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +10: [2023-05-09 16:44:23,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +10: [2023-05-09 16:44:23,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +10: [2023-05-09 16:44:23,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +10: [2023-05-09 16:44:23,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +10: [2023-05-09 16:44:23,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 4: [2023-05-09 16:44:23,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +10: [2023-05-09 16:44:23,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 4: [2023-05-09 16:44:23,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +15: [2023-05-09 16:44:23,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +15: [2023-05-09 16:44:23,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +19: [2023-05-09 16:44:23,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +19: [2023-05-09 16:44:23,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +19: [2023-05-09 16:44:23,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +19: [2023-05-09 16:44:23,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +25: [2023-05-09 16:44:23,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +25: [2023-05-09 16:44:23,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 4: [2023-05-09 16:44:23,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +29: [2023-05-09 16:44:23,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +31: [2023-05-09 16:44:23,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +31: [2023-05-09 16:44:23,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +14: [2023-05-09 16:44:23,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +27: [2023-05-09 16:44:23,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +14: [2023-05-09 16:44:23,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +14: [2023-05-09 16:44:23,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +14: [2023-05-09 16:44:23,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 1: [2023-05-09 16:44:23,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 1: [2023-05-09 16:44:23,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +27: [2023-05-09 16:44:23,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 4: [2023-05-09 16:44:23,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +29: [2023-05-09 16:44:23,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +29: [2023-05-09 16:44:23,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 1: [2023-05-09 16:44:23,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 4: [2023-05-09 16:44:23,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +19: [2023-05-09 16:44:23,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +25: [2023-05-09 16:44:23,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +12: [2023-05-09 16:44:23,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +12: [2023-05-09 16:44:23,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +12: [2023-05-09 16:44:23,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +12: [2023-05-09 16:44:23,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +12: [2023-05-09 16:44:23,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +12: [2023-05-09 16:44:23,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +12: [2023-05-09 16:44:23,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +12: [2023-05-09 16:44:23,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +19: [2023-05-09 16:44:23,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +25: [2023-05-09 16:44:23,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 8: [2023-05-09 16:44:23,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +15: [2023-05-09 16:44:23,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +27: [2023-05-09 16:44:23,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt. +27: [2023-05-09 16:44:23,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 8: [2023-05-09 16:44:23,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... +15: [2023-05-09 16:44:23,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt... + 6: [2023-05-09 16:44:23,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +10: [2023-05-09 16:44:23,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +10: [2023-05-09 16:44:23,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +14: [2023-05-09 16:44:23,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +14: [2023-05-09 16:44:23,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +12: [2023-05-09 16:44:23,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +16: [2023-05-09 16:44:23,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +16: [2023-05-09 16:44:23,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +16: [2023-05-09 16:44:23,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +16: [2023-05-09 16:44:23,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +12: [2023-05-09 16:44:23,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +12: [2023-05-09 16:44:23,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +12: [2023-05-09 16:44:23,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +12: [2023-05-09 16:44:23,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 6: [2023-05-09 16:44:23,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +16: [2023-05-09 16:44:23,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +16: [2023-05-09 16:44:23,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +16: [2023-05-09 16:44:23,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +16: [2023-05-09 16:44:23,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +12: [2023-05-09 16:44:23,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +12: [2023-05-09 16:44:23,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 6: [2023-05-09 16:44:23,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +12: [2023-05-09 16:44:23,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +16: [2023-05-09 16:44:23,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +16: [2023-05-09 16:44:23,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +16: [2023-05-09 16:44:23,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +16: [2023-05-09 16:44:23,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +16: [2023-05-09 16:44:23,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +16: [2023-05-09 16:44:23,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +14: [2023-05-09 16:44:23,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +14: [2023-05-09 16:44:23,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +14: [2023-05-09 16:44:23,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +14: [2023-05-09 16:44:23,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +14: [2023-05-09 16:44:23,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +14: [2023-05-09 16:44:23,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +14: [2023-05-09 16:44:23,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +16: [2023-05-09 16:44:23,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +14: [2023-05-09 16:44:23,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +16: [2023-05-09 16:44:23,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +14: [2023-05-09 16:44:23,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +14: [2023-05-09 16:44:23,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +23: [2023-05-09 16:44:23,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +23: [2023-05-09 16:44:23,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +23: [2023-05-09 16:44:23,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +23: [2023-05-09 16:44:23,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +23: [2023-05-09 16:44:23,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +19: [2023-05-09 16:44:23,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +19: [2023-05-09 16:44:23,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +19: [2023-05-09 16:44:23,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +19: [2023-05-09 16:44:23,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +19: [2023-05-09 16:44:23,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +19: [2023-05-09 16:44:23,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +19: [2023-05-09 16:44:23,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +23: [2023-05-09 16:44:23,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +19: [2023-05-09 16:44:23,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +23: [2023-05-09 16:44:23,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +23: [2023-05-09 16:44:23,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +14: [2023-05-09 16:44:23,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +14: [2023-05-09 16:44:23,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +14: [2023-05-09 16:44:23,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +19: [2023-05-09 16:44:23,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +14: [2023-05-09 16:44:23,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 2: [2023-05-09 16:44:23,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 2: [2023-05-09 16:44:23,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 8: [2023-05-09 16:44:23,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +14: [2023-05-09 16:44:23,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +23: [2023-05-09 16:44:23,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +23: [2023-05-09 16:44:23,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +23: [2023-05-09 16:44:23,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +19: [2023-05-09 16:44:23,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +19: [2023-05-09 16:44:23,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +19: [2023-05-09 16:44:23,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +14: [2023-05-09 16:44:23,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +23: [2023-05-09 16:44:23,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +19: [2023-05-09 16:44:23,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 9: [2023-05-09 16:44:23,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 9: [2023-05-09 16:44:23,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 9: [2023-05-09 16:44:23,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +23: [2023-05-09 16:44:23,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +19: [2023-05-09 16:44:23,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +23: [2023-05-09 16:44:23,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +23: [2023-05-09 16:44:23,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +19: [2023-05-09 16:44:23,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +19: [2023-05-09 16:44:23,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +12: [2023-05-09 16:44:23,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +12: [2023-05-09 16:44:23,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 2: [2023-05-09 16:44:23,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +12: [2023-05-09 16:44:23,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 2: [2023-05-09 16:44:23,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 2: [2023-05-09 16:44:23,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +12: [2023-05-09 16:44:23,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +23: [2023-05-09 16:44:23,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +22: [2023-05-09 16:44:23,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +22: [2023-05-09 16:44:23,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +22: [2023-05-09 16:44:23,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +22: [2023-05-09 16:44:23,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 2: [2023-05-09 16:44:23,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +12: [2023-05-09 16:44:23,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +12: [2023-05-09 16:44:23,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +12: [2023-05-09 16:44:23,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +12: [2023-05-09 16:44:23,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 9: [2023-05-09 16:44:23,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +22: [2023-05-09 16:44:23,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +22: [2023-05-09 16:44:23,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +22: [2023-05-09 16:44:23,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +22: [2023-05-09 16:44:23,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +12: [2023-05-09 16:44:23,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +12: [2023-05-09 16:44:23,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +12: [2023-05-09 16:44:23,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +12: [2023-05-09 16:44:23,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +12: [2023-05-09 16:44:23,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +12: [2023-05-09 16:44:23,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +11: [2023-05-09 16:44:23,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +11: [2023-05-09 16:44:23,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +11: [2023-05-09 16:44:23,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +12: [2023-05-09 16:44:23,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +11: [2023-05-09 16:44:23,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +12: [2023-05-09 16:44:23,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 9: [2023-05-09 16:44:23,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 9: [2023-05-09 16:44:23,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +25: [2023-05-09 16:44:23,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +25: [2023-05-09 16:44:23,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +25: [2023-05-09 16:44:23,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +25: [2023-05-09 16:44:23,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +25: [2023-05-09 16:44:23,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +25: [2023-05-09 16:44:23,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +25: [2023-05-09 16:44:23,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +26: [2023-05-09 16:44:23,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +26: [2023-05-09 16:44:23,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +25: [2023-05-09 16:44:23,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +26: [2023-05-09 16:44:23,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +26: [2023-05-09 16:44:23,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 0: [2023-05-09 16:44:23,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +22: [2023-05-09 16:44:23,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +22: [2023-05-09 16:44:23,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +22: [2023-05-09 16:44:23,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +22: [2023-05-09 16:44:23,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 0: [2023-05-09 16:44:23,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +11: [2023-05-09 16:44:23,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +11: [2023-05-09 16:44:23,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +22: [2023-05-09 16:44:23,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +22: [2023-05-09 16:44:23,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 0: [2023-05-09 16:44:23,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +11: [2023-05-09 16:44:23,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +11: [2023-05-09 16:44:23,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +26: [2023-05-09 16:44:23,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +25: [2023-05-09 16:44:23,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +29: [2023-05-09 16:44:23,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +29: [2023-05-09 16:44:23,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +29: [2023-05-09 16:44:23,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +17: [2023-05-09 16:44:23,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +26: [2023-05-09 16:44:23,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +29: [2023-05-09 16:44:23,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +17: [2023-05-09 16:44:23,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +17: [2023-05-09 16:44:23,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +17: [2023-05-09 16:44:23,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +26: [2023-05-09 16:44:23,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +26: [2023-05-09 16:44:23,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +22: [2023-05-09 16:44:23,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 5: [2023-05-09 16:44:23,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 5: [2023-05-09 16:44:23,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +22: [2023-05-09 16:44:23,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 5: [2023-05-09 16:44:23,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +28: [2023-05-09 16:44:23,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +28: [2023-05-09 16:44:23,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +28: [2023-05-09 16:44:23,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +28: [2023-05-09 16:44:23,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +11: [2023-05-09 16:44:23,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +25: [2023-05-09 16:44:23,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +25: [2023-05-09 16:44:23,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +11: [2023-05-09 16:44:23,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +25: [2023-05-09 16:44:23,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 7: [2023-05-09 16:44:23,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 7: [2023-05-09 16:44:23,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 7: [2023-05-09 16:44:23,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +29: [2023-05-09 16:44:23,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +17: [2023-05-09 16:44:23,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +29: [2023-05-09 16:44:23,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +29: [2023-05-09 16:44:23,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +17: [2023-05-09 16:44:23,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +17: [2023-05-09 16:44:23,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +11: [2023-05-09 16:44:23,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +11: [2023-05-09 16:44:23,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +29: [2023-05-09 16:44:23,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +17: [2023-05-09 16:44:23,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +28: [2023-05-09 16:44:23,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +20: [2023-05-09 16:44:23,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +20: [2023-05-09 16:44:23,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +20: [2023-05-09 16:44:23,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +20: [2023-05-09 16:44:23,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 6: [2023-05-09 16:44:23,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 5: [2023-05-09 16:44:23,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 3: [2023-05-09 16:44:23,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +28: [2023-05-09 16:44:23,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +28: [2023-05-09 16:44:23,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +26: [2023-05-09 16:44:23,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +25: [2023-05-09 16:44:23,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 3: [2023-05-09 16:44:23,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +26: [2023-05-09 16:44:23,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 6: [2023-05-09 16:44:23,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 6: [2023-05-09 16:44:23,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +26: [2023-05-09 16:44:23,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +26: [2023-05-09 16:44:23,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 7: [2023-05-09 16:44:23,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +28: [2023-05-09 16:44:23,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +25: [2023-05-09 16:44:23,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +17: [2023-05-09 16:44:23,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +21: [2023-05-09 16:44:23,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +24: [2023-05-09 16:44:23,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +21: [2023-05-09 16:44:23,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +24: [2023-05-09 16:44:23,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +24: [2023-05-09 16:44:23,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +21: [2023-05-09 16:44:23,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +24: [2023-05-09 16:44:23,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +17: [2023-05-09 16:44:23,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +21: [2023-05-09 16:44:23,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +29: [2023-05-09 16:44:23,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +29: [2023-05-09 16:44:23,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +29: [2023-05-09 16:44:23,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +29: [2023-05-09 16:44:23,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +11: [2023-05-09 16:44:23,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +11: [2023-05-09 16:44:23,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +26: [2023-05-09 16:44:23,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +26: [2023-05-09 16:44:23,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +29: [2023-05-09 16:44:23,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +29: [2023-05-09 16:44:23,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +25: [2023-05-09 16:44:23,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +11: [2023-05-09 16:44:23,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +20: [2023-05-09 16:44:23,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +20: [2023-05-09 16:44:23,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +11: [2023-05-09 16:44:23,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +26: [2023-05-09 16:44:23,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +17: [2023-05-09 16:44:23,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +26: [2023-05-09 16:44:23,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +17: [2023-05-09 16:44:23,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 0: [2023-05-09 16:44:23,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +20: [2023-05-09 16:44:23,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +20: [2023-05-09 16:44:23,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +28: [2023-05-09 16:44:23,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +28: [2023-05-09 16:44:23,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 5: [2023-05-09 16:44:23,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 0: [2023-05-09 16:44:23,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +21: [2023-05-09 16:44:23,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +24: [2023-05-09 16:44:23,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +23: [2023-05-09 16:44:23,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +23: [2023-05-09 16:44:23,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +23: [2023-05-09 16:44:23,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +23: [2023-05-09 16:44:23,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +17: [2023-05-09 16:44:23,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +17: [2023-05-09 16:44:23,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +24: [2023-05-09 16:44:23,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +25: [2023-05-09 16:44:23,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 5: [2023-05-09 16:44:23,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +24: [2023-05-09 16:44:23,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +24: [2023-05-09 16:44:23,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +21: [2023-05-09 16:44:23,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 7: [2023-05-09 16:44:23,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +29: [2023-05-09 16:44:23,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 7: [2023-05-09 16:44:23,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 7: [2023-05-09 16:44:23,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +28: [2023-05-09 16:44:23,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +28: [2023-05-09 16:44:23,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +28: [2023-05-09 16:44:23,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +28: [2023-05-09 16:44:23,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +21: [2023-05-09 16:44:23,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +21: [2023-05-09 16:44:23,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +29: [2023-05-09 16:44:23,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +30: [2023-05-09 16:44:23,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +30: [2023-05-09 16:44:23,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +30: [2023-05-09 16:44:23,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +30: [2023-05-09 16:44:23,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 6: [2023-05-09 16:44:23,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +17: [2023-05-09 16:44:23,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +23: [2023-05-09 16:44:23,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +23: [2023-05-09 16:44:23,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 3: [2023-05-09 16:44:23,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 3: [2023-05-09 16:44:23,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 3: [2023-05-09 16:44:23,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +17: [2023-05-09 16:44:23,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +23: [2023-05-09 16:44:23,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +23: [2023-05-09 16:44:23,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 7: [2023-05-09 16:44:23,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +20: [2023-05-09 16:44:23,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +20: [2023-05-09 16:44:23,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 7: [2023-05-09 16:44:23,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +30: [2023-05-09 16:44:23,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +24: [2023-05-09 16:44:23,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +30: [2023-05-09 16:44:23,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +30: [2023-05-09 16:44:23,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +24: [2023-05-09 16:44:23,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 5: [2023-05-09 16:44:23,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +30: [2023-05-09 16:44:23,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +24: [2023-05-09 16:44:23,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +24: [2023-05-09 16:44:23,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +24: [2023-05-09 16:44:23,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +24: [2023-05-09 16:44:23,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +24: [2023-05-09 16:44:23,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +24: [2023-05-09 16:44:23,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 3: [2023-05-09 16:44:23,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +28: [2023-05-09 16:44:23,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +21: [2023-05-09 16:44:23,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +21: [2023-05-09 16:44:23,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +28: [2023-05-09 16:44:23,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +21: [2023-05-09 16:44:23,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +21: [2023-05-09 16:44:23,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +20: [2023-05-09 16:44:23,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +20: [2023-05-09 16:44:23,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +20: [2023-05-09 16:44:23,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +20: [2023-05-09 16:44:23,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 0: [2023-05-09 16:44:23,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 0: [2023-05-09 16:44:23,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 0: [2023-05-09 16:44:23,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 0: [2023-05-09 16:44:23,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +23: [2023-05-09 16:44:23,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +23: [2023-05-09 16:44:23,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +23: [2023-05-09 16:44:23,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +23: [2023-05-09 16:44:23,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +20: [2023-05-09 16:44:23,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +30: [2023-05-09 16:44:23,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +20: [2023-05-09 16:44:23,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +30: [2023-05-09 16:44:23,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +27: [2023-05-09 16:44:23,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +27: [2023-05-09 16:44:23,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +27: [2023-05-09 16:44:23,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +27: [2023-05-09 16:44:23,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +27: [2023-05-09 16:44:23,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +27: [2023-05-09 16:44:23,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +27: [2023-05-09 16:44:23,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +30: [2023-05-09 16:44:23,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +13: [2023-05-09 16:44:23,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +13: [2023-05-09 16:44:23,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +13: [2023-05-09 16:44:23,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +13: [2023-05-09 16:44:23,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +30: [2023-05-09 16:44:23,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +13: [2023-05-09 16:44:23,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +13: [2023-05-09 16:44:23,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +15: [2023-05-09 16:44:23,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +15: [2023-05-09 16:44:23,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +15: [2023-05-09 16:44:23,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +15: [2023-05-09 16:44:23,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +15: [2023-05-09 16:44:23,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +15: [2023-05-09 16:44:23,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +15: [2023-05-09 16:44:23,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +15: [2023-05-09 16:44:23,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +30: [2023-05-09 16:44:23,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +30: [2023-05-09 16:44:23,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +27: [2023-05-09 16:44:23,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +21: [2023-05-09 16:44:23,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +13: [2023-05-09 16:44:23,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +21: [2023-05-09 16:44:23,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +21: [2023-05-09 16:44:23,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +21: [2023-05-09 16:44:23,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +21: [2023-05-09 16:44:23,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +21: [2023-05-09 16:44:23,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +21: [2023-05-09 16:44:23,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +16: [2023-05-09 16:44:23,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +16: [2023-05-09 16:44:23,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +16: [2023-05-09 16:44:23,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +16: [2023-05-09 16:44:23,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +16: [2023-05-09 16:44:23,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +13: [2023-05-09 16:44:23,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +16: [2023-05-09 16:44:23,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +16: [2023-05-09 16:44:23,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +16: [2023-05-09 16:44:23,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +30: [2023-05-09 16:44:23,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +21: [2023-05-09 16:44:23,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +30: [2023-05-09 16:44:23,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +21: [2023-05-09 16:44:23,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +23: [2023-05-09 16:44:23,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +26: [2023-05-09 16:44:23,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +26: [2023-05-09 16:44:23,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +26: [2023-05-09 16:44:23,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +26: [2023-05-09 16:44:23,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +26: [2023-05-09 16:44:23,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +26: [2023-05-09 16:44:23,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +26: [2023-05-09 16:44:23,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +23: [2023-05-09 16:44:23,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +26: [2023-05-09 16:44:23,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +21: [2023-05-09 16:44:23,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 1: [2023-05-09 16:44:23,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +21: [2023-05-09 16:44:23,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +23: [2023-05-09 16:44:23,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +23: [2023-05-09 16:44:23,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +21: [2023-05-09 16:44:23,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +18: [2023-05-09 16:44:23,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +18: [2023-05-09 16:44:23,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +18: [2023-05-09 16:44:23,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +18: [2023-05-09 16:44:23,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +27: [2023-05-09 16:44:23,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +27: [2023-05-09 16:44:23,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +15: [2023-05-09 16:44:23,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +19: [2023-05-09 16:44:23,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +15: [2023-05-09 16:44:23,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +19: [2023-05-09 16:44:23,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +19: [2023-05-09 16:44:23,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +15: [2023-05-09 16:44:23,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +19: [2023-05-09 16:44:23,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +15: [2023-05-09 16:44:23,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +13: [2023-05-09 16:44:23,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +13: [2023-05-09 16:44:23,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +13: [2023-05-09 16:44:23,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 4: [2023-05-09 16:44:23,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +16: [2023-05-09 16:44:23,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +16: [2023-05-09 16:44:23,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +16: [2023-05-09 16:44:23,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +13: [2023-05-09 16:44:23,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +16: [2023-05-09 16:44:23,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +13: [2023-05-09 16:44:23,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +13: [2023-05-09 16:44:23,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +20: [2023-05-09 16:44:23,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +20: [2023-05-09 16:44:23,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +20: [2023-05-09 16:44:23,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +20: [2023-05-09 16:44:23,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +21: [2023-05-09 16:44:23,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +21: [2023-05-09 16:44:23,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +20: [2023-05-09 16:44:23,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +20: [2023-05-09 16:44:23,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +20: [2023-05-09 16:44:23,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +20: [2023-05-09 16:44:23,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +26: [2023-05-09 16:44:23,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +26: [2023-05-09 16:44:23,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 1: [2023-05-09 16:44:23,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +26: [2023-05-09 16:44:23,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +27: [2023-05-09 16:44:23,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +26: [2023-05-09 16:44:23,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +27: [2023-05-09 16:44:23,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +18: [2023-05-09 16:44:23,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +18: [2023-05-09 16:44:23,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +18: [2023-05-09 16:44:23,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +18: [2023-05-09 16:44:23,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +15: [2023-05-09 16:44:23,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +15: [2023-05-09 16:44:23,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +27: [2023-05-09 16:44:23,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +13: [2023-05-09 16:44:23,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +13: [2023-05-09 16:44:23,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +13: [2023-05-09 16:44:23,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +15: [2023-05-09 16:44:23,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +16: [2023-05-09 16:44:23,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +25: [2023-05-09 16:44:23,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +16: [2023-05-09 16:44:23,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +25: [2023-05-09 16:44:23,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +21: [2023-05-09 16:44:23,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +25: [2023-05-09 16:44:23,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +19: [2023-05-09 16:44:23,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +19: [2023-05-09 16:44:23,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +15: [2023-05-09 16:44:23,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +19: [2023-05-09 16:44:23,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +19: [2023-05-09 16:44:23,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +27: [2023-05-09 16:44:23,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +13: [2023-05-09 16:44:23,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +16: [2023-05-09 16:44:23,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +13: [2023-05-09 16:44:23,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +16: [2023-05-09 16:44:23,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +21: [2023-05-09 16:44:23,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +21: [2023-05-09 16:44:23,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +21: [2023-05-09 16:44:23,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 0: [2023-05-09 16:44:23,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 4: [2023-05-09 16:44:23,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +27: [2023-05-09 16:44:23,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +26: [2023-05-09 16:44:23,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +13: [2023-05-09 16:44:23,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +13: [2023-05-09 16:44:23,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +25: [2023-05-09 16:44:23,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 1: [2023-05-09 16:44:23,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +26: [2023-05-09 16:44:23,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +26: [2023-05-09 16:44:23,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +26: [2023-05-09 16:44:23,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +20: [2023-05-09 16:44:23,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +20: [2023-05-09 16:44:23,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +20: [2023-05-09 16:44:23,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +21: [2023-05-09 16:44:23,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +27: [2023-05-09 16:44:23,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 0: [2023-05-09 16:44:23,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 4: [2023-05-09 16:44:23,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +20: [2023-05-09 16:44:23,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +18: [2023-05-09 16:44:23,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +18: [2023-05-09 16:44:23,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +25: [2023-05-09 16:44:23,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +25: [2023-05-09 16:44:23,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +21: [2023-05-09 16:44:23,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +11: [2023-05-09 16:44:23,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +18: [2023-05-09 16:44:23,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +11: [2023-05-09 16:44:23,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +11: [2023-05-09 16:44:23,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 8: [2023-05-09 16:44:23,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +31: [2023-05-09 16:44:23,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 0: [2023-05-09 16:44:23,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +31: [2023-05-09 16:44:23,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +31: [2023-05-09 16:44:23,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +25: [2023-05-09 16:44:23,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +31: [2023-05-09 16:44:23,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +25: [2023-05-09 16:44:23,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +18: [2023-05-09 16:44:23,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +13: [2023-05-09 16:44:23,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +11: [2023-05-09 16:44:23,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +11: [2023-05-09 16:44:23,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +11: [2023-05-09 16:44:23,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 0: [2023-05-09 16:44:23,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +13: [2023-05-09 16:44:23,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +18: [2023-05-09 16:44:23,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +19: [2023-05-09 16:44:23,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +18: [2023-05-09 16:44:23,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +13: [2023-05-09 16:44:23,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +19: [2023-05-09 16:44:23,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +13: [2023-05-09 16:44:23,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +13: [2023-05-09 16:44:23,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +13: [2023-05-09 16:44:23,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +20: [2023-05-09 16:44:23,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +20: [2023-05-09 16:44:23,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +13: [2023-05-09 16:44:23,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +20: [2023-05-09 16:44:23,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 1: [2023-05-09 16:44:23,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +11: [2023-05-09 16:44:23,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +13: [2023-05-09 16:44:23,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +15: [2023-05-09 16:44:23,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +15: [2023-05-09 16:44:23,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +15: [2023-05-09 16:44:23,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +13: [2023-05-09 16:44:23,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +13: [2023-05-09 16:44:23,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +11: [2023-05-09 16:44:23,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +27: [2023-05-09 16:44:23,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +27: [2023-05-09 16:44:23,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +18: [2023-05-09 16:44:23,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +15: [2023-05-09 16:44:23,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +18: [2023-05-09 16:44:23,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +10: [2023-05-09 16:44:23,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +10: [2023-05-09 16:44:23,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +10: [2023-05-09 16:44:23,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +10: [2023-05-09 16:44:23,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +20: [2023-05-09 16:44:23,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +31: [2023-05-09 16:44:23,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +27: [2023-05-09 16:44:23,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +19: [2023-05-09 16:44:23,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +19: [2023-05-09 16:44:23,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 4: [2023-05-09 16:44:23,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 4: [2023-05-09 16:44:23,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +31: [2023-05-09 16:44:23,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +31: [2023-05-09 16:44:23,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +27: [2023-05-09 16:44:23,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. + 4: [2023-05-09 16:44:23,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt. +19: [2023-05-09 16:44:23,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +31: [2023-05-09 16:44:23,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +13: [2023-05-09 16:44:23,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +19: [2023-05-09 16:44:23,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 8: [2023-05-09 16:44:23,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +28: [2023-05-09 16:44:23,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +28: [2023-05-09 16:44:23,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +28: [2023-05-09 16:44:23,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +28: [2023-05-09 16:44:23,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +17: [2023-05-09 16:44:23,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +17: [2023-05-09 16:44:23,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +17: [2023-05-09 16:44:23,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +17: [2023-05-09 16:44:23,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +17: [2023-05-09 16:44:23,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +17: [2023-05-09 16:44:23,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +19: [2023-05-09 16:44:23,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +17: [2023-05-09 16:44:23,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +17: [2023-05-09 16:44:23,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +15: [2023-05-09 16:44:23,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +28: [2023-05-09 16:44:23,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +28: [2023-05-09 16:44:23,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +15: [2023-05-09 16:44:23,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +22: [2023-05-09 16:44:23,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +22: [2023-05-09 16:44:23,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +22: [2023-05-09 16:44:23,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +27: [2023-05-09 16:44:23,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +22: [2023-05-09 16:44:23,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 2: [2023-05-09 16:44:23,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +22: [2023-05-09 16:44:23,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +11: [2023-05-09 16:44:23,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +22: [2023-05-09 16:44:23,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +22: [2023-05-09 16:44:23,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +19: [2023-05-09 16:44:23,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +27: [2023-05-09 16:44:23,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +10: [2023-05-09 16:44:23,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 2: [2023-05-09 16:44:23,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +28: [2023-05-09 16:44:23,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +31: [2023-05-09 16:44:23,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +31: [2023-05-09 16:44:23,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +22: [2023-05-09 16:44:23,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +31: [2023-05-09 16:44:23,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +31: [2023-05-09 16:44:23,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +31: [2023-05-09 16:44:23,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +15: [2023-05-09 16:44:23,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +10: [2023-05-09 16:44:23,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +15: [2023-05-09 16:44:23,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +10: [2023-05-09 16:44:23,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +28: [2023-05-09 16:44:23,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +10: [2023-05-09 16:44:23,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 6: [2023-05-09 16:44:23,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +10: [2023-05-09 16:44:23,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +10: [2023-05-09 16:44:23,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +27: [2023-05-09 16:44:23,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +10: [2023-05-09 16:44:23,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +31: [2023-05-09 16:44:23,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +10: [2023-05-09 16:44:23,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +27: [2023-05-09 16:44:23,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +31: [2023-05-09 16:44:23,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +10: [2023-05-09 16:44:23,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +31: [2023-05-09 16:44:23,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +10: [2023-05-09 16:44:23,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 5: [2023-05-09 16:44:23,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 5: [2023-05-09 16:44:23,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +10: [2023-05-09 16:44:23,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +31: [2023-05-09 16:44:23,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +31: [2023-05-09 16:44:23,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +10: [2023-05-09 16:44:23,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... +18: [2023-05-09 16:44:23,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +18: [2023-05-09 16:44:23,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +18: [2023-05-09 16:44:23,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +18: [2023-05-09 16:44:23,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +18: [2023-05-09 16:44:23,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +18: [2023-05-09 16:44:23,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +18: [2023-05-09 16:44:23,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +30: [2023-05-09 16:44:23,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +18: [2023-05-09 16:44:23,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +30: [2023-05-09 16:44:23,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +30: [2023-05-09 16:44:23,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +30: [2023-05-09 16:44:23,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +25: [2023-05-09 16:44:23,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +30: [2023-05-09 16:44:23,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +25: [2023-05-09 16:44:23,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +30: [2023-05-09 16:44:23,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +30: [2023-05-09 16:44:23,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +30: [2023-05-09 16:44:23,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 4: [2023-05-09 16:44:23,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt... + 7: [2023-05-09 16:44:23,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +25: [2023-05-09 16:44:23,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +28: [2023-05-09 16:44:23,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 8: [2023-05-09 16:44:23,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 7: [2023-05-09 16:44:23,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +25: [2023-05-09 16:44:23,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +25: [2023-05-09 16:44:23,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 2: [2023-05-09 16:44:23,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +22: [2023-05-09 16:44:23,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +17: [2023-05-09 16:44:23,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +17: [2023-05-09 16:44:23,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +25: [2023-05-09 16:44:23,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +17: [2023-05-09 16:44:23,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +11: [2023-05-09 16:44:23,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +17: [2023-05-09 16:44:23,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +10: [2023-05-09 16:44:23,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +15: [2023-05-09 16:44:23,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +10: [2023-05-09 16:44:23,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 8: [2023-05-09 16:44:23,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +10: [2023-05-09 16:44:23,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +15: [2023-05-09 16:44:23,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +15: [2023-05-09 16:44:23,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +15: [2023-05-09 16:44:23,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +12: [2023-05-09 16:44:23,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +12: [2023-05-09 16:44:23,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +12: [2023-05-09 16:44:23,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 3: [2023-05-09 16:44:23,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 3: [2023-05-09 16:44:23,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +24: [2023-05-09 16:44:23,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +12: [2023-05-09 16:44:23,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +12: [2023-05-09 16:44:23,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +12: [2023-05-09 16:44:23,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +24: [2023-05-09 16:44:23,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +24: [2023-05-09 16:44:23,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +24: [2023-05-09 16:44:23,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +24: [2023-05-09 16:44:23,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +24: [2023-05-09 16:44:23,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +24: [2023-05-09 16:44:23,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +22: [2023-05-09 16:44:23,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +22: [2023-05-09 16:44:23,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 6: [2023-05-09 16:44:23,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +25: [2023-05-09 16:44:23,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +12: [2023-05-09 16:44:23,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 9: [2023-05-09 16:44:23,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 9: [2023-05-09 16:44:23,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +24: [2023-05-09 16:44:24,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +31: [2023-05-09 16:44:24,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +31: [2023-05-09 16:44:24,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +11: [2023-05-09 16:44:24,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +12: [2023-05-09 16:44:24,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +31: [2023-05-09 16:44:24,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +31: [2023-05-09 16:44:24,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +31: [2023-05-09 16:44:24,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +31: [2023-05-09 16:44:24,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +28: [2023-05-09 16:44:24,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +25: [2023-05-09 16:44:24,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +27: [2023-05-09 16:44:24,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +28: [2023-05-09 16:44:24,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +27: [2023-05-09 16:44:24,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 9: [2023-05-09 16:44:24,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +18: [2023-05-09 16:44:24,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +18: [2023-05-09 16:44:24,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +22: [2023-05-09 16:44:24,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +27: [2023-05-09 16:44:24,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +27: [2023-05-09 16:44:24,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +27: [2023-05-09 16:44:24,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +11: [2023-05-09 16:44:24,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +27: [2023-05-09 16:44:24,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +17: [2023-05-09 16:44:24,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +15: [2023-05-09 16:44:24,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +15: [2023-05-09 16:44:24,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +30: [2023-05-09 16:44:24,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +30: [2023-05-09 16:44:24,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +17: [2023-05-09 16:44:24,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +10: [2023-05-09 16:44:24,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +11: [2023-05-09 16:44:24,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +17: [2023-05-09 16:44:24,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +22: [2023-05-09 16:44:24,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +18: [2023-05-09 16:44:24,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +17: [2023-05-09 16:44:24,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +30: [2023-05-09 16:44:24,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +30: [2023-05-09 16:44:24,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +22: [2023-05-09 16:44:24,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 6: [2023-05-09 16:44:24,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +31: [2023-05-09 16:44:24,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +22: [2023-05-09 16:44:24,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +28: [2023-05-09 16:44:24,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +15: [2023-05-09 16:44:24,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +31: [2023-05-09 16:44:24,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +27: [2023-05-09 16:44:24,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +28: [2023-05-09 16:44:24,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +15: [2023-05-09 16:44:24,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +27: [2023-05-09 16:44:24,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 7: [2023-05-09 16:44:24,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +18: [2023-05-09 16:44:24,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +31: [2023-05-09 16:44:24,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +18: [2023-05-09 16:44:24,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... + 4: [2023-05-09 16:44:24,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +30: [2023-05-09 16:44:24,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +30: [2023-05-09 16:44:24,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +31: [2023-05-09 16:44:24,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +12: [2023-05-09 16:44:24,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +31: [2023-05-09 16:44:24,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +11: [2023-05-09 16:44:24,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +12: [2023-05-09 16:44:24,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +30: [2023-05-09 16:44:24,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +30: [2023-05-09 16:44:24,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +18: [2023-05-09 16:44:24,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +16: [2023-05-09 16:44:24,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +24: [2023-05-09 16:44:24,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +24: [2023-05-09 16:44:24,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +16: [2023-05-09 16:44:24,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +16: [2023-05-09 16:44:24,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 4: [2023-05-09 16:44:24,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +16: [2023-05-09 16:44:24,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +11: [2023-05-09 16:44:24,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +11: [2023-05-09 16:44:24,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +22: [2023-05-09 16:44:24,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +18: [2023-05-09 16:44:24,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +28: [2023-05-09 16:44:24,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +28: [2023-05-09 16:44:24,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +10: [2023-05-09 16:44:24,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +10: [2023-05-09 16:44:24,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +10: [2023-05-09 16:44:24,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +10: [2023-05-09 16:44:24,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +10: [2023-05-09 16:44:24,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +10: [2023-05-09 16:44:24,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +10: [2023-05-09 16:44:24,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +31: [2023-05-09 16:44:24,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +10: [2023-05-09 16:44:24,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +12: [2023-05-09 16:44:24,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +31: [2023-05-09 16:44:24,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +24: [2023-05-09 16:44:24,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt. +24: [2023-05-09 16:44:24,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +24: [2023-05-09 16:44:24,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +24: [2023-05-09 16:44:24,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt... +18: [2023-05-09 16:44:24,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +16: [2023-05-09 16:44:24,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +12: [2023-05-09 16:44:24,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +28: [2023-05-09 16:44:24,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +16: [2023-05-09 16:44:24,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +10: [2023-05-09 16:44:24,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +10: [2023-05-09 16:44:24,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +16: [2023-05-09 16:44:24,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +16: [2023-05-09 16:44:24,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +12: [2023-05-09 16:44:24,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +31: [2023-05-09 16:44:24,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +24: [2023-05-09 16:44:24,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +24: [2023-05-09 16:44:24,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +10: [2023-05-09 16:44:24,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +14: [2023-05-09 16:44:24,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +14: [2023-05-09 16:44:24,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +14: [2023-05-09 16:44:24,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +14: [2023-05-09 16:44:24,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +14: [2023-05-09 16:44:24,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +12: [2023-05-09 16:44:24,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +14: [2023-05-09 16:44:24,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +14: [2023-05-09 16:44:24,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +10: [2023-05-09 16:44:24,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +14: [2023-05-09 16:44:24,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +12: [2023-05-09 16:44:24,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +16: [2023-05-09 16:44:24,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +16: [2023-05-09 16:44:24,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 9: [2023-05-09 16:44:24,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +12: [2023-05-09 16:44:24,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +16: [2023-05-09 16:44:24,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +16: [2023-05-09 16:44:24,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +14: [2023-05-09 16:44:24,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +16: [2023-05-09 16:44:24,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +16: [2023-05-09 16:44:24,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +14: [2023-05-09 16:44:24,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +16: [2023-05-09 16:44:24,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +16: [2023-05-09 16:44:24,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 1: [2023-05-09 16:44:24,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 1: [2023-05-09 16:44:24,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 1: [2023-05-09 16:44:24,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 1: [2023-05-09 16:44:24,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +14: [2023-05-09 16:44:24,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +14: [2023-05-09 16:44:24,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +14: [2023-05-09 16:44:24,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +14: [2023-05-09 16:44:24,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +14: [2023-05-09 16:44:24,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +14: [2023-05-09 16:44:24,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +14: [2023-05-09 16:44:24,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +14: [2023-05-09 16:44:24,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +14: [2023-05-09 16:44:24,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +14: [2023-05-09 16:44:24,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +14: [2023-05-09 16:44:24,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +14: [2023-05-09 16:44:24,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +14: [2023-05-09 16:44:24,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +14: [2023-05-09 16:44:24,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +25: [2023-05-09 16:44:24,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +25: [2023-05-09 16:44:24,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +25: [2023-05-09 16:44:24,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +25: [2023-05-09 16:44:24,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +25: [2023-05-09 16:44:24,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +25: [2023-05-09 16:44:24,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +25: [2023-05-09 16:44:24,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +25: [2023-05-09 16:44:24,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +19: [2023-05-09 16:44:24,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +19: [2023-05-09 16:44:24,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +19: [2023-05-09 16:44:24,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +19: [2023-05-09 16:44:24,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +19: [2023-05-09 16:44:24,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +19: [2023-05-09 16:44:24,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +12: [2023-05-09 16:44:24,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +12: [2023-05-09 16:44:24,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +12: [2023-05-09 16:44:24,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 8: [2023-05-09 16:44:24,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +19: [2023-05-09 16:44:24,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +12: [2023-05-09 16:44:24,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +19: [2023-05-09 16:44:24,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 8: [2023-05-09 16:44:24,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +14: [2023-05-09 16:44:24,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +14: [2023-05-09 16:44:24,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +23: [2023-05-09 16:44:24,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +23: [2023-05-09 16:44:24,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +23: [2023-05-09 16:44:24,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +23: [2023-05-09 16:44:24,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +23: [2023-05-09 16:44:24,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +23: [2023-05-09 16:44:24,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 2: [2023-05-09 16:44:24,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +25: [2023-05-09 16:44:24,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +23: [2023-05-09 16:44:24,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +12: [2023-05-09 16:44:24,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +23: [2023-05-09 16:44:24,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +12: [2023-05-09 16:44:24,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +12: [2023-05-09 16:44:24,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +12: [2023-05-09 16:44:24,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +19: [2023-05-09 16:44:24,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +14: [2023-05-09 16:44:24,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +14: [2023-05-09 16:44:24,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 1: [2023-05-09 16:44:24,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +19: [2023-05-09 16:44:24,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +25: [2023-05-09 16:44:24,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 2: [2023-05-09 16:44:24,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +14: [2023-05-09 16:44:24,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +14: [2023-05-09 16:44:24,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +25: [2023-05-09 16:44:24,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +25: [2023-05-09 16:44:24,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +23: [2023-05-09 16:44:24,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +23: [2023-05-09 16:44:24,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +12: [2023-05-09 16:44:24,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +12: [2023-05-09 16:44:24,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +19: [2023-05-09 16:44:24,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +14: [2023-05-09 16:44:24,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +14: [2023-05-09 16:44:24,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 8: [2023-05-09 16:44:24,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +19: [2023-05-09 16:44:24,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +12: [2023-05-09 16:44:24,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +12: [2023-05-09 16:44:24,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +12: [2023-05-09 16:44:24,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +12: [2023-05-09 16:44:24,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 0: [2023-05-09 16:44:24,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +11: [2023-05-09 16:44:24,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +11: [2023-05-09 16:44:24,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +11: [2023-05-09 16:44:24,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +11: [2023-05-09 16:44:24,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +12: [2023-05-09 16:44:24,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +12: [2023-05-09 16:44:24,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +23: [2023-05-09 16:44:24,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +25: [2023-05-09 16:44:24,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +25: [2023-05-09 16:44:24,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +25: [2023-05-09 16:44:24,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +23: [2023-05-09 16:44:24,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +19: [2023-05-09 16:44:24,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +19: [2023-05-09 16:44:24,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +29: [2023-05-09 16:44:24,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +23: [2023-05-09 16:44:24,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +23: [2023-05-09 16:44:24,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +29: [2023-05-09 16:44:24,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +29: [2023-05-09 16:44:24,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +29: [2023-05-09 16:44:24,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +29: [2023-05-09 16:44:24,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +29: [2023-05-09 16:44:24,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +29: [2023-05-09 16:44:24,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +29: [2023-05-09 16:44:24,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 2: [2023-05-09 16:44:24,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 0: [2023-05-09 16:44:24,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +11: [2023-05-09 16:44:24,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +25: [2023-05-09 16:44:24,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +11: [2023-05-09 16:44:24,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +19: [2023-05-09 16:44:24,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +19: [2023-05-09 16:44:24,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +11: [2023-05-09 16:44:24,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +11: [2023-05-09 16:44:24,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +23: [2023-05-09 16:44:24,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +23: [2023-05-09 16:44:24,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +29: [2023-05-09 16:44:24,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +29: [2023-05-09 16:44:24,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +29: [2023-05-09 16:44:24,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +11: [2023-05-09 16:44:24,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +11: [2023-05-09 16:44:24,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +11: [2023-05-09 16:44:24,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +11: [2023-05-09 16:44:24,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +29: [2023-05-09 16:44:24,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +20: [2023-05-09 16:44:24,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +20: [2023-05-09 16:44:24,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +20: [2023-05-09 16:44:24,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +20: [2023-05-09 16:44:24,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +11: [2023-05-09 16:44:24,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +11: [2023-05-09 16:44:24,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +11: [2023-05-09 16:44:24,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +11: [2023-05-09 16:44:24,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +29: [2023-05-09 16:44:24,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +29: [2023-05-09 16:44:24,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +29: [2023-05-09 16:44:24,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +20: [2023-05-09 16:44:24,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +20: [2023-05-09 16:44:24,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +20: [2023-05-09 16:44:24,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +20: [2023-05-09 16:44:24,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +29: [2023-05-09 16:44:24,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 9: [2023-05-09 16:44:24,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 9: [2023-05-09 16:44:24,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 9: [2023-05-09 16:44:24,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +17: [2023-05-09 16:44:24,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +17: [2023-05-09 16:44:24,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +17: [2023-05-09 16:44:24,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +17: [2023-05-09 16:44:24,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +20: [2023-05-09 16:44:24,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +26: [2023-05-09 16:44:24,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +26: [2023-05-09 16:44:24,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +26: [2023-05-09 16:44:24,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +26: [2023-05-09 16:44:24,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +20: [2023-05-09 16:44:24,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 6: [2023-05-09 16:44:24,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +28: [2023-05-09 16:44:24,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 6: [2023-05-09 16:44:24,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 6: [2023-05-09 16:44:24,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 6: [2023-05-09 16:44:24,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +28: [2023-05-09 16:44:24,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +28: [2023-05-09 16:44:24,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +28: [2023-05-09 16:44:24,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +22: [2023-05-09 16:44:24,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +22: [2023-05-09 16:44:24,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +22: [2023-05-09 16:44:24,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +22: [2023-05-09 16:44:24,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 9: [2023-05-09 16:44:24,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +20: [2023-05-09 16:44:24,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +20: [2023-05-09 16:44:24,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +20: [2023-05-09 16:44:24,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +20: [2023-05-09 16:44:24,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 9: [2023-05-09 16:44:24,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +17: [2023-05-09 16:44:24,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +17: [2023-05-09 16:44:24,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +17: [2023-05-09 16:44:24,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +17: [2023-05-09 16:44:24,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +26: [2023-05-09 16:44:24,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +26: [2023-05-09 16:44:24,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +27: [2023-05-09 16:44:24,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +27: [2023-05-09 16:44:24,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +27: [2023-05-09 16:44:24,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +27: [2023-05-09 16:44:24,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +27: [2023-05-09 16:44:24,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +27: [2023-05-09 16:44:24,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +27: [2023-05-09 16:44:24,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +28: [2023-05-09 16:44:24,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +20: [2023-05-09 16:44:24,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +28: [2023-05-09 16:44:24,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +28: [2023-05-09 16:44:24,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +27: [2023-05-09 16:44:24,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +26: [2023-05-09 16:44:24,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +22: [2023-05-09 16:44:24,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +22: [2023-05-09 16:44:24,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +28: [2023-05-09 16:44:24,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +20: [2023-05-09 16:44:24,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +22: [2023-05-09 16:44:24,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +26: [2023-05-09 16:44:24,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +22: [2023-05-09 16:44:24,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +10: [2023-05-09 16:44:24,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +10: [2023-05-09 16:44:24,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +10: [2023-05-09 16:44:24,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +10: [2023-05-09 16:44:24,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +10: [2023-05-09 16:44:24,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +17: [2023-05-09 16:44:24,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +10: [2023-05-09 16:44:24,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +10: [2023-05-09 16:44:24,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +17: [2023-05-09 16:44:24,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +10: [2023-05-09 16:44:24,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +29: [2023-05-09 16:44:24,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +29: [2023-05-09 16:44:24,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 9: [2023-05-09 16:44:24,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +29: [2023-05-09 16:44:24,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 9: [2023-05-09 16:44:24,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 9: [2023-05-09 16:44:24,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +29: [2023-05-09 16:44:24,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +28: [2023-05-09 16:44:24,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +28: [2023-05-09 16:44:24,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +27: [2023-05-09 16:44:24,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +27: [2023-05-09 16:44:24,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +11: [2023-05-09 16:44:24,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +11: [2023-05-09 16:44:24,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +11: [2023-05-09 16:44:24,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +11: [2023-05-09 16:44:24,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +11: [2023-05-09 16:44:24,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +11: [2023-05-09 16:44:24,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +11: [2023-05-09 16:44:24,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +26: [2023-05-09 16:44:24,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +16: [2023-05-09 16:44:24,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +16: [2023-05-09 16:44:24,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +17: [2023-05-09 16:44:24,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +16: [2023-05-09 16:44:24,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +17: [2023-05-09 16:44:24,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +16: [2023-05-09 16:44:24,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 6: [2023-05-09 16:44:24,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 6: [2023-05-09 16:44:24,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +26: [2023-05-09 16:44:24,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +16: [2023-05-09 16:44:24,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +16: [2023-05-09 16:44:24,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +31: [2023-05-09 16:44:24,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +31: [2023-05-09 16:44:24,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +26: [2023-05-09 16:44:24,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +26: [2023-05-09 16:44:24,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +31: [2023-05-09 16:44:24,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +31: [2023-05-09 16:44:24,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +17: [2023-05-09 16:44:24,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +17: [2023-05-09 16:44:24,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +16: [2023-05-09 16:44:24,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +11: [2023-05-09 16:44:24,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +16: [2023-05-09 16:44:24,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +31: [2023-05-09 16:44:24,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +31: [2023-05-09 16:44:24,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +31: [2023-05-09 16:44:24,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +31: [2023-05-09 16:44:24,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +22: [2023-05-09 16:44:24,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +22: [2023-05-09 16:44:24,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +26: [2023-05-09 16:44:24,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +26: [2023-05-09 16:44:24,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +26: [2023-05-09 16:44:24,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +29: [2023-05-09 16:44:24,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +26: [2023-05-09 16:44:24,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +22: [2023-05-09 16:44:24,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +29: [2023-05-09 16:44:24,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +27: [2023-05-09 16:44:24,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +29: [2023-05-09 16:44:24,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +27: [2023-05-09 16:44:24,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +26: [2023-05-09 16:44:24,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +26: [2023-05-09 16:44:24,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +29: [2023-05-09 16:44:24,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +28: [2023-05-09 16:44:24,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +28: [2023-05-09 16:44:24,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 3: [2023-05-09 16:44:24,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 3: [2023-05-09 16:44:24,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +22: [2023-05-09 16:44:24,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +22: [2023-05-09 16:44:24,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +15: [2023-05-09 16:44:24,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +15: [2023-05-09 16:44:24,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +15: [2023-05-09 16:44:24,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +22: [2023-05-09 16:44:24,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +15: [2023-05-09 16:44:24,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +15: [2023-05-09 16:44:24,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +15: [2023-05-09 16:44:24,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +15: [2023-05-09 16:44:24,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +27: [2023-05-09 16:44:24,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +17: [2023-05-09 16:44:24,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +17: [2023-05-09 16:44:24,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +26: [2023-05-09 16:44:24,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +26: [2023-05-09 16:44:24,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +15: [2023-05-09 16:44:24,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +21: [2023-05-09 16:44:24,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +21: [2023-05-09 16:44:24,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +10: [2023-05-09 16:44:24,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +28: [2023-05-09 16:44:24,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +28: [2023-05-09 16:44:24,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +28: [2023-05-09 16:44:24,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +22: [2023-05-09 16:44:24,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +22: [2023-05-09 16:44:24,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +27: [2023-05-09 16:44:24,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +28: [2023-05-09 16:44:24,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +26: [2023-05-09 16:44:24,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +21: [2023-05-09 16:44:24,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 6: [2023-05-09 16:44:24,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +21: [2023-05-09 16:44:24,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 6: [2023-05-09 16:44:24,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +26: [2023-05-09 16:44:24,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 6: [2023-05-09 16:44:24,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +16: [2023-05-09 16:44:24,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +16: [2023-05-09 16:44:24,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +10: [2023-05-09 16:44:24,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +31: [2023-05-09 16:44:24,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +10: [2023-05-09 16:44:24,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +31: [2023-05-09 16:44:24,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +31: [2023-05-09 16:44:24,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 8: [2023-05-09 16:44:24,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +27: [2023-05-09 16:44:24,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +27: [2023-05-09 16:44:24,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +10: [2023-05-09 16:44:24,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +21: [2023-05-09 16:44:24,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +29: [2023-05-09 16:44:24,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +21: [2023-05-09 16:44:24,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +29: [2023-05-09 16:44:24,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 3: [2023-05-09 16:44:24,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +15: [2023-05-09 16:44:24,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +11: [2023-05-09 16:44:24,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +11: [2023-05-09 16:44:24,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +11: [2023-05-09 16:44:24,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +21: [2023-05-09 16:44:24,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +21: [2023-05-09 16:44:24,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +10: [2023-05-09 16:44:24,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +26: [2023-05-09 16:44:24,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +26: [2023-05-09 16:44:24,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +16: [2023-05-09 16:44:24,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +16: [2023-05-09 16:44:24,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +15: [2023-05-09 16:44:24,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +15: [2023-05-09 16:44:24,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +10: [2023-05-09 16:44:24,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +26: [2023-05-09 16:44:24,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +10: [2023-05-09 16:44:24,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +29: [2023-05-09 16:44:24,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +29: [2023-05-09 16:44:24,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +31: [2023-05-09 16:44:24,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +29: [2023-05-09 16:44:24,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +26: [2023-05-09 16:44:24,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 8: [2023-05-09 16:44:24,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +31: [2023-05-09 16:44:24,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +31: [2023-05-09 16:44:24,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +16: [2023-05-09 16:44:24,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +29: [2023-05-09 16:44:24,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +30: [2023-05-09 16:44:24,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +15: [2023-05-09 16:44:24,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +30: [2023-05-09 16:44:24,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +30: [2023-05-09 16:44:24,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 8: [2023-05-09 16:44:24,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +30: [2023-05-09 16:44:24,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +16: [2023-05-09 16:44:24,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +29: [2023-05-09 16:44:24,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +29: [2023-05-09 16:44:24,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 6: [2023-05-09 16:44:24,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +12: [2023-05-09 16:44:24,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +12: [2023-05-09 16:44:24,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +12: [2023-05-09 16:44:24,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +12: [2023-05-09 16:44:24,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +12: [2023-05-09 16:44:24,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +12: [2023-05-09 16:44:24,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +12: [2023-05-09 16:44:24,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +12: [2023-05-09 16:44:24,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +10: [2023-05-09 16:44:24,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +26: [2023-05-09 16:44:24,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +26: [2023-05-09 16:44:24,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +26: [2023-05-09 16:44:24,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +31: [2023-05-09 16:44:24,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +26: [2023-05-09 16:44:24,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +11: [2023-05-09 16:44:24,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +11: [2023-05-09 16:44:24,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +11: [2023-05-09 16:44:24,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +15: [2023-05-09 16:44:24,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +15: [2023-05-09 16:44:24,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +21: [2023-05-09 16:44:24,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +21: [2023-05-09 16:44:24,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +16: [2023-05-09 16:44:24,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +21: [2023-05-09 16:44:24,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +21: [2023-05-09 16:44:24,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +28: [2023-05-09 16:44:24,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +28: [2023-05-09 16:44:24,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +28: [2023-05-09 16:44:24,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +28: [2023-05-09 16:44:24,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +28: [2023-05-09 16:44:24,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +28: [2023-05-09 16:44:24,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +28: [2023-05-09 16:44:24,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +28: [2023-05-09 16:44:24,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +21: [2023-05-09 16:44:24,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +21: [2023-05-09 16:44:24,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +30: [2023-05-09 16:44:24,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +16: [2023-05-09 16:44:24,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +30: [2023-05-09 16:44:24,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +21: [2023-05-09 16:44:24,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +21: [2023-05-09 16:44:24,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +21: [2023-05-09 16:44:24,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +21: [2023-05-09 16:44:24,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +21: [2023-05-09 16:44:24,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +21: [2023-05-09 16:44:24,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 7: [2023-05-09 16:44:24,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 7: [2023-05-09 16:44:24,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +13: [2023-05-09 16:44:24,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +13: [2023-05-09 16:44:24,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 6: [2023-05-09 16:44:24,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +13: [2023-05-09 16:44:24,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +30: [2023-05-09 16:44:24,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +13: [2023-05-09 16:44:24,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +13: [2023-05-09 16:44:24,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +13: [2023-05-09 16:44:24,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +21: [2023-05-09 16:44:24,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +21: [2023-05-09 16:44:24,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +30: [2023-05-09 16:44:24,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +31: [2023-05-09 16:44:24,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 0: [2023-05-09 16:44:24,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 0: [2023-05-09 16:44:24,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 2: [2023-05-09 16:44:24,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +13: [2023-05-09 16:44:24,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +13: [2023-05-09 16:44:24,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +26: [2023-05-09 16:44:24,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +15: [2023-05-09 16:44:24,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +26: [2023-05-09 16:44:24,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +12: [2023-05-09 16:44:24,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +11: [2023-05-09 16:44:24,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +18: [2023-05-09 16:44:24,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +18: [2023-05-09 16:44:24,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +18: [2023-05-09 16:44:24,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +18: [2023-05-09 16:44:24,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +25: [2023-05-09 16:44:24,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 6: [2023-05-09 16:44:24,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +25: [2023-05-09 16:44:24,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 1: [2023-05-09 16:44:24,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +22: [2023-05-09 16:44:24,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +22: [2023-05-09 16:44:24,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +22: [2023-05-09 16:44:24,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +18: [2023-05-09 16:44:24,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +18: [2023-05-09 16:44:24,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +22: [2023-05-09 16:44:24,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +22: [2023-05-09 16:44:24,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +18: [2023-05-09 16:44:24,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +22: [2023-05-09 16:44:24,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +22: [2023-05-09 16:44:24,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +22: [2023-05-09 16:44:24,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +18: [2023-05-09 16:44:24,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +15: [2023-05-09 16:44:24,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +15: [2023-05-09 16:44:24,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +15: [2023-05-09 16:44:24,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 6: [2023-05-09 16:44:24,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +25: [2023-05-09 16:44:24,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 5: [2023-05-09 16:44:24,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +25: [2023-05-09 16:44:24,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 5: [2023-05-09 16:44:24,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 5: [2023-05-09 16:44:24,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +30: [2023-05-09 16:44:24,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +30: [2023-05-09 16:44:24,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +30: [2023-05-09 16:44:24,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +30: [2023-05-09 16:44:24,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +30: [2023-05-09 16:44:24,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +30: [2023-05-09 16:44:24,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +30: [2023-05-09 16:44:24,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +17: [2023-05-09 16:44:24,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +30: [2023-05-09 16:44:24,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +17: [2023-05-09 16:44:24,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +21: [2023-05-09 16:44:24,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +21: [2023-05-09 16:44:24,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 7: [2023-05-09 16:44:24,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +17: [2023-05-09 16:44:24,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +28: [2023-05-09 16:44:24,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +24: [2023-05-09 16:44:24,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +24: [2023-05-09 16:44:24,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +21: [2023-05-09 16:44:24,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +24: [2023-05-09 16:44:24,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +24: [2023-05-09 16:44:24,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +24: [2023-05-09 16:44:24,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +24: [2023-05-09 16:44:24,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +24: [2023-05-09 16:44:24,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +24: [2023-05-09 16:44:24,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +30: [2023-05-09 16:44:24,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +30: [2023-05-09 16:44:24,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +13: [2023-05-09 16:44:24,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +17: [2023-05-09 16:44:24,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +17: [2023-05-09 16:44:24,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +17: [2023-05-09 16:44:24,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +17: [2023-05-09 16:44:24,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +15: [2023-05-09 16:44:24,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 7: [2023-05-09 16:44:24,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +12: [2023-05-09 16:44:24,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +12: [2023-05-09 16:44:24,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +10: [2023-05-09 16:44:24,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +17: [2023-05-09 16:44:24,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +11: [2023-05-09 16:44:24,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +15: [2023-05-09 16:44:24,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +10: [2023-05-09 16:44:24,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +10: [2023-05-09 16:44:24,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +10: [2023-05-09 16:44:24,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +12: [2023-05-09 16:44:24,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +28: [2023-05-09 16:44:24,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +28: [2023-05-09 16:44:24,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +28: [2023-05-09 16:44:24,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +13: [2023-05-09 16:44:24,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +22: [2023-05-09 16:44:24,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +25: [2023-05-09 16:44:24,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +25: [2023-05-09 16:44:24,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +18: [2023-05-09 16:44:24,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +22: [2023-05-09 16:44:24,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +28: [2023-05-09 16:44:24,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +18: [2023-05-09 16:44:24,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +18: [2023-05-09 16:44:24,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +30: [2023-05-09 16:44:24,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +25: [2023-05-09 16:44:24,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +25: [2023-05-09 16:44:24,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +15: [2023-05-09 16:44:24,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +12: [2023-05-09 16:44:24,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +22: [2023-05-09 16:44:24,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +30: [2023-05-09 16:44:24,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +15: [2023-05-09 16:44:24,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +17: [2023-05-09 16:44:24,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +24: [2023-05-09 16:44:24,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +15: [2023-05-09 16:44:24,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +15: [2023-05-09 16:44:24,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +30: [2023-05-09 16:44:24,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +22: [2023-05-09 16:44:24,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +21: [2023-05-09 16:44:24,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +21: [2023-05-09 16:44:24,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +21: [2023-05-09 16:44:24,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +21: [2023-05-09 16:44:24,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +13: [2023-05-09 16:44:24,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +12: [2023-05-09 16:44:24,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 2: [2023-05-09 16:44:24,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +30: [2023-05-09 16:44:24,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +30: [2023-05-09 16:44:24,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +30: [2023-05-09 16:44:24,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +12: [2023-05-09 16:44:24,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +17: [2023-05-09 16:44:24,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +28: [2023-05-09 16:44:24,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +22: [2023-05-09 16:44:24,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +16: [2023-05-09 16:44:24,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +16: [2023-05-09 16:44:24,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +18: [2023-05-09 16:44:24,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +10: [2023-05-09 16:44:24,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +10: [2023-05-09 16:44:24,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +16: [2023-05-09 16:44:24,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +16: [2023-05-09 16:44:24,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 2: [2023-05-09 16:44:24,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 5: [2023-05-09 16:44:24,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 7: [2023-05-09 16:44:24,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +28: [2023-05-09 16:44:24,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +30: [2023-05-09 16:44:24,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +30: [2023-05-09 16:44:24,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +10: [2023-05-09 16:44:24,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +10: [2023-05-09 16:44:24,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +28: [2023-05-09 16:44:24,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 7: [2023-05-09 16:44:24,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +13: [2023-05-09 16:44:24,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +18: [2023-05-09 16:44:24,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +18: [2023-05-09 16:44:24,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +22: [2023-05-09 16:44:24,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +22: [2023-05-09 16:44:24,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +30: [2023-05-09 16:44:24,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +13: [2023-05-09 16:44:24,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +22: [2023-05-09 16:44:24,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +24: [2023-05-09 16:44:24,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +24: [2023-05-09 16:44:24,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +21: [2023-05-09 16:44:24,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +17: [2023-05-09 16:44:24,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +24: [2023-05-09 16:44:24,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +24: [2023-05-09 16:44:24,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +30: [2023-05-09 16:44:24,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +30: [2023-05-09 16:44:24,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +21: [2023-05-09 16:44:24,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +25: [2023-05-09 16:44:24,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +12: [2023-05-09 16:44:24,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 4: [2023-05-09 16:44:24,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +25: [2023-05-09 16:44:24,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +18: [2023-05-09 16:44:24,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +30: [2023-05-09 16:44:24,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 2: [2023-05-09 16:44:24,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +21: [2023-05-09 16:44:24,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +17: [2023-05-09 16:44:24,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +17: [2023-05-09 16:44:24,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +16: [2023-05-09 16:44:24,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +16: [2023-05-09 16:44:24,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +16: [2023-05-09 16:44:24,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +13: [2023-05-09 16:44:24,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +30: [2023-05-09 16:44:24,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +30: [2023-05-09 16:44:24,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +16: [2023-05-09 16:44:24,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +25: [2023-05-09 16:44:24,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +25: [2023-05-09 16:44:24,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +25: [2023-05-09 16:44:24,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +25: [2023-05-09 16:44:24,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +25: [2023-05-09 16:44:24,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +25: [2023-05-09 16:44:24,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 7: [2023-05-09 16:44:24,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +13: [2023-05-09 16:44:24,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +17: [2023-05-09 16:44:24,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +24: [2023-05-09 16:44:24,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +18: [2023-05-09 16:44:24,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +15: [2023-05-09 16:44:24,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +15: [2023-05-09 16:44:24,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +24: [2023-05-09 16:44:24,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +15: [2023-05-09 16:44:24,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +15: [2023-05-09 16:44:24,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 2: [2023-05-09 16:44:24,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +15: [2023-05-09 16:44:24,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +15: [2023-05-09 16:44:24,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +24: [2023-05-09 16:44:24,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +17: [2023-05-09 16:44:24,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +10: [2023-05-09 16:44:24,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +10: [2023-05-09 16:44:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +13: [2023-05-09 16:44:24,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +10: [2023-05-09 16:44:24,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +16: [2023-05-09 16:44:24,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +17: [2023-05-09 16:44:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +10: [2023-05-09 16:44:24,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +10: [2023-05-09 16:44:24,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +16: [2023-05-09 16:44:24,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +10: [2023-05-09 16:44:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 9: [2023-05-09 16:44:24,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +15: [2023-05-09 16:44:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +15: [2023-05-09 16:44:24,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 7: [2023-05-09 16:44:24,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 5: [2023-05-09 16:44:24,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 5: [2023-05-09 16:44:24,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +10: [2023-05-09 16:44:24,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 5: [2023-05-09 16:44:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +10: [2023-05-09 16:44:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +16: [2023-05-09 16:44:24,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +16: [2023-05-09 16:44:24,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +16: [2023-05-09 16:44:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +16: [2023-05-09 16:44:24,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 7: [2023-05-09 16:44:24,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +31: [2023-05-09 16:44:24,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +31: [2023-05-09 16:44:24,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +31: [2023-05-09 16:44:24,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +16: [2023-05-09 16:44:24,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +19: [2023-05-09 16:44:24,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +19: [2023-05-09 16:44:24,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 3: [2023-05-09 16:44:24,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 3: [2023-05-09 16:44:24,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 3: [2023-05-09 16:44:24,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +31: [2023-05-09 16:44:24,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 3: [2023-05-09 16:44:24,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +19: [2023-05-09 16:44:24,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +19: [2023-05-09 16:44:24,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +16: [2023-05-09 16:44:24,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +14: [2023-05-09 16:44:24,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +14: [2023-05-09 16:44:24,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +14: [2023-05-09 16:44:24,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +14: [2023-05-09 16:44:24,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +31: [2023-05-09 16:44:24,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +19: [2023-05-09 16:44:24,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +31: [2023-05-09 16:44:24,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +31: [2023-05-09 16:44:24,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +31: [2023-05-09 16:44:24,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +19: [2023-05-09 16:44:24,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +19: [2023-05-09 16:44:24,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +19: [2023-05-09 16:44:24,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +31: [2023-05-09 16:44:24,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +14: [2023-05-09 16:44:24,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +14: [2023-05-09 16:44:24,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +31: [2023-05-09 16:44:24,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 4: [2023-05-09 16:44:24,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 4: [2023-05-09 16:44:24,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 4: [2023-05-09 16:44:24,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +19: [2023-05-09 16:44:24,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +14: [2023-05-09 16:44:24,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +14: [2023-05-09 16:44:24,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +19: [2023-05-09 16:44:24,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 4: [2023-05-09 16:44:24,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +24: [2023-05-09 16:44:24,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +24: [2023-05-09 16:44:24,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +24: [2023-05-09 16:44:24,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +24: [2023-05-09 16:44:24,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +19: [2023-05-09 16:44:24,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +19: [2023-05-09 16:44:24,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +19: [2023-05-09 16:44:24,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +19: [2023-05-09 16:44:24,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +31: [2023-05-09 16:44:24,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +31: [2023-05-09 16:44:24,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +31: [2023-05-09 16:44:24,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +31: [2023-05-09 16:44:24,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +20: [2023-05-09 16:44:24,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +20: [2023-05-09 16:44:24,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +20: [2023-05-09 16:44:24,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +20: [2023-05-09 16:44:24,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +20: [2023-05-09 16:44:24,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +20: [2023-05-09 16:44:24,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +20: [2023-05-09 16:44:24,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +20: [2023-05-09 16:44:24,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +23: [2023-05-09 16:44:24,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 3: [2023-05-09 16:44:24,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +23: [2023-05-09 16:44:24,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +23: [2023-05-09 16:44:24,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +23: [2023-05-09 16:44:24,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 3: [2023-05-09 16:44:24,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 3: [2023-05-09 16:44:24,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +18: [2023-05-09 16:44:24,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +18: [2023-05-09 16:44:24,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 4: [2023-05-09 16:44:24,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +18: [2023-05-09 16:44:24,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +27: [2023-05-09 16:44:24,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +27: [2023-05-09 16:44:24,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +27: [2023-05-09 16:44:24,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +13: [2023-05-09 16:44:24,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +13: [2023-05-09 16:44:24,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 4: [2023-05-09 16:44:24,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +27: [2023-05-09 16:44:24,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. + 4: [2023-05-09 16:44:24,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +24: [2023-05-09 16:44:24,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +24: [2023-05-09 16:44:24,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +24: [2023-05-09 16:44:24,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +18: [2023-05-09 16:44:24,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +31: [2023-05-09 16:44:24,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +13: [2023-05-09 16:44:24,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +19: [2023-05-09 16:44:24,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +13: [2023-05-09 16:44:24,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt. +14: [2023-05-09 16:44:24,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +14: [2023-05-09 16:44:24,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +31: [2023-05-09 16:44:24,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +24: [2023-05-09 16:44:24,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +19: [2023-05-09 16:44:24,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +23: [2023-05-09 16:44:24,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +20: [2023-05-09 16:44:24,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +23: [2023-05-09 16:44:24,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +20: [2023-05-09 16:44:24,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +23: [2023-05-09 16:44:24,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +23: [2023-05-09 16:44:24,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +14: [2023-05-09 16:44:24,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +14: [2023-05-09 16:44:24,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +27: [2023-05-09 16:44:24,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +20: [2023-05-09 16:44:24,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +27: [2023-05-09 16:44:24,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +27: [2023-05-09 16:44:24,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +13: [2023-05-09 16:44:24,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +20: [2023-05-09 16:44:24,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +13: [2023-05-09 16:44:24,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +18: [2023-05-09 16:44:24,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +18: [2023-05-09 16:44:24,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 2: [2023-05-09 16:44:24,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 2: [2023-05-09 16:44:24,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 2: [2023-05-09 16:44:24,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +18: [2023-05-09 16:44:24,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +27: [2023-05-09 16:44:24,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +14: [2023-05-09 16:44:24,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +18: [2023-05-09 16:44:24,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +14: [2023-05-09 16:44:24,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +24: [2023-05-09 16:44:24,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +13: [2023-05-09 16:44:24,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +13: [2023-05-09 16:44:24,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt... +24: [2023-05-09 16:44:24,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +24: [2023-05-09 16:44:24,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +24: [2023-05-09 16:44:24,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +24: [2023-05-09 16:44:24,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +24: [2023-05-09 16:44:24,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +20: [2023-05-09 16:44:24,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +20: [2023-05-09 16:44:24,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +14: [2023-05-09 16:44:24,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +23: [2023-05-09 16:44:24,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +14: [2023-05-09 16:44:24,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 2: [2023-05-09 16:44:24,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +20: [2023-05-09 16:44:24,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +20: [2023-05-09 16:44:24,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +23: [2023-05-09 16:44:24,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +27: [2023-05-09 16:44:24,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +23: [2023-05-09 16:44:24,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +27: [2023-05-09 16:44:24,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +23: [2023-05-09 16:44:24,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 4: [2023-05-09 16:44:24,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 2: [2023-05-09 16:44:24,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +13: [2023-05-09 16:44:24,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +18: [2023-05-09 16:44:24,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +13: [2023-05-09 16:44:24,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +18: [2023-05-09 16:44:24,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +18: [2023-05-09 16:44:24,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +27: [2023-05-09 16:44:24,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +18: [2023-05-09 16:44:24,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +27: [2023-05-09 16:44:24,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 4: [2023-05-09 16:44:24,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +23: [2023-05-09 16:44:24,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +23: [2023-05-09 16:44:24,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +23: [2023-05-09 16:44:24,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +23: [2023-05-09 16:44:24,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +13: [2023-05-09 16:44:24,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +13: [2023-05-09 16:44:24,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +13: [2023-05-09 16:44:24,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +13: [2023-05-09 16:44:24,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 4: [2023-05-09 16:44:24,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +18: [2023-05-09 16:44:24,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +27: [2023-05-09 16:44:24,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +18: [2023-05-09 16:44:24,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +27: [2023-05-09 16:44:24,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 0: [2023-05-09 16:44:24,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +13: [2023-05-09 16:44:24,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +13: [2023-05-09 16:44:24,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 2: [2023-05-09 16:44:24,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +24: [2023-05-09 16:44:24,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 4: [2023-05-09 16:44:24,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 1: [2023-05-09 16:44:24,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 1: [2023-05-09 16:44:24,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 1: [2023-05-09 16:44:24,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +24: [2023-05-09 16:44:24,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +27: [2023-05-09 16:44:24,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 0: [2023-05-09 16:44:24,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +27: [2023-05-09 16:44:24,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... +18: [2023-05-09 16:44:24,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt. +18: [2023-05-09 16:44:24,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt... + 2: [2023-05-09 16:44:24,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 2: [2023-05-09 16:44:24,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 1: [2023-05-09 16:44:24,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +14: [2023-05-09 16:44:24,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +14: [2023-05-09 16:44:24,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +14: [2023-05-09 16:44:24,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +19: [2023-05-09 16:44:24,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +19: [2023-05-09 16:44:24,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +19: [2023-05-09 16:44:24,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +19: [2023-05-09 16:44:24,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +14: [2023-05-09 16:44:24,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +27: [2023-05-09 16:44:24,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +27: [2023-05-09 16:44:24,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +27: [2023-05-09 16:44:24,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +25: [2023-05-09 16:44:24,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +25: [2023-05-09 16:44:24,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +25: [2023-05-09 16:44:24,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +14: [2023-05-09 16:44:24,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +19: [2023-05-09 16:44:24,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +25: [2023-05-09 16:44:24,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +14: [2023-05-09 16:44:24,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +25: [2023-05-09 16:44:24,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +19: [2023-05-09 16:44:24,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +19: [2023-05-09 16:44:24,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +14: [2023-05-09 16:44:24,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +25: [2023-05-09 16:44:24,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +25: [2023-05-09 16:44:24,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +19: [2023-05-09 16:44:24,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +27: [2023-05-09 16:44:24,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +27: [2023-05-09 16:44:24,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +27: [2023-05-09 16:44:24,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +27: [2023-05-09 16:44:24,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +25: [2023-05-09 16:44:24,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +14: [2023-05-09 16:44:24,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +27: [2023-05-09 16:44:24,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +12: [2023-05-09 16:44:24,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +12: [2023-05-09 16:44:24,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +12: [2023-05-09 16:44:24,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +12: [2023-05-09 16:44:24,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 0: [2023-05-09 16:44:24,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +14: [2023-05-09 16:44:24,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +25: [2023-05-09 16:44:24,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +14: [2023-05-09 16:44:24,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +12: [2023-05-09 16:44:24,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +19: [2023-05-09 16:44:24,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +12: [2023-05-09 16:44:24,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +12: [2023-05-09 16:44:24,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +27: [2023-05-09 16:44:24,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +12: [2023-05-09 16:44:24,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 1: [2023-05-09 16:44:24,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 1: [2023-05-09 16:44:24,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +14: [2023-05-09 16:44:24,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +25: [2023-05-09 16:44:24,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +25: [2023-05-09 16:44:24,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +19: [2023-05-09 16:44:24,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +19: [2023-05-09 16:44:24,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +19: [2023-05-09 16:44:24,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +27: [2023-05-09 16:44:24,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +25: [2023-05-09 16:44:24,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +27: [2023-05-09 16:44:24,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +19: [2023-05-09 16:44:24,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +14: [2023-05-09 16:44:24,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +14: [2023-05-09 16:44:24,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +27: [2023-05-09 16:44:24,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +23: [2023-05-09 16:44:24,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +23: [2023-05-09 16:44:24,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +23: [2023-05-09 16:44:24,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +23: [2023-05-09 16:44:24,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +23: [2023-05-09 16:44:24,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +23: [2023-05-09 16:44:24,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +23: [2023-05-09 16:44:24,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +23: [2023-05-09 16:44:24,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +14: [2023-05-09 16:44:24,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +19: [2023-05-09 16:44:24,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +12: [2023-05-09 16:44:24,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +12: [2023-05-09 16:44:24,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 0: [2023-05-09 16:44:24,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +27: [2023-05-09 16:44:24,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +25: [2023-05-09 16:44:24,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +25: [2023-05-09 16:44:24,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +19: [2023-05-09 16:44:24,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +27: [2023-05-09 16:44:24,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +19: [2023-05-09 16:44:24,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +27: [2023-05-09 16:44:24,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +12: [2023-05-09 16:44:24,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +12: [2023-05-09 16:44:24,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 8: [2023-05-09 16:44:24,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +14: [2023-05-09 16:44:24,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +12: [2023-05-09 16:44:24,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +12: [2023-05-09 16:44:24,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +25: [2023-05-09 16:44:24,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +11: [2023-05-09 16:44:24,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +11: [2023-05-09 16:44:24,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +11: [2023-05-09 16:44:24,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +11: [2023-05-09 16:44:24,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +12: [2023-05-09 16:44:24,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +27: [2023-05-09 16:44:24,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +12: [2023-05-09 16:44:24,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +23: [2023-05-09 16:44:24,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +23: [2023-05-09 16:44:24,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +23: [2023-05-09 16:44:24,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +23: [2023-05-09 16:44:24,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +14: [2023-05-09 16:44:24,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +11: [2023-05-09 16:44:24,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +25: [2023-05-09 16:44:24,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +11: [2023-05-09 16:44:24,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +11: [2023-05-09 16:44:24,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +23: [2023-05-09 16:44:24,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +23: [2023-05-09 16:44:24,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +11: [2023-05-09 16:44:24,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +23: [2023-05-09 16:44:24,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +23: [2023-05-09 16:44:24,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +20: [2023-05-09 16:44:24,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +20: [2023-05-09 16:44:24,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +20: [2023-05-09 16:44:24,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +20: [2023-05-09 16:44:24,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +11: [2023-05-09 16:44:24,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +11: [2023-05-09 16:44:24,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +17: [2023-05-09 16:44:24,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +29: [2023-05-09 16:44:24,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +29: [2023-05-09 16:44:24,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +29: [2023-05-09 16:44:24,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +17: [2023-05-09 16:44:24,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +17: [2023-05-09 16:44:24,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +29: [2023-05-09 16:44:24,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +21: [2023-05-09 16:44:24,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +21: [2023-05-09 16:44:24,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +17: [2023-05-09 16:44:24,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +21: [2023-05-09 16:44:24,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +21: [2023-05-09 16:44:24,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +22: [2023-05-09 16:44:24,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +22: [2023-05-09 16:44:24,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +22: [2023-05-09 16:44:24,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +22: [2023-05-09 16:44:24,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +28: [2023-05-09 16:44:24,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +16: [2023-05-09 16:44:24,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +16: [2023-05-09 16:44:24,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +16: [2023-05-09 16:44:24,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +16: [2023-05-09 16:44:24,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +28: [2023-05-09 16:44:24,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +28: [2023-05-09 16:44:24,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +16: [2023-05-09 16:44:24,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +16: [2023-05-09 16:44:24,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +16: [2023-05-09 16:44:24,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +28: [2023-05-09 16:44:24,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +16: [2023-05-09 16:44:24,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +20: [2023-05-09 16:44:24,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +31: [2023-05-09 16:44:24,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +11: [2023-05-09 16:44:24,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +31: [2023-05-09 16:44:24,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +31: [2023-05-09 16:44:24,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +11: [2023-05-09 16:44:24,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +11: [2023-05-09 16:44:24,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +31: [2023-05-09 16:44:24,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +31: [2023-05-09 16:44:24,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +29: [2023-05-09 16:44:24,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +31: [2023-05-09 16:44:24,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +29: [2023-05-09 16:44:24,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +29: [2023-05-09 16:44:24,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +20: [2023-05-09 16:44:24,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +11: [2023-05-09 16:44:24,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +29: [2023-05-09 16:44:24,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +31: [2023-05-09 16:44:24,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +29: [2023-05-09 16:44:24,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +29: [2023-05-09 16:44:24,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +20: [2023-05-09 16:44:24,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +20: [2023-05-09 16:44:24,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +31: [2023-05-09 16:44:24,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +29: [2023-05-09 16:44:24,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +22: [2023-05-09 16:44:24,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +22: [2023-05-09 16:44:24,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +29: [2023-05-09 16:44:24,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +29: [2023-05-09 16:44:24,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +22: [2023-05-09 16:44:24,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +29: [2023-05-09 16:44:24,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +22: [2023-05-09 16:44:24,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +21: [2023-05-09 16:44:24,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +17: [2023-05-09 16:44:24,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +28: [2023-05-09 16:44:24,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +17: [2023-05-09 16:44:24,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +17: [2023-05-09 16:44:24,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +28: [2023-05-09 16:44:24,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +29: [2023-05-09 16:44:24,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +28: [2023-05-09 16:44:24,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +29: [2023-05-09 16:44:24,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +17: [2023-05-09 16:44:24,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +21: [2023-05-09 16:44:24,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +21: [2023-05-09 16:44:24,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +28: [2023-05-09 16:44:24,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +16: [2023-05-09 16:44:24,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +21: [2023-05-09 16:44:24,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +16: [2023-05-09 16:44:24,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 9: [2023-05-09 16:44:24,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 9: [2023-05-09 16:44:24,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +16: [2023-05-09 16:44:24,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +16: [2023-05-09 16:44:24,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +11: [2023-05-09 16:44:24,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +31: [2023-05-09 16:44:24,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +11: [2023-05-09 16:44:24,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +31: [2023-05-09 16:44:24,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +31: [2023-05-09 16:44:24,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +20: [2023-05-09 16:44:24,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +29: [2023-05-09 16:44:24,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +20: [2023-05-09 16:44:24,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +22: [2023-05-09 16:44:24,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +22: [2023-05-09 16:44:24,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +21: [2023-05-09 16:44:24,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +29: [2023-05-09 16:44:24,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +22: [2023-05-09 16:44:24,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +22: [2023-05-09 16:44:24,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +28: [2023-05-09 16:44:24,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +21: [2023-05-09 16:44:24,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +16: [2023-05-09 16:44:24,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +29: [2023-05-09 16:44:24,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 6: [2023-05-09 16:44:24,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 6: [2023-05-09 16:44:24,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +20: [2023-05-09 16:44:24,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +20: [2023-05-09 16:44:24,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +28: [2023-05-09 16:44:24,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +31: [2023-05-09 16:44:24,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +21: [2023-05-09 16:44:24,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +29: [2023-05-09 16:44:24,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +21: [2023-05-09 16:44:24,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +21: [2023-05-09 16:44:24,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +17: [2023-05-09 16:44:24,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +21: [2023-05-09 16:44:24,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +29: [2023-05-09 16:44:24,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +16: [2023-05-09 16:44:24,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +17: [2023-05-09 16:44:24,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +29: [2023-05-09 16:44:24,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +17: [2023-05-09 16:44:24,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +17: [2023-05-09 16:44:24,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +29: [2023-05-09 16:44:24,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +29: [2023-05-09 16:44:24,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +16: [2023-05-09 16:44:24,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +26: [2023-05-09 16:44:24,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +26: [2023-05-09 16:44:24,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +26: [2023-05-09 16:44:24,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +26: [2023-05-09 16:44:24,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +29: [2023-05-09 16:44:24,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +22: [2023-05-09 16:44:24,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +22: [2023-05-09 16:44:24,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +28: [2023-05-09 16:44:24,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +31: [2023-05-09 16:44:24,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +28: [2023-05-09 16:44:24,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +28: [2023-05-09 16:44:24,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +28: [2023-05-09 16:44:24,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +22: [2023-05-09 16:44:24,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +29: [2023-05-09 16:44:24,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +31: [2023-05-09 16:44:24,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +22: [2023-05-09 16:44:24,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +16: [2023-05-09 16:44:24,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +30: [2023-05-09 16:44:24,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 3: [2023-05-09 16:44:24,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +20: [2023-05-09 16:44:24,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +30: [2023-05-09 16:44:24,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +31: [2023-05-09 16:44:24,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +30: [2023-05-09 16:44:24,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 3: [2023-05-09 16:44:24,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +30: [2023-05-09 16:44:24,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +20: [2023-05-09 16:44:24,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +17: [2023-05-09 16:44:24,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +17: [2023-05-09 16:44:24,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +17: [2023-05-09 16:44:24,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +29: [2023-05-09 16:44:24,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +28: [2023-05-09 16:44:24,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +29: [2023-05-09 16:44:24,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +28: [2023-05-09 16:44:24,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +29: [2023-05-09 16:44:24,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +17: [2023-05-09 16:44:24,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 6: [2023-05-09 16:44:24,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +20: [2023-05-09 16:44:24,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +26: [2023-05-09 16:44:24,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +20: [2023-05-09 16:44:24,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +26: [2023-05-09 16:44:24,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +26: [2023-05-09 16:44:24,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +26: [2023-05-09 16:44:24,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +11: [2023-05-09 16:44:24,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +11: [2023-05-09 16:44:24,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +29: [2023-05-09 16:44:24,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +11: [2023-05-09 16:44:24,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +31: [2023-05-09 16:44:24,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +30: [2023-05-09 16:44:24,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +11: [2023-05-09 16:44:24,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +11: [2023-05-09 16:44:24,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +30: [2023-05-09 16:44:24,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +30: [2023-05-09 16:44:24,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +30: [2023-05-09 16:44:24,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +21: [2023-05-09 16:44:24,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +11: [2023-05-09 16:44:24,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +11: [2023-05-09 16:44:24,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +11: [2023-05-09 16:44:24,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +21: [2023-05-09 16:44:24,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 9: [2023-05-09 16:44:24,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 8: [2023-05-09 16:44:24,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 8: [2023-05-09 16:44:24,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 8: [2023-05-09 16:44:24,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 8: [2023-05-09 16:44:24,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +26: [2023-05-09 16:44:24,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +29: [2023-05-09 16:44:24,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +26: [2023-05-09 16:44:24,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 6: [2023-05-09 16:44:24,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +21: [2023-05-09 16:44:24,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +21: [2023-05-09 16:44:24,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +21: [2023-05-09 16:44:24,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +21: [2023-05-09 16:44:24,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +30: [2023-05-09 16:44:24,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +30: [2023-05-09 16:44:24,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +30: [2023-05-09 16:44:24,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +30: [2023-05-09 16:44:24,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +21: [2023-05-09 16:44:24,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +21: [2023-05-09 16:44:24,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +21: [2023-05-09 16:44:24,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 6: [2023-05-09 16:44:24,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +26: [2023-05-09 16:44:24,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +30: [2023-05-09 16:44:24,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +26: [2023-05-09 16:44:24,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 9: [2023-05-09 16:44:24,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +30: [2023-05-09 16:44:24,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 9: [2023-05-09 16:44:24,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +26: [2023-05-09 16:44:24,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +26: [2023-05-09 16:44:24,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 8: [2023-05-09 16:44:24,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +13: [2023-05-09 16:44:24,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +13: [2023-05-09 16:44:24,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +11: [2023-05-09 16:44:24,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +13: [2023-05-09 16:44:24,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +13: [2023-05-09 16:44:24,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +21: [2023-05-09 16:44:24,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +29: [2023-05-09 16:44:24,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 9: [2023-05-09 16:44:24,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +26: [2023-05-09 16:44:24,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +13: [2023-05-09 16:44:24,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +26: [2023-05-09 16:44:24,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +30: [2023-05-09 16:44:24,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +30: [2023-05-09 16:44:24,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +13: [2023-05-09 16:44:24,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +13: [2023-05-09 16:44:24,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +13: [2023-05-09 16:44:24,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +21: [2023-05-09 16:44:24,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +21: [2023-05-09 16:44:24,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +11: [2023-05-09 16:44:24,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +13: [2023-05-09 16:44:24,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +13: [2023-05-09 16:44:24,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +11: [2023-05-09 16:44:24,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +11: [2023-05-09 16:44:24,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 9: [2023-05-09 16:44:24,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +21: [2023-05-09 16:44:24,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +21: [2023-05-09 16:44:24,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +11: [2023-05-09 16:44:24,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 8: [2023-05-09 16:44:24,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 9: [2023-05-09 16:44:24,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +21: [2023-05-09 16:44:24,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +15: [2023-05-09 16:44:24,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +13: [2023-05-09 16:44:24,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +11: [2023-05-09 16:44:24,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +15: [2023-05-09 16:44:24,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +15: [2023-05-09 16:44:24,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +13: [2023-05-09 16:44:24,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +15: [2023-05-09 16:44:24,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +15: [2023-05-09 16:44:24,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +15: [2023-05-09 16:44:24,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +15: [2023-05-09 16:44:24,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +11: [2023-05-09 16:44:24,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +28: [2023-05-09 16:44:24,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +28: [2023-05-09 16:44:24,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +13: [2023-05-09 16:44:24,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +28: [2023-05-09 16:44:24,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +28: [2023-05-09 16:44:24,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +28: [2023-05-09 16:44:24,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +28: [2023-05-09 16:44:24,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +28: [2023-05-09 16:44:24,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +15: [2023-05-09 16:44:24,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +10: [2023-05-09 16:44:24,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +10: [2023-05-09 16:44:24,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +10: [2023-05-09 16:44:24,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +10: [2023-05-09 16:44:24,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +10: [2023-05-09 16:44:24,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +10: [2023-05-09 16:44:24,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 1: [2023-05-09 16:44:24,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 0: [2023-05-09 16:44:24,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +10: [2023-05-09 16:44:24,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +10: [2023-05-09 16:44:24,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +28: [2023-05-09 16:44:24,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +18: [2023-05-09 16:44:24,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +18: [2023-05-09 16:44:24,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +18: [2023-05-09 16:44:24,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +18: [2023-05-09 16:44:24,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +18: [2023-05-09 16:44:24,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +18: [2023-05-09 16:44:24,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +18: [2023-05-09 16:44:24,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +18: [2023-05-09 16:44:24,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +21: [2023-05-09 16:44:24,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +11: [2023-05-09 16:44:24,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +21: [2023-05-09 16:44:24,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +13: [2023-05-09 16:44:24,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +15: [2023-05-09 16:44:24,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +21: [2023-05-09 16:44:24,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +15: [2023-05-09 16:44:24,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +15: [2023-05-09 16:44:24,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +10: [2023-05-09 16:44:24,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +28: [2023-05-09 16:44:24,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +28: [2023-05-09 16:44:24,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +18: [2023-05-09 16:44:24,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +13: [2023-05-09 16:44:24,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +18: [2023-05-09 16:44:24,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +10: [2023-05-09 16:44:24,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +10: [2023-05-09 16:44:24,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +15: [2023-05-09 16:44:24,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +15: [2023-05-09 16:44:24,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +28: [2023-05-09 16:44:24,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +10: [2023-05-09 16:44:24,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +28: [2023-05-09 16:44:24,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +15: [2023-05-09 16:44:24,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +25: [2023-05-09 16:44:24,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 4: [2023-05-09 16:44:24,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +28: [2023-05-09 16:44:24,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +25: [2023-05-09 16:44:24,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +25: [2023-05-09 16:44:24,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +25: [2023-05-09 16:44:24,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +10: [2023-05-09 16:44:24,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +18: [2023-05-09 16:44:24,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +13: [2023-05-09 16:44:24,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 7: [2023-05-09 16:44:24,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 7: [2023-05-09 16:44:24,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 7: [2023-05-09 16:44:24,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +18: [2023-05-09 16:44:24,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +18: [2023-05-09 16:44:24,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +15: [2023-05-09 16:44:24,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +10: [2023-05-09 16:44:24,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +28: [2023-05-09 16:44:24,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +24: [2023-05-09 16:44:24,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +24: [2023-05-09 16:44:24,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +24: [2023-05-09 16:44:24,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +24: [2023-05-09 16:44:24,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +24: [2023-05-09 16:44:24,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +24: [2023-05-09 16:44:24,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +24: [2023-05-09 16:44:24,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +18: [2023-05-09 16:44:24,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +24: [2023-05-09 16:44:24,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +25: [2023-05-09 16:44:24,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +10: [2023-05-09 16:44:24,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +28: [2023-05-09 16:44:24,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +10: [2023-05-09 16:44:24,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +25: [2023-05-09 16:44:24,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +19: [2023-05-09 16:44:24,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +19: [2023-05-09 16:44:24,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 7: [2023-05-09 16:44:24,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +19: [2023-05-09 16:44:24,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 7: [2023-05-09 16:44:24,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +19: [2023-05-09 16:44:24,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +16: [2023-05-09 16:44:24,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +16: [2023-05-09 16:44:24,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +16: [2023-05-09 16:44:24,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 0: [2023-05-09 16:44:24,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +16: [2023-05-09 16:44:24,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 3: [2023-05-09 16:44:24,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 3: [2023-05-09 16:44:24,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +25: [2023-05-09 16:44:24,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +25: [2023-05-09 16:44:24,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 0: [2023-05-09 16:44:24,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 0: [2023-05-09 16:44:24,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 3: [2023-05-09 16:44:24,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +18: [2023-05-09 16:44:24,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +15: [2023-05-09 16:44:24,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +28: [2023-05-09 16:44:24,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 0: [2023-05-09 16:44:24,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 2: [2023-05-09 16:44:24,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +27: [2023-05-09 16:44:24,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +27: [2023-05-09 16:44:24,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 2: [2023-05-09 16:44:24,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +27: [2023-05-09 16:44:24,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 1: [2023-05-09 16:44:24,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +27: [2023-05-09 16:44:24,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +18: [2023-05-09 16:44:24,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +17: [2023-05-09 16:44:24,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +17: [2023-05-09 16:44:24,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +17: [2023-05-09 16:44:24,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +17: [2023-05-09 16:44:24,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +17: [2023-05-09 16:44:24,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +17: [2023-05-09 16:44:24,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +17: [2023-05-09 16:44:24,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +17: [2023-05-09 16:44:24,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +19: [2023-05-09 16:44:24,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +19: [2023-05-09 16:44:24,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +25: [2023-05-09 16:44:24,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +25: [2023-05-09 16:44:24,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 3: [2023-05-09 16:44:24,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +16: [2023-05-09 16:44:24,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +16: [2023-05-09 16:44:24,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +12: [2023-05-09 16:44:24,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +12: [2023-05-09 16:44:24,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +12: [2023-05-09 16:44:24,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +12: [2023-05-09 16:44:24,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +12: [2023-05-09 16:44:24,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +19: [2023-05-09 16:44:24,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +12: [2023-05-09 16:44:24,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +16: [2023-05-09 16:44:24,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +12: [2023-05-09 16:44:24,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +12: [2023-05-09 16:44:24,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +27: [2023-05-09 16:44:24,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +24: [2023-05-09 16:44:24,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +24: [2023-05-09 16:44:24,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +19: [2023-05-09 16:44:24,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +16: [2023-05-09 16:44:24,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +27: [2023-05-09 16:44:24,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +26: [2023-05-09 16:44:24,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +26: [2023-05-09 16:44:24,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +26: [2023-05-09 16:44:24,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +26: [2023-05-09 16:44:24,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +26: [2023-05-09 16:44:24,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +26: [2023-05-09 16:44:24,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +26: [2023-05-09 16:44:24,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +26: [2023-05-09 16:44:24,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +27: [2023-05-09 16:44:24,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +27: [2023-05-09 16:44:24,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +22: [2023-05-09 16:44:24,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 7: [2023-05-09 16:44:24,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +22: [2023-05-09 16:44:24,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +22: [2023-05-09 16:44:24,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +22: [2023-05-09 16:44:24,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +22: [2023-05-09 16:44:24,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +22: [2023-05-09 16:44:24,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +22: [2023-05-09 16:44:24,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +22: [2023-05-09 16:44:24,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +17: [2023-05-09 16:44:24,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 7: [2023-05-09 16:44:24,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 6: [2023-05-09 16:44:24,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +24: [2023-05-09 16:44:24,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +24: [2023-05-09 16:44:24,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +24: [2023-05-09 16:44:24,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +19: [2023-05-09 16:44:24,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +19: [2023-05-09 16:44:24,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +17: [2023-05-09 16:44:24,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +17: [2023-05-09 16:44:24,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +19: [2023-05-09 16:44:24,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +19: [2023-05-09 16:44:24,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 5: [2023-05-09 16:44:24,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +25: [2023-05-09 16:44:24,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +17: [2023-05-09 16:44:24,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +25: [2023-05-09 16:44:24,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 5: [2023-05-09 16:44:24,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 3: [2023-05-09 16:44:24,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +16: [2023-05-09 16:44:24,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 6: [2023-05-09 16:44:24,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +27: [2023-05-09 16:44:24,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +16: [2023-05-09 16:44:24,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +22: [2023-05-09 16:44:24,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +27: [2023-05-09 16:44:24,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 3: [2023-05-09 16:44:24,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +17: [2023-05-09 16:44:24,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +24: [2023-05-09 16:44:24,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +30: [2023-05-09 16:44:24,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +30: [2023-05-09 16:44:24,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +30: [2023-05-09 16:44:24,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +30: [2023-05-09 16:44:24,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +12: [2023-05-09 16:44:24,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +12: [2023-05-09 16:44:24,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +30: [2023-05-09 16:44:24,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +30: [2023-05-09 16:44:24,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +27: [2023-05-09 16:44:24,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +27: [2023-05-09 16:44:24,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +30: [2023-05-09 16:44:24,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +30: [2023-05-09 16:44:24,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +16: [2023-05-09 16:44:24,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +16: [2023-05-09 16:44:24,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +18: [2023-05-09 16:44:24,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +26: [2023-05-09 16:44:24,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +26: [2023-05-09 16:44:24,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +18: [2023-05-09 16:44:24,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +18: [2023-05-09 16:44:24,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 2: [2023-05-09 16:44:24,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +26: [2023-05-09 16:44:24,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +26: [2023-05-09 16:44:24,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +23: [2023-05-09 16:44:24,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +23: [2023-05-09 16:44:24,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +23: [2023-05-09 16:44:24,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +23: [2023-05-09 16:44:24,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +14: [2023-05-09 16:44:24,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +14: [2023-05-09 16:44:24,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +24: [2023-05-09 16:44:24,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +14: [2023-05-09 16:44:24,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +18: [2023-05-09 16:44:24,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +14: [2023-05-09 16:44:24,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 3: [2023-05-09 16:44:24,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +22: [2023-05-09 16:44:24,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +22: [2023-05-09 16:44:24,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +17: [2023-05-09 16:44:24,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +16: [2023-05-09 16:44:24,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +17: [2023-05-09 16:44:24,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +27: [2023-05-09 16:44:24,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +16: [2023-05-09 16:44:24,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 6: [2023-05-09 16:44:24,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +19: [2023-05-09 16:44:24,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +27: [2023-05-09 16:44:24,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +19: [2023-05-09 16:44:24,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +17: [2023-05-09 16:44:24,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +25: [2023-05-09 16:44:24,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 4: [2023-05-09 16:44:24,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 4: [2023-05-09 16:44:24,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +22: [2023-05-09 16:44:24,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +12: [2023-05-09 16:44:24,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +19: [2023-05-09 16:44:24,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +24: [2023-05-09 16:44:24,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +19: [2023-05-09 16:44:24,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +25: [2023-05-09 16:44:24,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +15: [2023-05-09 16:44:24,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +15: [2023-05-09 16:44:24,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +15: [2023-05-09 16:44:24,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +25: [2023-05-09 16:44:24,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 5: [2023-05-09 16:44:24,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +27: [2023-05-09 16:44:24,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +15: [2023-05-09 16:44:24,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +27: [2023-05-09 16:44:24,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +25: [2023-05-09 16:44:24,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +18: [2023-05-09 16:44:24,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +12: [2023-05-09 16:44:24,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +26: [2023-05-09 16:44:24,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +12: [2023-05-09 16:44:24,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 1: [2023-05-09 16:44:24,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 1: [2023-05-09 16:44:24,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +16: [2023-05-09 16:44:24,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +23: [2023-05-09 16:44:24,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +23: [2023-05-09 16:44:24,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +14: [2023-05-09 16:44:24,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +14: [2023-05-09 16:44:24,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +18: [2023-05-09 16:44:24,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +14: [2023-05-09 16:44:24,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +14: [2023-05-09 16:44:24,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +16: [2023-05-09 16:44:24,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +23: [2023-05-09 16:44:24,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +30: [2023-05-09 16:44:24,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +30: [2023-05-09 16:44:24,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +18: [2023-05-09 16:44:24,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +18: [2023-05-09 16:44:24,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +26: [2023-05-09 16:44:24,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +26: [2023-05-09 16:44:24,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +23: [2023-05-09 16:44:24,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +22: [2023-05-09 16:44:24,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +22: [2023-05-09 16:44:24,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +26: [2023-05-09 16:44:24,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +30: [2023-05-09 16:44:24,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +30: [2023-05-09 16:44:24,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +12: [2023-05-09 16:44:24,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +22: [2023-05-09 16:44:24,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +15: [2023-05-09 16:44:24,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +15: [2023-05-09 16:44:24,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +12: [2023-05-09 16:44:24,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +12: [2023-05-09 16:44:24,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +12: [2023-05-09 16:44:24,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +12: [2023-05-09 16:44:24,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +12: [2023-05-09 16:44:24,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +15: [2023-05-09 16:44:24,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +12: [2023-05-09 16:44:24,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +15: [2023-05-09 16:44:24,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +12: [2023-05-09 16:44:24,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +12: [2023-05-09 16:44:24,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +12: [2023-05-09 16:44:24,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +12: [2023-05-09 16:44:24,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +24: [2023-05-09 16:44:24,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +24: [2023-05-09 16:44:24,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +24: [2023-05-09 16:44:24,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 7: [2023-05-09 16:44:24,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +30: [2023-05-09 16:44:24,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +30: [2023-05-09 16:44:24,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +24: [2023-05-09 16:44:24,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 1: [2023-05-09 16:44:24,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +25: [2023-05-09 16:44:24,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +25: [2023-05-09 16:44:24,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +25: [2023-05-09 16:44:24,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +25: [2023-05-09 16:44:24,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +25: [2023-05-09 16:44:24,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +25: [2023-05-09 16:44:24,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +18: [2023-05-09 16:44:24,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +25: [2023-05-09 16:44:24,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 5: [2023-05-09 16:44:24,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 5: [2023-05-09 16:44:24,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +18: [2023-05-09 16:44:24,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 5: [2023-05-09 16:44:24,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +14: [2023-05-09 16:44:24,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +30: [2023-05-09 16:44:24,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +30: [2023-05-09 16:44:24,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +14: [2023-05-09 16:44:24,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +25: [2023-05-09 16:44:24,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +23: [2023-05-09 16:44:24,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +23: [2023-05-09 16:44:24,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +23: [2023-05-09 16:44:24,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +23: [2023-05-09 16:44:24,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +22: [2023-05-09 16:44:24,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +12: [2023-05-09 16:44:24,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +18: [2023-05-09 16:44:24,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +18: [2023-05-09 16:44:24,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +12: [2023-05-09 16:44:24,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +12: [2023-05-09 16:44:24,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +12: [2023-05-09 16:44:24,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +14: [2023-05-09 16:44:24,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +14: [2023-05-09 16:44:24,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +14: [2023-05-09 16:44:24,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +14: [2023-05-09 16:44:24,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +14: [2023-05-09 16:44:24,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +14: [2023-05-09 16:44:24,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +24: [2023-05-09 16:44:24,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +14: [2023-05-09 16:44:24,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +14: [2023-05-09 16:44:24,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +24: [2023-05-09 16:44:24,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +27: [2023-05-09 16:44:24,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +27: [2023-05-09 16:44:24,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +27: [2023-05-09 16:44:24,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +27: [2023-05-09 16:44:24,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +27: [2023-05-09 16:44:24,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +27: [2023-05-09 16:44:24,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +27: [2023-05-09 16:44:24,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +27: [2023-05-09 16:44:24,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +23: [2023-05-09 16:44:24,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +23: [2023-05-09 16:44:24,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +23: [2023-05-09 16:44:24,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +23: [2023-05-09 16:44:24,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +20: [2023-05-09 16:44:24,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +20: [2023-05-09 16:44:24,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +20: [2023-05-09 16:44:24,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +23: [2023-05-09 16:44:24,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +20: [2023-05-09 16:44:24,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +20: [2023-05-09 16:44:24,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +24: [2023-05-09 16:44:24,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +20: [2023-05-09 16:44:24,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +20: [2023-05-09 16:44:24,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +24: [2023-05-09 16:44:24,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +14: [2023-05-09 16:44:24,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +14: [2023-05-09 16:44:24,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +14: [2023-05-09 16:44:24,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +14: [2023-05-09 16:44:24,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +20: [2023-05-09 16:44:24,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +23: [2023-05-09 16:44:24,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +23: [2023-05-09 16:44:24,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +23: [2023-05-09 16:44:24,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +15: [2023-05-09 16:44:24,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +15: [2023-05-09 16:44:24,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +15: [2023-05-09 16:44:24,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +15: [2023-05-09 16:44:24,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +15: [2023-05-09 16:44:24,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +15: [2023-05-09 16:44:24,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 5: [2023-05-09 16:44:24,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +23: [2023-05-09 16:44:24,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +23: [2023-05-09 16:44:24,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 5: [2023-05-09 16:44:24,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +25: [2023-05-09 16:44:24,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +23: [2023-05-09 16:44:24,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +18: [2023-05-09 16:44:24,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +18: [2023-05-09 16:44:24,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +23: [2023-05-09 16:44:24,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +18: [2023-05-09 16:44:24,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +18: [2023-05-09 16:44:24,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 4: [2023-05-09 16:44:24,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 4: [2023-05-09 16:44:24,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 1: [2023-05-09 16:44:24,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +15: [2023-05-09 16:44:24,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +15: [2023-05-09 16:44:24,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +14: [2023-05-09 16:44:24,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +31: [2023-05-09 16:44:24,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +31: [2023-05-09 16:44:24,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +31: [2023-05-09 16:44:24,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +31: [2023-05-09 16:44:24,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +31: [2023-05-09 16:44:24,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +31: [2023-05-09 16:44:24,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +31: [2023-05-09 16:44:24,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +31: [2023-05-09 16:44:24,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +24: [2023-05-09 16:44:24,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +24: [2023-05-09 16:44:24,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 0: [2023-05-09 16:44:24,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +24: [2023-05-09 16:44:24,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +24: [2023-05-09 16:44:24,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +14: [2023-05-09 16:44:24,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +14: [2023-05-09 16:44:24,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +25: [2023-05-09 16:44:24,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +27: [2023-05-09 16:44:24,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +20: [2023-05-09 16:44:24,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +20: [2023-05-09 16:44:24,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +27: [2023-05-09 16:44:24,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +14: [2023-05-09 16:44:24,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +24: [2023-05-09 16:44:24,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +24: [2023-05-09 16:44:24,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +20: [2023-05-09 16:44:24,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +23: [2023-05-09 16:44:24,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +25: [2023-05-09 16:44:24,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +25: [2023-05-09 16:44:24,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +25: [2023-05-09 16:44:24,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +25: [2023-05-09 16:44:24,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +25: [2023-05-09 16:44:24,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +27: [2023-05-09 16:44:24,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +14: [2023-05-09 16:44:24,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +27: [2023-05-09 16:44:24,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +12: [2023-05-09 16:44:24,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt... +12: [2023-05-09 16:44:24,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt... +12: [2023-05-09 16:44:24,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt... +12: [2023-05-09 16:44:24,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt... + 4: [2023-05-09 16:44:24,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 1: [2023-05-09 16:44:24,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +31: [2023-05-09 16:44:24,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +14: [2023-05-09 16:44:24,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +14: [2023-05-09 16:44:24,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +14: [2023-05-09 16:44:24,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +14: [2023-05-09 16:44:24,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +14: [2023-05-09 16:44:24,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +14: [2023-05-09 16:44:24,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +14: [2023-05-09 16:44:24,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +23: [2023-05-09 16:44:24,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +14: [2023-05-09 16:44:24,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 5: [2023-05-09 16:44:24,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +31: [2023-05-09 16:44:24,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +13: [2023-05-09 16:44:24,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +13: [2023-05-09 16:44:24,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +13: [2023-05-09 16:44:24,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 0: [2023-05-09 16:44:24,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +31: [2023-05-09 16:44:24,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +27: [2023-05-09 16:44:24,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +13: [2023-05-09 16:44:24,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +20: [2023-05-09 16:44:24,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +24: [2023-05-09 16:44:24,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +24: [2023-05-09 16:44:24,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... + 6: [2023-05-09 16:44:24,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt... + 6: [2023-05-09 16:44:24,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt... + 6: [2023-05-09 16:44:24,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt... + 6: [2023-05-09 16:44:24,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt... +23: [2023-05-09 16:44:24,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +23: [2023-05-09 16:44:24,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +23: [2023-05-09 16:44:24,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +23: [2023-05-09 16:44:24,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +25: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +23: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +20: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +20: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +25: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +25: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +27: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +25: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +14: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +25: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +23: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +20: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +20: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +25: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +25: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +20: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +20: [2023-05-09 16:44:24,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +20: [2023-05-09 16:44:24,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +20: [2023-05-09 16:44:24,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +20: [2023-05-09 16:44:24,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +25: [2023-05-09 16:44:24,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +20: [2023-05-09 16:44:24,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +20: [2023-05-09 16:44:24,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +20: [2023-05-09 16:44:24,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +31: [2023-05-09 16:44:24,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. + 5: [2023-05-09 16:44:24,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +31: [2023-05-09 16:44:24,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +31: [2023-05-09 16:44:24,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +31: [2023-05-09 16:44:24,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt. +27: [2023-05-09 16:44:24,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +27: [2023-05-09 16:44:24,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +27: [2023-05-09 16:44:24,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +27: [2023-05-09 16:44:24,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +27: [2023-05-09 16:44:24,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +27: [2023-05-09 16:44:24,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +27: [2023-05-09 16:44:24,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +27: [2023-05-09 16:44:24,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +27: [2023-05-09 16:44:24,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +27: [2023-05-09 16:44:24,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +27: [2023-05-09 16:44:24,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +27: [2023-05-09 16:44:24,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +27: [2023-05-09 16:44:24,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +27: [2023-05-09 16:44:24,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +31: [2023-05-09 16:44:24,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +25: [2023-05-09 16:44:24,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +14: [2023-05-09 16:44:24,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +13: [2023-05-09 16:44:24,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +14: [2023-05-09 16:44:24,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +14: [2023-05-09 16:44:24,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +14: [2023-05-09 16:44:24,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +13: [2023-05-09 16:44:24,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +13: [2023-05-09 16:44:24,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +20: [2023-05-09 16:44:24,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +23: [2023-05-09 16:44:24,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +13: [2023-05-09 16:44:24,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +20: [2023-05-09 16:44:24,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +31: [2023-05-09 16:44:24,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +20: [2023-05-09 16:44:24,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +23: [2023-05-09 16:44:24,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +14: [2023-05-09 16:44:24,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +23: [2023-05-09 16:44:24,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +23: [2023-05-09 16:44:24,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +20: [2023-05-09 16:44:24,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +23: [2023-05-09 16:44:24,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +23: [2023-05-09 16:44:24,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +14: [2023-05-09 16:44:24,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +14: [2023-05-09 16:44:24,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +23: [2023-05-09 16:44:24,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +31: [2023-05-09 16:44:24,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +23: [2023-05-09 16:44:24,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +14: [2023-05-09 16:44:24,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +23: [2023-05-09 16:44:24,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +23: [2023-05-09 16:44:24,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +23: [2023-05-09 16:44:24,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +31: [2023-05-09 16:44:24,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +23: [2023-05-09 16:44:24,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +31: [2023-05-09 16:44:24,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +31: [2023-05-09 16:44:24,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +31: [2023-05-09 16:44:24,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +31: [2023-05-09 16:44:24,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +31: [2023-05-09 16:44:24,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +31: [2023-05-09 16:44:24,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +31: [2023-05-09 16:44:24,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt... +31: [2023-05-09 16:44:24,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +31: [2023-05-09 16:44:24,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +31: [2023-05-09 16:44:24,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +31: [2023-05-09 16:44:24,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +31: [2023-05-09 16:44:24,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +31: [2023-05-09 16:44:24,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +25: [2023-05-09 16:44:24,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +25: [2023-05-09 16:44:24,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +25: [2023-05-09 16:44:24,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +25: [2023-05-09 16:44:24,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +13: [2023-05-09 16:44:24,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +13: [2023-05-09 16:44:24,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +31: [2023-05-09 16:44:24,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +31: [2023-05-09 16:44:24,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +31: [2023-05-09 16:44:24,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +31: [2023-05-09 16:44:24,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +13: [2023-05-09 16:44:24,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +13: [2023-05-09 16:44:24,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +19: [2023-05-09 16:44:24,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +19: [2023-05-09 16:44:24,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +31: [2023-05-09 16:44:24,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +19: [2023-05-09 16:44:24,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +19: [2023-05-09 16:44:24,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +19: [2023-05-09 16:44:24,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +19: [2023-05-09 16:44:24,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +31: [2023-05-09 16:44:24,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +25: [2023-05-09 16:44:24,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt... +25: [2023-05-09 16:44:24,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt... +25: [2023-05-09 16:44:24,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt... +25: [2023-05-09 16:44:24,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt... + 0: [2023-05-09 16:44:24,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +19: [2023-05-09 16:44:24,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +29: [2023-05-09 16:44:24,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +29: [2023-05-09 16:44:24,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +16: [2023-05-09 16:44:24,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +16: [2023-05-09 16:44:24,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +29: [2023-05-09 16:44:24,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +16: [2023-05-09 16:44:24,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +16: [2023-05-09 16:44:24,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +29: [2023-05-09 16:44:24,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +29: [2023-05-09 16:44:24,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +29: [2023-05-09 16:44:24,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +19: [2023-05-09 16:44:24,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +16: [2023-05-09 16:44:24,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +16: [2023-05-09 16:44:24,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +29: [2023-05-09 16:44:24,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +29: [2023-05-09 16:44:24,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +16: [2023-05-09 16:44:24,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 0: > overriding learning rate value to 0.0002 + 0: > overriding minimum learning rate value to 2e-05 + 0: > overriding warmup iterations value to 0 + 0: > overriding total number of iterations value to 1 + 0: > overriding decay style value to cosine +16: [2023-05-09 16:44:24,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +11: [2023-05-09 16:44:24,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +11: [2023-05-09 16:44:24,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +11: [2023-05-09 16:44:24,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +11: [2023-05-09 16:44:24,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +11: [2023-05-09 16:44:24,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +11: [2023-05-09 16:44:24,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... + 0: [2023-05-09 16:44:24,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... + 0: [2023-05-09 16:44:24,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... + 0: [2023-05-09 16:44:24,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... +13: [2023-05-09 16:44:24,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +11: [2023-05-09 16:44:24,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +13: [2023-05-09 16:44:24,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +11: [2023-05-09 16:44:24,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +31: [2023-05-09 16:44:24,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +31: [2023-05-09 16:44:24,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +31: [2023-05-09 16:44:24,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +31: [2023-05-09 16:44:24,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +14: [2023-05-09 16:44:24,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt... +14: [2023-05-09 16:44:24,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt... +14: [2023-05-09 16:44:24,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt... +14: [2023-05-09 16:44:24,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt... +31: [2023-05-09 16:44:24,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +31: [2023-05-09 16:44:24,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +13: [2023-05-09 16:44:24,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt. +19: [2023-05-09 16:44:24,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +13: [2023-05-09 16:44:24,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt... +29: [2023-05-09 16:44:24,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +20: [2023-05-09 16:44:24,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt... +20: [2023-05-09 16:44:24,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt... +16: [2023-05-09 16:44:24,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +20: [2023-05-09 16:44:24,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt... +20: [2023-05-09 16:44:24,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt... +16: [2023-05-09 16:44:24,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +29: [2023-05-09 16:44:24,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +19: [2023-05-09 16:44:24,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +27: [2023-05-09 16:44:24,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt... +27: [2023-05-09 16:44:24,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt... +27: [2023-05-09 16:44:24,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt... +27: [2023-05-09 16:44:24,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt... +19: [2023-05-09 16:44:24,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +11: [2023-05-09 16:44:24,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +11: [2023-05-09 16:44:24,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +23: [2023-05-09 16:44:24,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt... +23: [2023-05-09 16:44:24,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt... +23: [2023-05-09 16:44:24,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt... +23: [2023-05-09 16:44:24,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt... +16: [2023-05-09 16:44:24,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +19: [2023-05-09 16:44:24,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +19: [2023-05-09 16:44:24,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +19: [2023-05-09 16:44:24,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +16: [2023-05-09 16:44:24,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +19: [2023-05-09 16:44:24,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +29: [2023-05-09 16:44:24,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +19: [2023-05-09 16:44:24,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +19: [2023-05-09 16:44:24,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +19: [2023-05-09 16:44:24,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +31: [2023-05-09 16:44:24,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt... +31: [2023-05-09 16:44:24,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt... +31: [2023-05-09 16:44:24,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt... +31: [2023-05-09 16:44:24,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt... +29: [2023-05-09 16:44:24,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +29: [2023-05-09 16:44:24,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +29: [2023-05-09 16:44:24,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +29: [2023-05-09 16:44:24,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +16: [2023-05-09 16:44:24,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +29: [2023-05-09 16:44:24,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +29: [2023-05-09 16:44:24,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +29: [2023-05-09 16:44:24,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +29: [2023-05-09 16:44:24,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +29: [2023-05-09 16:44:24,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +16: [2023-05-09 16:44:24,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +16: [2023-05-09 16:44:24,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +10: [2023-05-09 16:44:24,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +10: [2023-05-09 16:44:24,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +10: [2023-05-09 16:44:24,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +10: [2023-05-09 16:44:24,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +10: [2023-05-09 16:44:24,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +16: [2023-05-09 16:44:24,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +16: [2023-05-09 16:44:24,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +10: [2023-05-09 16:44:24,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +10: [2023-05-09 16:44:24,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +10: [2023-05-09 16:44:24,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +16: [2023-05-09 16:44:24,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +16: [2023-05-09 16:44:24,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +11: [2023-05-09 16:44:24,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +19: [2023-05-09 16:44:24,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +11: [2023-05-09 16:44:24,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +19: [2023-05-09 16:44:24,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +11: [2023-05-09 16:44:24,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +11: [2023-05-09 16:44:24,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +11: [2023-05-09 16:44:24,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +11: [2023-05-09 16:44:24,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +13: [2023-05-09 16:44:24,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +29: [2023-05-09 16:44:24,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +13: [2023-05-09 16:44:24,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +13: [2023-05-09 16:44:24,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +16: [2023-05-09 16:44:24,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +11: [2023-05-09 16:44:24,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +13: [2023-05-09 16:44:24,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +13: [2023-05-09 16:44:24,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +13: [2023-05-09 16:44:24,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +29: [2023-05-09 16:44:24,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +11: [2023-05-09 16:44:24,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +29: [2023-05-09 16:44:24,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +29: [2023-05-09 16:44:24,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +16: [2023-05-09 16:44:24,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +13: [2023-05-09 16:44:24,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +16: [2023-05-09 16:44:24,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +16: [2023-05-09 16:44:24,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +11: [2023-05-09 16:44:24,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +16: [2023-05-09 16:44:24,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +13: [2023-05-09 16:44:24,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +15: [2023-05-09 16:44:24,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +15: [2023-05-09 16:44:24,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +15: [2023-05-09 16:44:24,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +15: [2023-05-09 16:44:24,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +29: [2023-05-09 16:44:24,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +29: [2023-05-09 16:44:24,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +29: [2023-05-09 16:44:24,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +29: [2023-05-09 16:44:24,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +15: [2023-05-09 16:44:24,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +15: [2023-05-09 16:44:24,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +15: [2023-05-09 16:44:24,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +15: [2023-05-09 16:44:24,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +10: [2023-05-09 16:44:24,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +19: [2023-05-09 16:44:24,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +19: [2023-05-09 16:44:24,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +19: [2023-05-09 16:44:24,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +19: [2023-05-09 16:44:24,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +19: [2023-05-09 16:44:24,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +19: [2023-05-09 16:44:24,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +19: [2023-05-09 16:44:24,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +19: [2023-05-09 16:44:24,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +16: [2023-05-09 16:44:24,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +16: [2023-05-09 16:44:24,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +16: [2023-05-09 16:44:24,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +16: [2023-05-09 16:44:24,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +11: [2023-05-09 16:44:24,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +11: [2023-05-09 16:44:24,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +11: [2023-05-09 16:44:24,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +11: [2023-05-09 16:44:24,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +11: [2023-05-09 16:44:24,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +10: [2023-05-09 16:44:24,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +13: [2023-05-09 16:44:24,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +15: [2023-05-09 16:44:24,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +10: [2023-05-09 16:44:24,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +10: [2023-05-09 16:44:24,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +10: [2023-05-09 16:44:24,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +15: [2023-05-09 16:44:24,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +15: [2023-05-09 16:44:24,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +15: [2023-05-09 16:44:24,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +10: [2023-05-09 16:44:24,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +10: [2023-05-09 16:44:24,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +10: [2023-05-09 16:44:24,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +11: [2023-05-09 16:44:24,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +11: [2023-05-09 16:44:24,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +11: [2023-05-09 16:44:24,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +11: [2023-05-09 16:44:24,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +10: [2023-05-09 16:44:24,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +13: [2023-05-09 16:44:24,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +13: [2023-05-09 16:44:24,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +15: [2023-05-09 16:44:24,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +19: [2023-05-09 16:44:24,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt... +19: [2023-05-09 16:44:24,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt... +19: [2023-05-09 16:44:24,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt... +19: [2023-05-09 16:44:24,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt... +10: [2023-05-09 16:44:24,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +15: [2023-05-09 16:44:24,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +13: [2023-05-09 16:44:24,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +13: [2023-05-09 16:44:24,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 3: [2023-05-09 16:44:24,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt... + 3: [2023-05-09 16:44:24,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt... + 3: [2023-05-09 16:44:24,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt... + 3: [2023-05-09 16:44:24,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt... +13: [2023-05-09 16:44:24,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +10: [2023-05-09 16:44:24,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +10: [2023-05-09 16:44:24,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +10: [2023-05-09 16:44:24,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +10: [2023-05-09 16:44:24,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +10: [2023-05-09 16:44:24,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +15: [2023-05-09 16:44:24,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +15: [2023-05-09 16:44:24,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +10: [2023-05-09 16:44:24,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +15: [2023-05-09 16:44:24,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +15: [2023-05-09 16:44:24,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +15: [2023-05-09 16:44:24,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +15: [2023-05-09 16:44:24,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt... + 8: [2023-05-09 16:44:24,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt... + 8: [2023-05-09 16:44:24,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt... + 8: [2023-05-09 16:44:24,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt... +15: [2023-05-09 16:44:24,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +15: [2023-05-09 16:44:24,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +15: [2023-05-09 16:44:24,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +15: [2023-05-09 16:44:24,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +15: [2023-05-09 16:44:24,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +15: [2023-05-09 16:44:24,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +15: [2023-05-09 16:44:24,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +15: [2023-05-09 16:44:24,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +29: [2023-05-09 16:44:24,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt... +29: [2023-05-09 16:44:24,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt... +29: [2023-05-09 16:44:24,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt... +29: [2023-05-09 16:44:24,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt... + 9: [2023-05-09 16:44:24,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +10: [2023-05-09 16:44:24,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +10: [2023-05-09 16:44:24,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +10: [2023-05-09 16:44:24,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +10: [2023-05-09 16:44:24,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +10: [2023-05-09 16:44:24,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +16: [2023-05-09 16:44:24,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt... +16: [2023-05-09 16:44:24,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt... +16: [2023-05-09 16:44:24,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt... +16: [2023-05-09 16:44:24,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt... +10: [2023-05-09 16:44:24,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +10: [2023-05-09 16:44:24,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +10: [2023-05-09 16:44:24,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +13: [2023-05-09 16:44:24,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +13: [2023-05-09 16:44:24,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +13: [2023-05-09 16:44:24,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +13: [2023-05-09 16:44:24,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +13: [2023-05-09 16:44:24,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +21: [2023-05-09 16:44:24,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +21: [2023-05-09 16:44:24,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +10: [2023-05-09 16:44:24,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +21: [2023-05-09 16:44:24,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +21: [2023-05-09 16:44:24,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +21: [2023-05-09 16:44:24,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +21: [2023-05-09 16:44:24,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +21: [2023-05-09 16:44:24,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +21: [2023-05-09 16:44:24,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +10: [2023-05-09 16:44:24,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +10: [2023-05-09 16:44:24,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +10: [2023-05-09 16:44:24,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +13: [2023-05-09 16:44:24,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +13: [2023-05-09 16:44:24,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +11: [2023-05-09 16:44:24,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt... +11: [2023-05-09 16:44:24,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt... + 9: [2023-05-09 16:44:24,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +13: [2023-05-09 16:44:24,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +11: [2023-05-09 16:44:24,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt... +13: [2023-05-09 16:44:24,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +11: [2023-05-09 16:44:24,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt... + 9: [2023-05-09 16:44:24,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +13: [2023-05-09 16:44:24,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +18: [2023-05-09 16:44:24,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +18: [2023-05-09 16:44:24,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +18: [2023-05-09 16:44:24,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +18: [2023-05-09 16:44:24,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +18: [2023-05-09 16:44:24,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +18: [2023-05-09 16:44:24,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +18: [2023-05-09 16:44:24,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +18: [2023-05-09 16:44:24,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +21: [2023-05-09 16:44:24,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +10: [2023-05-09 16:44:24,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +10: [2023-05-09 16:44:24,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +13: [2023-05-09 16:44:24,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +13: [2023-05-09 16:44:24,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +13: [2023-05-09 16:44:24,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +13: [2023-05-09 16:44:24,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +21: [2023-05-09 16:44:24,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +21: [2023-05-09 16:44:24,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +21: [2023-05-09 16:44:24,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +18: [2023-05-09 16:44:24,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +21: [2023-05-09 16:44:24,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +10: [2023-05-09 16:44:24,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +10: [2023-05-09 16:44:24,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +10: [2023-05-09 16:44:24,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +10: [2023-05-09 16:44:24,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 1: [2023-05-09 16:44:24,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +18: [2023-05-09 16:44:24,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +21: [2023-05-09 16:44:24,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +21: [2023-05-09 16:44:24,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +28: [2023-05-09 16:44:24,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +28: [2023-05-09 16:44:24,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +28: [2023-05-09 16:44:24,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +28: [2023-05-09 16:44:24,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +21: [2023-05-09 16:44:24,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +28: [2023-05-09 16:44:24,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +28: [2023-05-09 16:44:24,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +28: [2023-05-09 16:44:24,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +28: [2023-05-09 16:44:24,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt... + 9: [2023-05-09 16:44:24,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt... + 9: [2023-05-09 16:44:24,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt... + 9: [2023-05-09 16:44:24,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt... + 4: [2023-05-09 16:44:24,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +21: [2023-05-09 16:44:24,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +21: [2023-05-09 16:44:24,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +15: [2023-05-09 16:44:24,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt... +15: [2023-05-09 16:44:24,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt... +15: [2023-05-09 16:44:24,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt... +15: [2023-05-09 16:44:24,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt... +21: [2023-05-09 16:44:24,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +21: [2023-05-09 16:44:24,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +21: [2023-05-09 16:44:24,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +21: [2023-05-09 16:44:24,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +21: [2023-05-09 16:44:24,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +21: [2023-05-09 16:44:24,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +18: [2023-05-09 16:44:24,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +21: [2023-05-09 16:44:24,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +26: [2023-05-09 16:44:24,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +26: [2023-05-09 16:44:24,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +26: [2023-05-09 16:44:24,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +26: [2023-05-09 16:44:24,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +21: [2023-05-09 16:44:24,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +26: [2023-05-09 16:44:24,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +26: [2023-05-09 16:44:24,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +21: [2023-05-09 16:44:24,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +21: [2023-05-09 16:44:24,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +18: [2023-05-09 16:44:24,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +26: [2023-05-09 16:44:24,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +10: [2023-05-09 16:44:24,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +26: [2023-05-09 16:44:24,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +18: [2023-05-09 16:44:24,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +18: [2023-05-09 16:44:24,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +13: [2023-05-09 16:44:24,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt... +13: [2023-05-09 16:44:24,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt... +13: [2023-05-09 16:44:24,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt... +13: [2023-05-09 16:44:24,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt... +18: [2023-05-09 16:44:24,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +10: [2023-05-09 16:44:24,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +10: [2023-05-09 16:44:24,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt... +10: [2023-05-09 16:44:24,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt... +10: [2023-05-09 16:44:24,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt... +10: [2023-05-09 16:44:24,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt... +18: [2023-05-09 16:44:24,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +18: [2023-05-09 16:44:24,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +18: [2023-05-09 16:44:24,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +18: [2023-05-09 16:44:24,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +24: [2023-05-09 16:44:24,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +24: [2023-05-09 16:44:24,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +24: [2023-05-09 16:44:24,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +24: [2023-05-09 16:44:24,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +24: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +24: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +24: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +24: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +22: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +17: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +17: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +22: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +17: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +22: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +17: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +17: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +30: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +17: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +22: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +30: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +22: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +22: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +30: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +17: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +30: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +30: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +17: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +22: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +30: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +28: [2023-05-09 16:44:24,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +30: [2023-05-09 16:44:24,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +30: [2023-05-09 16:44:24,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +18: [2023-05-09 16:44:24,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +22: [2023-05-09 16:44:24,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +18: [2023-05-09 16:44:24,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +18: [2023-05-09 16:44:24,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +18: [2023-05-09 16:44:24,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +18: [2023-05-09 16:44:24,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +28: [2023-05-09 16:44:24,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +28: [2023-05-09 16:44:24,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +26: [2023-05-09 16:44:24,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +28: [2023-05-09 16:44:24,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +18: [2023-05-09 16:44:24,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +18: [2023-05-09 16:44:24,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +18: [2023-05-09 16:44:24,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +18: [2023-05-09 16:44:24,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +24: [2023-05-09 16:44:24,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +22: [2023-05-09 16:44:24,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +28: [2023-05-09 16:44:24,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +17: [2023-05-09 16:44:24,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +17: [2023-05-09 16:44:24,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +22: [2023-05-09 16:44:24,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +26: [2023-05-09 16:44:24,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +30: [2023-05-09 16:44:24,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +17: [2023-05-09 16:44:24,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +30: [2023-05-09 16:44:24,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +17: [2023-05-09 16:44:24,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +29: [2023-05-09 16:44:24,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +29: [2023-05-09 16:44:24,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +30: [2023-05-09 16:44:24,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +30: [2023-05-09 16:44:24,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +29: [2023-05-09 16:44:24,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +26: [2023-05-09 16:44:24,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 2: [2023-05-09 16:44:24,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +26: [2023-05-09 16:44:24,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +26: [2023-05-09 16:44:24,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +28: [2023-05-09 16:44:24,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +26: [2023-05-09 16:44:24,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 2: [2023-05-09 16:44:24,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +28: [2023-05-09 16:44:24,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +28: [2023-05-09 16:44:24,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +28: [2023-05-09 16:44:24,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +22: [2023-05-09 16:44:24,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +28: [2023-05-09 16:44:24,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +28: [2023-05-09 16:44:24,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +26: [2023-05-09 16:44:24,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +28: [2023-05-09 16:44:24,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +24: [2023-05-09 16:44:24,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +24: [2023-05-09 16:44:24,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +28: [2023-05-09 16:44:24,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +28: [2023-05-09 16:44:24,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +29: [2023-05-09 16:44:24,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +24: [2023-05-09 16:44:24,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +28: [2023-05-09 16:44:24,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +28: [2023-05-09 16:44:24,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +28: [2023-05-09 16:44:24,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +28: [2023-05-09 16:44:24,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +28: [2023-05-09 16:44:24,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +28: [2023-05-09 16:44:24,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 8: [2023-05-09 16:44:24,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +17: [2023-05-09 16:44:24,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +24: [2023-05-09 16:44:24,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +24: [2023-05-09 16:44:24,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +24: [2023-05-09 16:44:24,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +21: [2023-05-09 16:44:24,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt... +21: [2023-05-09 16:44:24,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt... +21: [2023-05-09 16:44:24,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt... +21: [2023-05-09 16:44:24,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt... +22: [2023-05-09 16:44:24,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +22: [2023-05-09 16:44:24,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +30: [2023-05-09 16:44:24,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +26: [2023-05-09 16:44:24,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +30: [2023-05-09 16:44:24,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +30: [2023-05-09 16:44:24,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +29: [2023-05-09 16:44:24,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +17: [2023-05-09 16:44:24,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +26: [2023-05-09 16:44:24,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +30: [2023-05-09 16:44:24,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +24: [2023-05-09 16:44:24,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +26: [2023-05-09 16:44:24,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +26: [2023-05-09 16:44:24,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +17: [2023-05-09 16:44:24,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +17: [2023-05-09 16:44:24,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +30: [2023-05-09 16:44:24,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +30: [2023-05-09 16:44:24,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +29: [2023-05-09 16:44:24,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +30: [2023-05-09 16:44:24,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +30: [2023-05-09 16:44:24,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +30: [2023-05-09 16:44:24,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +30: [2023-05-09 16:44:24,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +30: [2023-05-09 16:44:24,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +29: [2023-05-09 16:44:24,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +26: [2023-05-09 16:44:24,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +22: [2023-05-09 16:44:24,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +30: [2023-05-09 16:44:24,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +30: [2023-05-09 16:44:24,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +22: [2023-05-09 16:44:24,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +22: [2023-05-09 16:44:24,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +30: [2023-05-09 16:44:24,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +22: [2023-05-09 16:44:24,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +30: [2023-05-09 16:44:24,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +22: [2023-05-09 16:44:24,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +24: [2023-05-09 16:44:24,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +30: [2023-05-09 16:44:24,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +26: [2023-05-09 16:44:24,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +22: [2023-05-09 16:44:24,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +22: [2023-05-09 16:44:24,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +20: [2023-05-09 16:44:24,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +20: [2023-05-09 16:44:24,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +26: [2023-05-09 16:44:24,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +24: [2023-05-09 16:44:24,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +24: [2023-05-09 16:44:24,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +22: [2023-05-09 16:44:24,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +22: [2023-05-09 16:44:24,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +26: [2023-05-09 16:44:24,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +26: [2023-05-09 16:44:24,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +26: [2023-05-09 16:44:24,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +24: [2023-05-09 16:44:24,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +22: [2023-05-09 16:44:24,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +26: [2023-05-09 16:44:24,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +26: [2023-05-09 16:44:24,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +17: [2023-05-09 16:44:24,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +17: [2023-05-09 16:44:24,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +17: [2023-05-09 16:44:24,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +17: [2023-05-09 16:44:24,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +17: [2023-05-09 16:44:24,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +17: [2023-05-09 16:44:24,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +29: [2023-05-09 16:44:24,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +17: [2023-05-09 16:44:24,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +17: [2023-05-09 16:44:24,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +17: [2023-05-09 16:44:24,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +17: [2023-05-09 16:44:24,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +17: [2023-05-09 16:44:24,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +17: [2023-05-09 16:44:24,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +12: [2023-05-09 16:44:24,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +12: [2023-05-09 16:44:24,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +12: [2023-05-09 16:44:24,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +26: [2023-05-09 16:44:24,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +24: [2023-05-09 16:44:24,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +12: [2023-05-09 16:44:24,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 4: [2023-05-09 16:44:24,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +24: [2023-05-09 16:44:24,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +24: [2023-05-09 16:44:24,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +24: [2023-05-09 16:44:24,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt... + 1: [2023-05-09 16:44:24,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt... +20: [2023-05-09 16:44:24,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 1: [2023-05-09 16:44:24,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt... + 1: [2023-05-09 16:44:24,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt... + 2: [2023-05-09 16:44:24,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +20: [2023-05-09 16:44:24,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +26: [2023-05-09 16:44:24,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 0: [2023-05-09 16:44:24,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 0: [2023-05-09 16:44:24,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 8: [2023-05-09 16:44:24,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +26: [2023-05-09 16:44:24,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +26: [2023-05-09 16:44:24,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +17: [2023-05-09 16:44:24,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +17: [2023-05-09 16:44:24,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +17: [2023-05-09 16:44:24,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 0: [2023-05-09 16:44:24,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +26: [2023-05-09 16:44:24,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +24: [2023-05-09 16:44:24,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 2: [2023-05-09 16:44:24,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +24: [2023-05-09 16:44:24,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +24: [2023-05-09 16:44:24,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +17: [2023-05-09 16:44:24,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 2: [2023-05-09 16:44:24,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +24: [2023-05-09 16:44:24,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +22: [2023-05-09 16:44:24,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +18: [2023-05-09 16:44:24,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt... +20: [2023-05-09 16:44:24,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +18: [2023-05-09 16:44:24,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt... +18: [2023-05-09 16:44:24,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt... +18: [2023-05-09 16:44:24,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt... +26: [2023-05-09 16:44:24,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +28: [2023-05-09 16:44:24,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 2: [2023-05-09 16:44:24,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +26: [2023-05-09 16:44:24,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +28: [2023-05-09 16:44:24,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +28: [2023-05-09 16:44:24,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +26: [2023-05-09 16:44:24,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +28: [2023-05-09 16:44:24,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 7: [2023-05-09 16:44:24,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +26: [2023-05-09 16:44:24,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +26: [2023-05-09 16:44:24,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +12: [2023-05-09 16:44:24,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +12: [2023-05-09 16:44:24,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +29: [2023-05-09 16:44:24,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +26: [2023-05-09 16:44:24,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +29: [2023-05-09 16:44:24,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 7: [2023-05-09 16:44:24,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +12: [2023-05-09 16:44:24,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +20: [2023-05-09 16:44:24,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +20: [2023-05-09 16:44:24,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +20: [2023-05-09 16:44:24,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +26: [2023-05-09 16:44:24,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +26: [2023-05-09 16:44:24,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 8: [2023-05-09 16:44:24,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +12: [2023-05-09 16:44:24,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 8: [2023-05-09 16:44:24,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +29: [2023-05-09 16:44:24,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +29: [2023-05-09 16:44:24,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 0: [2023-05-09 16:44:24,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +17: [2023-05-09 16:44:24,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 0: [2023-05-09 16:44:24,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +22: [2023-05-09 16:44:24,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +17: [2023-05-09 16:44:24,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +17: [2023-05-09 16:44:24,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +22: [2023-05-09 16:44:24,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +22: [2023-05-09 16:44:24,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +22: [2023-05-09 16:44:24,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +28: [2023-05-09 16:44:24,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +17: [2023-05-09 16:44:24,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +28: [2023-05-09 16:44:24,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +28: [2023-05-09 16:44:24,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +28: [2023-05-09 16:44:24,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +29: [2023-05-09 16:44:24,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +29: [2023-05-09 16:44:24,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 8: [2023-05-09 16:44:24,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +12: [2023-05-09 16:44:24,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +12: [2023-05-09 16:44:24,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +12: [2023-05-09 16:44:24,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +12: [2023-05-09 16:44:24,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +26: [2023-05-09 16:44:24,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt... + 4: [2023-05-09 16:44:24,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt... + 4: [2023-05-09 16:44:24,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt... + 4: [2023-05-09 16:44:24,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt... +26: [2023-05-09 16:44:24,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 8: [2023-05-09 16:44:24,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 8: [2023-05-09 16:44:24,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +17: [2023-05-09 16:44:24,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +20: [2023-05-09 16:44:24,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +29: [2023-05-09 16:44:24,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +17: [2023-05-09 16:44:24,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +20: [2023-05-09 16:44:24,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 2: [2023-05-09 16:44:24,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 2: [2023-05-09 16:44:24,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 2: [2023-05-09 16:44:24,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 2: [2023-05-09 16:44:24,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 2: [2023-05-09 16:44:24,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +29: [2023-05-09 16:44:24,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 2: [2023-05-09 16:44:24,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +26: [2023-05-09 16:44:24,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +12: [2023-05-09 16:44:24,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +12: [2023-05-09 16:44:24,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +17: [2023-05-09 16:44:24,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +17: [2023-05-09 16:44:24,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +12: [2023-05-09 16:44:24,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +12: [2023-05-09 16:44:24,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +25: [2023-05-09 16:44:24,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +28: [2023-05-09 16:44:24,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +26: [2023-05-09 16:44:24,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +25: [2023-05-09 16:44:24,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +25: [2023-05-09 16:44:24,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +25: [2023-05-09 16:44:24,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +17: [2023-05-09 16:44:24,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +17: [2023-05-09 16:44:24,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +28: [2023-05-09 16:44:24,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +17: [2023-05-09 16:44:24,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +17: [2023-05-09 16:44:24,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +20: [2023-05-09 16:44:24,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +20: [2023-05-09 16:44:24,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +20: [2023-05-09 16:44:24,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +20: [2023-05-09 16:44:24,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 8: [2023-05-09 16:44:24,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +20: [2023-05-09 16:44:24,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 0: [2023-05-09 16:44:24,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +28: [2023-05-09 16:44:24,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt... +28: [2023-05-09 16:44:24,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt... +28: [2023-05-09 16:44:24,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt... +24: [2023-05-09 16:44:24,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt... +24: [2023-05-09 16:44:24,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt... +24: [2023-05-09 16:44:24,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt... +24: [2023-05-09 16:44:24,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt... +28: [2023-05-09 16:44:24,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt... + 8: [2023-05-09 16:44:24,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +30: [2023-05-09 16:44:24,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt... +30: [2023-05-09 16:44:24,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt... +30: [2023-05-09 16:44:24,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt... +30: [2023-05-09 16:44:24,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt... +20: [2023-05-09 16:44:24,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +28: [2023-05-09 16:44:24,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +28: [2023-05-09 16:44:24,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +28: [2023-05-09 16:44:24,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +28: [2023-05-09 16:44:24,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +28: [2023-05-09 16:44:24,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +28: [2023-05-09 16:44:24,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 2: [2023-05-09 16:44:24,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +25: [2023-05-09 16:44:24,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +22: [2023-05-09 16:44:24,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt... +22: [2023-05-09 16:44:24,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt... +22: [2023-05-09 16:44:24,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt... +22: [2023-05-09 16:44:24,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt... +26: [2023-05-09 16:44:24,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt... +26: [2023-05-09 16:44:24,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt... +26: [2023-05-09 16:44:24,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt... +26: [2023-05-09 16:44:24,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt... +25: [2023-05-09 16:44:24,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +25: [2023-05-09 16:44:24,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +25: [2023-05-09 16:44:24,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt... + 7: [2023-05-09 16:44:24,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt... + 7: [2023-05-09 16:44:24,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt... + 5: [2023-05-09 16:44:24,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt... + 5: [2023-05-09 16:44:24,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt... + 5: [2023-05-09 16:44:24,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt... + 5: [2023-05-09 16:44:24,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt... + 7: [2023-05-09 16:44:24,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt... +17: [2023-05-09 16:44:24,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt... +17: [2023-05-09 16:44:24,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt... +17: [2023-05-09 16:44:24,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt... +17: [2023-05-09 16:44:24,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt... + 0: [2023-05-09 16:44:24,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 0: [2023-05-09 16:44:24,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt... + 2: [2023-05-09 16:44:24,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt... + 2: [2023-05-09 16:44:24,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt... + 2: [2023-05-09 16:44:24,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt... + 6: [2023-05-09 16:44:24,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 6: [2023-05-09 16:44:24,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 6: [2023-05-09 16:44:24,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 0: [2023-05-09 16:44:24,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 0: [2023-05-09 16:44:24,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 6: [2023-05-09 16:44:24,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +14: [2023-05-09 16:44:24,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +14: [2023-05-09 16:44:24,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +14: [2023-05-09 16:44:24,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +14: [2023-05-09 16:44:24,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +25: [2023-05-09 16:44:24,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +25: [2023-05-09 16:44:24,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 2: [2023-05-09 16:44:24,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 6: [2023-05-09 16:44:24,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +14: [2023-05-09 16:44:24,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +14: [2023-05-09 16:44:24,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 2: [2023-05-09 16:44:24,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 2: [2023-05-09 16:44:24,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +14: [2023-05-09 16:44:24,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +14: [2023-05-09 16:44:24,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +25: [2023-05-09 16:44:24,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +25: [2023-05-09 16:44:24,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 2: [2023-05-09 16:44:24,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +25: [2023-05-09 16:44:24,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +25: [2023-05-09 16:44:24,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +27: [2023-05-09 16:44:24,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +25: [2023-05-09 16:44:24,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +25: [2023-05-09 16:44:24,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +27: [2023-05-09 16:44:24,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +27: [2023-05-09 16:44:24,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 2: [2023-05-09 16:44:24,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +21: [2023-05-09 16:44:24,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +21: [2023-05-09 16:44:24,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +21: [2023-05-09 16:44:24,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +27: [2023-05-09 16:44:24,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +16: [2023-05-09 16:44:24,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +16: [2023-05-09 16:44:24,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +16: [2023-05-09 16:44:24,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +16: [2023-05-09 16:44:24,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +19: [2023-05-09 16:44:24,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +23: [2023-05-09 16:44:24,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +23: [2023-05-09 16:44:24,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +23: [2023-05-09 16:44:24,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +18: [2023-05-09 16:44:24,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +23: [2023-05-09 16:44:24,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +18: [2023-05-09 16:44:24,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +21: [2023-05-09 16:44:24,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +30: [2023-05-09 16:44:24,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +30: [2023-05-09 16:44:24,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +30: [2023-05-09 16:44:24,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +19: [2023-05-09 16:44:24,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +30: [2023-05-09 16:44:24,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +15: [2023-05-09 16:44:24,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +15: [2023-05-09 16:44:24,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +15: [2023-05-09 16:44:24,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +15: [2023-05-09 16:44:24,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +22: [2023-05-09 16:44:24,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +22: [2023-05-09 16:44:24,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +22: [2023-05-09 16:44:24,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +22: [2023-05-09 16:44:24,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 6: [2023-05-09 16:44:24,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +19: [2023-05-09 16:44:24,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 6: [2023-05-09 16:44:24,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 6: [2023-05-09 16:44:24,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 6: [2023-05-09 16:44:24,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +19: [2023-05-09 16:44:24,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +18: [2023-05-09 16:44:24,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 9: [2023-05-09 16:44:24,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 9: [2023-05-09 16:44:24,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 9: [2023-05-09 16:44:24,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 9: [2023-05-09 16:44:24,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +27: [2023-05-09 16:44:24,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +27: [2023-05-09 16:44:24,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +18: [2023-05-09 16:44:24,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +31: [2023-05-09 16:44:24,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +31: [2023-05-09 16:44:24,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +31: [2023-05-09 16:44:24,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +21: [2023-05-09 16:44:24,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +31: [2023-05-09 16:44:24,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +24: [2023-05-09 16:44:24,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +24: [2023-05-09 16:44:24,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +24: [2023-05-09 16:44:24,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 6: [2023-05-09 16:44:24,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +24: [2023-05-09 16:44:24,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 6: [2023-05-09 16:44:24,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +27: [2023-05-09 16:44:24,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +27: [2023-05-09 16:44:24,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +16: [2023-05-09 16:44:24,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +16: [2023-05-09 16:44:24,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +16: [2023-05-09 16:44:24,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +19: [2023-05-09 16:44:24,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +16: [2023-05-09 16:44:24,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +21: [2023-05-09 16:44:24,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +21: [2023-05-09 16:44:24,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +30: [2023-05-09 16:44:24,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +30: [2023-05-09 16:44:24,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +19: [2023-05-09 16:44:24,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +21: [2023-05-09 16:44:24,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +23: [2023-05-09 16:44:24,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +18: [2023-05-09 16:44:24,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +30: [2023-05-09 16:44:24,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +23: [2023-05-09 16:44:24,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +23: [2023-05-09 16:44:24,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +30: [2023-05-09 16:44:24,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +23: [2023-05-09 16:44:24,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +15: [2023-05-09 16:44:24,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +22: [2023-05-09 16:44:24,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +18: [2023-05-09 16:44:24,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +15: [2023-05-09 16:44:24,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +15: [2023-05-09 16:44:24,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +15: [2023-05-09 16:44:24,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +14: [2023-05-09 16:44:24,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +14: [2023-05-09 16:44:24,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +18: [2023-05-09 16:44:24,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +14: [2023-05-09 16:44:24,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +14: [2023-05-09 16:44:24,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +19: [2023-05-09 16:44:24,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +19: [2023-05-09 16:44:24,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +22: [2023-05-09 16:44:24,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +22: [2023-05-09 16:44:24,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +18: [2023-05-09 16:44:24,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +22: [2023-05-09 16:44:24,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +24: [2023-05-09 16:44:24,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +31: [2023-05-09 16:44:24,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 6: [2023-05-09 16:44:24,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 3: [2023-05-09 16:44:24,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 3: [2023-05-09 16:44:24,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 3: [2023-05-09 16:44:24,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 3: [2023-05-09 16:44:24,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +31: [2023-05-09 16:44:24,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +24: [2023-05-09 16:44:24,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +31: [2023-05-09 16:44:24,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +24: [2023-05-09 16:44:24,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +31: [2023-05-09 16:44:24,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +16: [2023-05-09 16:44:24,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +24: [2023-05-09 16:44:24,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +27: [2023-05-09 16:44:24,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +16: [2023-05-09 16:44:24,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +27: [2023-05-09 16:44:24,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +21: [2023-05-09 16:44:24,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +27: [2023-05-09 16:44:24,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +27: [2023-05-09 16:44:24,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +27: [2023-05-09 16:44:24,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +27: [2023-05-09 16:44:24,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +21: [2023-05-09 16:44:24,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +23: [2023-05-09 16:44:24,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +30: [2023-05-09 16:44:24,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +14: [2023-05-09 16:44:24,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +30: [2023-05-09 16:44:24,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +23: [2023-05-09 16:44:24,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +14: [2023-05-09 16:44:24,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +19: [2023-05-09 16:44:24,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +19: [2023-05-09 16:44:24,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +19: [2023-05-09 16:44:24,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +19: [2023-05-09 16:44:24,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +30: [2023-05-09 16:44:24,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +21: [2023-05-09 16:44:24,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +30: [2023-05-09 16:44:24,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +21: [2023-05-09 16:44:24,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +16: [2023-05-09 16:44:24,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +16: [2023-05-09 16:44:24,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +23: [2023-05-09 16:44:24,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +14: [2023-05-09 16:44:24,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +23: [2023-05-09 16:44:24,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +23: [2023-05-09 16:44:24,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +23: [2023-05-09 16:44:24,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 3: [2023-05-09 16:44:24,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +15: [2023-05-09 16:44:24,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +14: [2023-05-09 16:44:24,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +16: [2023-05-09 16:44:24,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +16: [2023-05-09 16:44:24,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +18: [2023-05-09 16:44:24,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +16: [2023-05-09 16:44:24,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +16: [2023-05-09 16:44:24,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +18: [2023-05-09 16:44:24,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +23: [2023-05-09 16:44:24,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +23: [2023-05-09 16:44:24,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 3: [2023-05-09 16:44:24,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +19: [2023-05-09 16:44:24,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +19: [2023-05-09 16:44:24,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +24: [2023-05-09 16:44:24,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 1: [2023-05-09 16:44:24,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +24: [2023-05-09 16:44:24,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 3: [2023-05-09 16:44:24,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +22: [2023-05-09 16:44:24,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +15: [2023-05-09 16:44:24,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +15: [2023-05-09 16:44:24,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +22: [2023-05-09 16:44:24,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +22: [2023-05-09 16:44:24,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +22: [2023-05-09 16:44:24,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +22: [2023-05-09 16:44:24,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +22: [2023-05-09 16:44:24,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +24: [2023-05-09 16:44:24,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +24: [2023-05-09 16:44:24,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +15: [2023-05-09 16:44:24,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 9: [2023-05-09 16:44:24,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +15: [2023-05-09 16:44:24,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +18: [2023-05-09 16:44:24,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +18: [2023-05-09 16:44:24,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 9: [2023-05-09 16:44:24,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 9: [2023-05-09 16:44:24,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 9: [2023-05-09 16:44:24,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 9: [2023-05-09 16:44:24,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +15: [2023-05-09 16:44:24,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 1: [2023-05-09 16:44:24,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +15: [2023-05-09 16:44:24,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +15: [2023-05-09 16:44:24,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +27: [2023-05-09 16:44:24,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +19: [2023-05-09 16:44:24,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +21: [2023-05-09 16:44:24,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +19: [2023-05-09 16:44:24,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +18: [2023-05-09 16:44:24,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +21: [2023-05-09 16:44:24,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +27: [2023-05-09 16:44:24,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 5: [2023-05-09 16:44:24,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 5: [2023-05-09 16:44:24,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +21: [2023-05-09 16:44:24,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +30: [2023-05-09 16:44:24,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +31: [2023-05-09 16:44:24,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +31: [2023-05-09 16:44:24,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +31: [2023-05-09 16:44:24,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 5: [2023-05-09 16:44:24,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +21: [2023-05-09 16:44:24,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +31: [2023-05-09 16:44:24,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 5: [2023-05-09 16:44:24,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +30: [2023-05-09 16:44:24,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +30: [2023-05-09 16:44:24,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +18: [2023-05-09 16:44:24,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +30: [2023-05-09 16:44:24,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +31: [2023-05-09 16:44:24,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +31: [2023-05-09 16:44:24,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +24: [2023-05-09 16:44:24,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +18: [2023-05-09 16:44:24,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +24: [2023-05-09 16:44:24,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 3: [2023-05-09 16:44:24,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +18: [2023-05-09 16:44:24,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 5: [2023-05-09 16:44:24,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +22: [2023-05-09 16:44:24,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 5: [2023-05-09 16:44:24,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +22: [2023-05-09 16:44:24,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 3: [2023-05-09 16:44:24,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +24: [2023-05-09 16:44:24,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +31: [2023-05-09 16:44:24,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 3: [2023-05-09 16:44:24,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 1: [2023-05-09 16:44:24,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +24: [2023-05-09 16:44:24,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 1: [2023-05-09 16:44:24,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +31: [2023-05-09 16:44:24,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 1: [2023-05-09 16:44:24,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 3: [2023-05-09 16:44:24,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 1: [2023-05-09 16:44:24,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 3: [2023-05-09 16:44:24,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 5: [2023-05-09 16:44:24,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 5: [2023-05-09 16:44:24,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 7: [2023-05-09 16:44:24,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 7: [2023-05-09 16:44:24,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 7: [2023-05-09 16:44:24,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 7: [2023-05-09 16:44:24,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 5: [2023-05-09 16:44:24,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 5: [2023-05-09 16:44:24,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 5: [2023-05-09 16:44:24,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +11: [2023-05-09 16:44:24,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +11: [2023-05-09 16:44:24,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 4: [2023-05-09 16:44:24,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 4: [2023-05-09 16:44:24,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 4: [2023-05-09 16:44:24,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +11: [2023-05-09 16:44:24,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +11: [2023-05-09 16:44:24,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 7: [2023-05-09 16:44:24,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +11: [2023-05-09 16:44:24,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +11: [2023-05-09 16:44:24,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +13: [2023-05-09 16:44:24,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +13: [2023-05-09 16:44:24,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +13: [2023-05-09 16:44:24,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. +13: [2023-05-09 16:44:24,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt. + 4: [2023-05-09 16:44:24,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +11: [2023-05-09 16:44:24,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +11: [2023-05-09 16:44:24,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 4: [2023-05-09 16:44:24,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... + 7: [2023-05-09 16:44:24,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +13: [2023-05-09 16:44:24,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +13: [2023-05-09 16:44:24,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +13: [2023-05-09 16:44:24,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +13: [2023-05-09 16:44:24,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt... +11: [2023-05-09 16:44:24,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +11: [2023-05-09 16:44:24,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +11: [2023-05-09 16:44:24,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +11: [2023-05-09 16:44:24,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +11: [2023-05-09 16:44:24,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +11: [2023-05-09 16:44:24,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 7: [2023-05-09 16:44:24,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 7: [2023-05-09 16:44:24,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +11: [2023-05-09 16:44:24,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +11: [2023-05-09 16:44:24,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 7: [2023-05-09 16:44:24,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 7: [2023-05-09 16:44:24,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +10: [2023-05-09 16:44:24,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +10: [2023-05-09 16:44:24,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +10: [2023-05-09 16:44:24,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +13: [2023-05-09 16:44:24,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +10: [2023-05-09 16:44:24,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +13: [2023-05-09 16:44:24,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 4: [2023-05-09 16:44:24,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +13: [2023-05-09 16:44:24,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +13: [2023-05-09 16:44:24,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 4: [2023-05-09 16:44:24,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. + 4: [2023-05-09 16:44:24,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 4: [2023-05-09 16:44:24,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... + 4: [2023-05-09 16:44:24,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +10: [2023-05-09 16:44:24,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +26: [2023-05-09 16:44:24,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +26: [2023-05-09 16:44:24,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +26: [2023-05-09 16:44:24,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +26: [2023-05-09 16:44:24,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +13: [2023-05-09 16:44:24,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +13: [2023-05-09 16:44:24,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +10: [2023-05-09 16:44:24,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +10: [2023-05-09 16:44:24,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 4: [2023-05-09 16:44:24,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +10: [2023-05-09 16:44:24,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +13: [2023-05-09 16:44:24,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt. +13: [2023-05-09 16:44:24,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt... +26: [2023-05-09 16:44:24,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +26: [2023-05-09 16:44:24,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +26: [2023-05-09 16:44:24,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +26: [2023-05-09 16:44:24,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:25,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt. +12: [2023-05-09 16:44:25,001] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 96 +17: [2023-05-09 16:44:25,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +17: [2023-05-09 16:44:25,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +17: [2023-05-09 16:44:25,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +26: [2023-05-09 16:44:25,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +10: [2023-05-09 16:44:25,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +26: [2023-05-09 16:44:25,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +10: [2023-05-09 16:44:25,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +17: [2023-05-09 16:44:25,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +12: [2023-05-09 16:44:25,008] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 96 +26: [2023-05-09 16:44:25,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +26: [2023-05-09 16:44:25,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +26: [2023-05-09 16:44:25,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +26: [2023-05-09 16:44:25,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +10: [2023-05-09 16:44:25,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +10: [2023-05-09 16:44:25,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +10: [2023-05-09 16:44:25,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +26: [2023-05-09 16:44:25,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +26: [2023-05-09 16:44:25,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +17: [2023-05-09 16:44:25,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +10: [2023-05-09 16:44:25,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +10: [2023-05-09 16:44:25,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +17: [2023-05-09 16:44:25,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +17: [2023-05-09 16:44:25,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +17: [2023-05-09 16:44:25,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +10: [2023-05-09 16:44:25,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +12: [2023-05-09 16:44:25,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt. +12: [2023-05-09 16:44:25,030] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 102 +17: [2023-05-09 16:44:25,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +12: [2023-05-09 16:44:25,035] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 102 +17: [2023-05-09 16:44:25,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 0: [2023-05-09 16:44:25,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 0: [2023-05-09 16:44:25,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 0: [2023-05-09 16:44:25,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 0: [2023-05-09 16:44:25,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +17: [2023-05-09 16:44:25,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +17: [2023-05-09 16:44:25,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +17: [2023-05-09 16:44:25,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +17: [2023-05-09 16:44:25,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +17: [2023-05-09 16:44:25,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +17: [2023-05-09 16:44:25,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +27: [2023-05-09 16:44:25,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt. +27: [2023-05-09 16:44:25,041] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 220 + 0: [2023-05-09 16:44:25,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 0: [2023-05-09 16:44:25,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +20: [2023-05-09 16:44:25,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +20: [2023-05-09 16:44:25,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +20: [2023-05-09 16:44:25,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +20: [2023-05-09 16:44:25,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 0: [2023-05-09 16:44:25,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:25,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 0: [2023-05-09 16:44:25,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:25,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +12: [2023-05-09 16:44:25,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +27: [2023-05-09 16:44:25,048] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 220 +12: [2023-05-09 16:44:25,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 6: [2023-05-09 16:44:25,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 6: [2023-05-09 16:44:25,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 6: [2023-05-09 16:44:25,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 8: [2023-05-09 16:44:25,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 8: [2023-05-09 16:44:25,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 8: [2023-05-09 16:44:25,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +28: [2023-05-09 16:44:25,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +28: [2023-05-09 16:44:25,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +28: [2023-05-09 16:44:25,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +28: [2023-05-09 16:44:25,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +25: [2023-05-09 16:44:25,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 6: [2023-05-09 16:44:25,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 8: [2023-05-09 16:44:25,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +25: [2023-05-09 16:44:25,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +25: [2023-05-09 16:44:25,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +25: [2023-05-09 16:44:25,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +12: [2023-05-09 16:44:25,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt. +12: [2023-05-09 16:44:25,055] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 100 +22: [2023-05-09 16:44:25,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +22: [2023-05-09 16:44:25,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +22: [2023-05-09 16:44:25,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +22: [2023-05-09 16:44:25,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +29: [2023-05-09 16:44:25,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +29: [2023-05-09 16:44:25,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +29: [2023-05-09 16:44:25,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +29: [2023-05-09 16:44:25,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +20: [2023-05-09 16:44:25,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +20: [2023-05-09 16:44:25,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +20: [2023-05-09 16:44:25,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +20: [2023-05-09 16:44:25,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:25,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:25,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:25,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:25,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 6: [2023-05-09 16:44:25,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 6: [2023-05-09 16:44:25,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:25,061] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 100 +28: [2023-05-09 16:44:25,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +25: [2023-05-09 16:44:25,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +28: [2023-05-09 16:44:25,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 8: [2023-05-09 16:44:25,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 8: [2023-05-09 16:44:25,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +28: [2023-05-09 16:44:25,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +28: [2023-05-09 16:44:25,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 5: [2023-05-09 16:44:25,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 5: [2023-05-09 16:44:25,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 5: [2023-05-09 16:44:25,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 6: [2023-05-09 16:44:25,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 5: [2023-05-09 16:44:25,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +25: [2023-05-09 16:44:25,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +25: [2023-05-09 16:44:25,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 8: [2023-05-09 16:44:25,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +25: [2023-05-09 16:44:25,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +29: [2023-05-09 16:44:25,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +29: [2023-05-09 16:44:25,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 6: [2023-05-09 16:44:25,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +29: [2023-05-09 16:44:25,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 7: [2023-05-09 16:44:25,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 7: [2023-05-09 16:44:25,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +22: [2023-05-09 16:44:25,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +29: [2023-05-09 16:44:25,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 2: [2023-05-09 16:44:25,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 2: [2023-05-09 16:44:25,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 8: [2023-05-09 16:44:25,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +22: [2023-05-09 16:44:25,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +22: [2023-05-09 16:44:25,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +22: [2023-05-09 16:44:25,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 2: [2023-05-09 16:44:25,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 2: [2023-05-09 16:44:25,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 7: [2023-05-09 16:44:25,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 7: [2023-05-09 16:44:25,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +12: [2023-05-09 16:44:25,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +12: [2023-05-09 16:44:25,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 0: [2023-05-09 16:44:25,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 0: [2023-05-09 16:44:25,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +20: [2023-05-09 16:44:25,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 0: [2023-05-09 16:44:25,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 0: [2023-05-09 16:44:25,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +20: [2023-05-09 16:44:25,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +20: [2023-05-09 16:44:25,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +20: [2023-05-09 16:44:25,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 5: [2023-05-09 16:44:25,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +20: [2023-05-09 16:44:25,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +20: [2023-05-09 16:44:25,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 5: [2023-05-09 16:44:25,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 5: [2023-05-09 16:44:25,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:25,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +25: [2023-05-09 16:44:25,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +12: [2023-05-09 16:44:25,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +15: [2023-05-09 16:44:25,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +15: [2023-05-09 16:44:25,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +15: [2023-05-09 16:44:25,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 5: [2023-05-09 16:44:25,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +25: [2023-05-09 16:44:25,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +15: [2023-05-09 16:44:25,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 7: [2023-05-09 16:44:25,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +20: [2023-05-09 16:44:25,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +28: [2023-05-09 16:44:25,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +20: [2023-05-09 16:44:25,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +28: [2023-05-09 16:44:25,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 6: [2023-05-09 16:44:25,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 7: [2023-05-09 16:44:25,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 6: [2023-05-09 16:44:25,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 8: [2023-05-09 16:44:25,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 7: [2023-05-09 16:44:25,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 7: [2023-05-09 16:44:25,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 8: [2023-05-09 16:44:25,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +27: [2023-05-09 16:44:25,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt. +27: [2023-05-09 16:44:25,080] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 222 +29: [2023-05-09 16:44:25,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 0: [2023-05-09 16:44:25,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 0: [2023-05-09 16:44:25,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 6: [2023-05-09 16:44:25,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 6: [2023-05-09 16:44:25,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +29: [2023-05-09 16:44:25,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:25,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 2: [2023-05-09 16:44:25,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:25,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 2: [2023-05-09 16:44:25,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +11: [2023-05-09 16:44:25,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt. + 0: [2023-05-09 16:44:25,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 0: [2023-05-09 16:44:25,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +11: [2023-05-09 16:44:25,082] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 92 +28: [2023-05-09 16:44:25,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +28: [2023-05-09 16:44:25,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +28: [2023-05-09 16:44:25,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +28: [2023-05-09 16:44:25,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +25: [2023-05-09 16:44:25,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +29: [2023-05-09 16:44:25,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +29: [2023-05-09 16:44:25,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +15: [2023-05-09 16:44:25,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +27: [2023-05-09 16:44:25,085] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 222 +15: [2023-05-09 16:44:25,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +25: [2023-05-09 16:44:25,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +29: [2023-05-09 16:44:25,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +29: [2023-05-09 16:44:25,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +22: [2023-05-09 16:44:25,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +15: [2023-05-09 16:44:25,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +22: [2023-05-09 16:44:25,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +22: [2023-05-09 16:44:25,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +22: [2023-05-09 16:44:25,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +22: [2023-05-09 16:44:25,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 6: [2023-05-09 16:44:25,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +29: [2023-05-09 16:44:25,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 6: [2023-05-09 16:44:25,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +22: [2023-05-09 16:44:25,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +29: [2023-05-09 16:44:25,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +11: [2023-05-09 16:44:25,087] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 92 +15: [2023-05-09 16:44:25,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 5: [2023-05-09 16:44:25,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +28: [2023-05-09 16:44:25,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +28: [2023-05-09 16:44:25,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 8: [2023-05-09 16:44:25,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 8: [2023-05-09 16:44:25,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 6: [2023-05-09 16:44:25,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 5: [2023-05-09 16:44:25,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 5: [2023-05-09 16:44:25,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +12: [2023-05-09 16:44:25,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +12: [2023-05-09 16:44:25,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 6: [2023-05-09 16:44:25,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 2: [2023-05-09 16:44:25,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +22: [2023-05-09 16:44:25,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +22: [2023-05-09 16:44:25,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 7: [2023-05-09 16:44:25,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +25: [2023-05-09 16:44:25,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +25: [2023-05-09 16:44:25,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +25: [2023-05-09 16:44:25,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 7: [2023-05-09 16:44:25,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +25: [2023-05-09 16:44:25,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 8: [2023-05-09 16:44:25,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 5: [2023-05-09 16:44:25,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 5: [2023-05-09 16:44:25,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 5: [2023-05-09 16:44:25,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 8: [2023-05-09 16:44:25,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 7: [2023-05-09 16:44:25,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 2: [2023-05-09 16:44:25,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 7: [2023-05-09 16:44:25,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 8: [2023-05-09 16:44:25,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 8: [2023-05-09 16:44:25,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 7: [2023-05-09 16:44:25,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 7: [2023-05-09 16:44:25,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 5: [2023-05-09 16:44:25,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 6: [2023-05-09 16:44:25,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt. + 6: [2023-05-09 16:44:25,099] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 54 + 5: [2023-05-09 16:44:25,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 3: [2023-05-09 16:44:25,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt. +25: [2023-05-09 16:44:25,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt. +25: [2023-05-09 16:44:25,100] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 200 + 3: [2023-05-09 16:44:25,100] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 30 + 7: [2023-05-09 16:44:25,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 7: [2023-05-09 16:44:25,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +15: [2023-05-09 16:44:25,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +15: [2023-05-09 16:44:25,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 6: [2023-05-09 16:44:25,103] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 54 +25: [2023-05-09 16:44:25,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt. +25: [2023-05-09 16:44:25,104] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 206 +15: [2023-05-09 16:44:25,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +15: [2023-05-09 16:44:25,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 3: [2023-05-09 16:44:25,105] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 30 +25: [2023-05-09 16:44:25,105] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 200 +24: [2023-05-09 16:44:25,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +24: [2023-05-09 16:44:25,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +30: [2023-05-09 16:44:25,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +30: [2023-05-09 16:44:25,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +30: [2023-05-09 16:44:25,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +30: [2023-05-09 16:44:25,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +24: [2023-05-09 16:44:25,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +24: [2023-05-09 16:44:25,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 6: [2023-05-09 16:44:25,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt. + 6: [2023-05-09 16:44:25,107] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 50 +15: [2023-05-09 16:44:25,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +15: [2023-05-09 16:44:25,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +12: [2023-05-09 16:44:25,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt. +12: [2023-05-09 16:44:25,109] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 98 +25: [2023-05-09 16:44:25,110] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 206 + 6: [2023-05-09 16:44:25,112] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 50 +30: [2023-05-09 16:44:25,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +30: [2023-05-09 16:44:25,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +30: [2023-05-09 16:44:25,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 6: [2023-05-09 16:44:25,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt. + 6: [2023-05-09 16:44:25,116] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 48 +24: [2023-05-09 16:44:25,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +24: [2023-05-09 16:44:25,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +12: [2023-05-09 16:44:25,113] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 98 + 3: [2023-05-09 16:44:25,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt. +30: [2023-05-09 16:44:25,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 3: [2023-05-09 16:44:25,117] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 24 +24: [2023-05-09 16:44:25,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +29: [2023-05-09 16:44:25,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt. +29: [2023-05-09 16:44:25,119] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 238 +24: [2023-05-09 16:44:25,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +15: [2023-05-09 16:44:25,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +23: [2023-05-09 16:44:25,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt. +23: [2023-05-09 16:44:25,120] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 188 +14: [2023-05-09 16:44:25,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt. +10: [2023-05-09 16:44:25,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt. +14: [2023-05-09 16:44:25,120] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 118 +10: [2023-05-09 16:44:25,120] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 80 + 6: [2023-05-09 16:44:25,120] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 48 +15: [2023-05-09 16:44:25,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 3: [2023-05-09 16:44:25,121] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 24 + 0: [2023-05-09 16:44:25,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. + 0: [2023-05-09 16:44:25,122] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 6 +20: [2023-05-09 16:44:25,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt. +20: [2023-05-09 16:44:25,122] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 160 +29: [2023-05-09 16:44:25,124] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 238 +23: [2023-05-09 16:44:25,124] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 188 +14: [2023-05-09 16:44:25,124] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 118 +10: [2023-05-09 16:44:25,124] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 80 +20: [2023-05-09 16:44:25,127] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 160 +30: [2023-05-09 16:44:25,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 0: [2023-05-09 16:44:25,128] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 6 +24: [2023-05-09 16:44:25,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +30: [2023-05-09 16:44:25,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +14: [2023-05-09 16:44:25,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt. +30: [2023-05-09 16:44:25,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +30: [2023-05-09 16:44:25,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +14: [2023-05-09 16:44:25,128] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 112 +30: [2023-05-09 16:44:25,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +30: [2023-05-09 16:44:25,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +31: [2023-05-09 16:44:25,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt. +31: [2023-05-09 16:44:25,132] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 254 +24: [2023-05-09 16:44:25,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +14: [2023-05-09 16:44:25,133] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 112 +24: [2023-05-09 16:44:25,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +13: [2023-05-09 16:44:25,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt. +24: [2023-05-09 16:44:25,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +13: [2023-05-09 16:44:25,134] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 104 +24: [2023-05-09 16:44:25,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +24: [2023-05-09 16:44:25,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +30: [2023-05-09 16:44:25,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt. +30: [2023-05-09 16:44:25,135] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 244 + 6: [2023-05-09 16:44:25,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt. +25: [2023-05-09 16:44:25,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt. +25: [2023-05-09 16:44:25,137] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 204 + 6: [2023-05-09 16:44:25,137] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 52 +30: [2023-05-09 16:44:25,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +30: [2023-05-09 16:44:25,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +13: [2023-05-09 16:44:25,138] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 104 +31: [2023-05-09 16:44:25,138] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 254 +30: [2023-05-09 16:44:25,140] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 244 +14: [2023-05-09 16:44:25,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt. +14: [2023-05-09 16:44:25,141] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 114 + 6: [2023-05-09 16:44:25,141] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 52 +10: [2023-05-09 16:44:25,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt. +16: [2023-05-09 16:44:25,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt. +16: [2023-05-09 16:44:25,142] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 134 +10: [2023-05-09 16:44:25,142] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 82 +25: [2023-05-09 16:44:25,142] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 204 +15: [2023-05-09 16:44:25,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt. +24: [2023-05-09 16:44:25,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +15: [2023-05-09 16:44:25,143] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 124 +24: [2023-05-09 16:44:25,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +23: [2023-05-09 16:44:25,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +23: [2023-05-09 16:44:25,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +23: [2023-05-09 16:44:25,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +23: [2023-05-09 16:44:25,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +14: [2023-05-09 16:44:25,146] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 114 +10: [2023-05-09 16:44:25,146] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 82 +16: [2023-05-09 16:44:25,146] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 134 +18: [2023-05-09 16:44:25,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +18: [2023-05-09 16:44:25,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +15: [2023-05-09 16:44:25,147] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 124 +11: [2023-05-09 16:44:25,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt. + 1: [2023-05-09 16:44:25,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 1: [2023-05-09 16:44:25,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 1: [2023-05-09 16:44:25,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +11: [2023-05-09 16:44:25,148] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 88 + 1: [2023-05-09 16:44:25,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +18: [2023-05-09 16:44:25,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +18: [2023-05-09 16:44:25,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +19: [2023-05-09 16:44:25,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +19: [2023-05-09 16:44:25,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +19: [2023-05-09 16:44:25,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +19: [2023-05-09 16:44:25,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +15: [2023-05-09 16:44:25,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt. +15: [2023-05-09 16:44:25,153] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 126 +11: [2023-05-09 16:44:25,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +11: [2023-05-09 16:44:25,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +11: [2023-05-09 16:44:25,153] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 88 +11: [2023-05-09 16:44:25,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +23: [2023-05-09 16:44:25,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +11: [2023-05-09 16:44:25,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +23: [2023-05-09 16:44:25,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +23: [2023-05-09 16:44:25,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +23: [2023-05-09 16:44:25,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +27: [2023-05-09 16:44:25,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +27: [2023-05-09 16:44:25,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +27: [2023-05-09 16:44:25,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +21: [2023-05-09 16:44:25,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +27: [2023-05-09 16:44:25,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 8: [2023-05-09 16:44:25,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt. + 8: [2023-05-09 16:44:25,157] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 64 +15: [2023-05-09 16:44:25,157] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 126 +18: [2023-05-09 16:44:25,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 1: [2023-05-09 16:44:25,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +21: [2023-05-09 16:44:25,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 1: [2023-05-09 16:44:25,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +15: [2023-05-09 16:44:25,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt. + 1: [2023-05-09 16:44:25,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +15: [2023-05-09 16:44:25,159] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 122 +18: [2023-05-09 16:44:25,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +18: [2023-05-09 16:44:25,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +18: [2023-05-09 16:44:25,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +19: [2023-05-09 16:44:25,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +19: [2023-05-09 16:44:25,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +16: [2023-05-09 16:44:25,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +16: [2023-05-09 16:44:25,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +16: [2023-05-09 16:44:25,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +16: [2023-05-09 16:44:25,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +19: [2023-05-09 16:44:25,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 1: [2023-05-09 16:44:25,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 8: [2023-05-09 16:44:25,162] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 64 +19: [2023-05-09 16:44:25,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +11: [2023-05-09 16:44:25,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +11: [2023-05-09 16:44:25,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +21: [2023-05-09 16:44:25,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +15: [2023-05-09 16:44:25,163] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 122 +11: [2023-05-09 16:44:25,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +27: [2023-05-09 16:44:25,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +27: [2023-05-09 16:44:25,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +21: [2023-05-09 16:44:25,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 3: [2023-05-09 16:44:25,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt. + 3: [2023-05-09 16:44:25,165] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 28 +27: [2023-05-09 16:44:25,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +11: [2023-05-09 16:44:25,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +21: [2023-05-09 16:44:25,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +27: [2023-05-09 16:44:25,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +31: [2023-05-09 16:44:25,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +23: [2023-05-09 16:44:25,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +31: [2023-05-09 16:44:25,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +31: [2023-05-09 16:44:25,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +10: [2023-05-09 16:44:25,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +31: [2023-05-09 16:44:25,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +10: [2023-05-09 16:44:25,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +23: [2023-05-09 16:44:25,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +23: [2023-05-09 16:44:25,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +10: [2023-05-09 16:44:25,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +23: [2023-05-09 16:44:25,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +10: [2023-05-09 16:44:25,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +23: [2023-05-09 16:44:25,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt. +21: [2023-05-09 16:44:25,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 3: [2023-05-09 16:44:25,169] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 28 +23: [2023-05-09 16:44:25,169] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 186 +16: [2023-05-09 16:44:25,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +16: [2023-05-09 16:44:25,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +16: [2023-05-09 16:44:25,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +16: [2023-05-09 16:44:25,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 1: [2023-05-09 16:44:25,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 0: [2023-05-09 16:44:25,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. +18: [2023-05-09 16:44:25,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +21: [2023-05-09 16:44:25,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 0: [2023-05-09 16:44:25,173] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 4 +23: [2023-05-09 16:44:25,173] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 186 +23: [2023-05-09 16:44:25,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +11: [2023-05-09 16:44:25,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +21: [2023-05-09 16:44:25,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +19: [2023-05-09 16:44:25,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt. + 1: [2023-05-09 16:44:25,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +11: [2023-05-09 16:44:25,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +19: [2023-05-09 16:44:25,175] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 156 +18: [2023-05-09 16:44:25,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +23: [2023-05-09 16:44:25,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +31: [2023-05-09 16:44:25,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +19: [2023-05-09 16:44:25,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +19: [2023-05-09 16:44:25,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +18: [2023-05-09 16:44:25,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +18: [2023-05-09 16:44:25,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +11: [2023-05-09 16:44:25,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +11: [2023-05-09 16:44:25,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +19: [2023-05-09 16:44:25,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +19: [2023-05-09 16:44:25,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +31: [2023-05-09 16:44:25,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +31: [2023-05-09 16:44:25,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 0: [2023-05-09 16:44:25,177] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 4 +19: [2023-05-09 16:44:25,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +10: [2023-05-09 16:44:25,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +10: [2023-05-09 16:44:25,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +19: [2023-05-09 16:44:25,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +18: [2023-05-09 16:44:25,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +18: [2023-05-09 16:44:25,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +10: [2023-05-09 16:44:25,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +10: [2023-05-09 16:44:25,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 1: [2023-05-09 16:44:25,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 1: [2023-05-09 16:44:25,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +31: [2023-05-09 16:44:25,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 1: [2023-05-09 16:44:25,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 1: [2023-05-09 16:44:25,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +19: [2023-05-09 16:44:25,180] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 156 +27: [2023-05-09 16:44:25,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +27: [2023-05-09 16:44:25,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +27: [2023-05-09 16:44:25,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +27: [2023-05-09 16:44:25,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +27: [2023-05-09 16:44:25,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +27: [2023-05-09 16:44:25,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +16: [2023-05-09 16:44:25,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +23: [2023-05-09 16:44:25,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +16: [2023-05-09 16:44:25,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +16: [2023-05-09 16:44:25,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 8: [2023-05-09 16:44:25,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt. + 8: [2023-05-09 16:44:25,184] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 70 +29: [2023-05-09 16:44:25,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt. +29: [2023-05-09 16:44:25,185] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 234 +21: [2023-05-09 16:44:25,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +23: [2023-05-09 16:44:25,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +16: [2023-05-09 16:44:25,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +18: [2023-05-09 16:44:25,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +11: [2023-05-09 16:44:25,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +16: [2023-05-09 16:44:25,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +16: [2023-05-09 16:44:25,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +16: [2023-05-09 16:44:25,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +16: [2023-05-09 16:44:25,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +18: [2023-05-09 16:44:25,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 9: [2023-05-09 16:44:25,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt. +21: [2023-05-09 16:44:25,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 9: [2023-05-09 16:44:25,187] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 76 +21: [2023-05-09 16:44:25,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +11: [2023-05-09 16:44:25,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +21: [2023-05-09 16:44:25,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +29: [2023-05-09 16:44:25,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt. +29: [2023-05-09 16:44:25,188] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 236 + 8: [2023-05-09 16:44:25,189] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 70 +21: [2023-05-09 16:44:25,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +29: [2023-05-09 16:44:25,189] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 234 +21: [2023-05-09 16:44:25,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 1: [2023-05-09 16:44:25,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +19: [2023-05-09 16:44:25,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +20: [2023-05-09 16:44:25,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt. +20: [2023-05-09 16:44:25,191] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 166 +27: [2023-05-09 16:44:25,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 1: [2023-05-09 16:44:25,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt. + 1: [2023-05-09 16:44:25,191] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 10 + 9: [2023-05-09 16:44:25,191] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 76 + 3: [2023-05-09 16:44:25,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 3: [2023-05-09 16:44:25,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 3: [2023-05-09 16:44:25,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 3: [2023-05-09 16:44:25,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 1: [2023-05-09 16:44:25,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +19: [2023-05-09 16:44:25,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +29: [2023-05-09 16:44:25,192] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 236 +27: [2023-05-09 16:44:25,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +13: [2023-05-09 16:44:25,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +13: [2023-05-09 16:44:25,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +13: [2023-05-09 16:44:25,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +13: [2023-05-09 16:44:25,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 4: [2023-05-09 16:44:25,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +31: [2023-05-09 16:44:25,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 4: [2023-05-09 16:44:25,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 4: [2023-05-09 16:44:25,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 4: [2023-05-09 16:44:25,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +31: [2023-05-09 16:44:25,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +27: [2023-05-09 16:44:25,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt. +27: [2023-05-09 16:44:25,195] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 216 + 9: [2023-05-09 16:44:25,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt. +15: [2023-05-09 16:44:25,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt. + 9: [2023-05-09 16:44:25,195] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 72 +15: [2023-05-09 16:44:25,195] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 120 + 1: [2023-05-09 16:44:25,195] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 10 +11: [2023-05-09 16:44:25,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +20: [2023-05-09 16:44:25,196] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 166 +31: [2023-05-09 16:44:25,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +31: [2023-05-09 16:44:25,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 9: [2023-05-09 16:44:25,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +11: [2023-05-09 16:44:25,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 9: [2023-05-09 16:44:25,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. + 9: [2023-05-09 16:44:25,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +10: [2023-05-09 16:44:25,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 9: [2023-05-09 16:44:25,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt. +10: [2023-05-09 16:44:25,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +27: [2023-05-09 16:44:25,199] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 216 +10: [2023-05-09 16:44:25,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 9: [2023-05-09 16:44:25,199] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 72 +10: [2023-05-09 16:44:25,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +10: [2023-05-09 16:44:25,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +10: [2023-05-09 16:44:25,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +21: [2023-05-09 16:44:25,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +15: [2023-05-09 16:44:25,200] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 120 +31: [2023-05-09 16:44:25,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 9: [2023-05-09 16:44:25,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt. + 9: [2023-05-09 16:44:25,201] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 78 +13: [2023-05-09 16:44:25,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +13: [2023-05-09 16:44:25,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +21: [2023-05-09 16:44:25,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 3: [2023-05-09 16:44:25,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +13: [2023-05-09 16:44:25,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 3: [2023-05-09 16:44:25,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 3: [2023-05-09 16:44:25,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +31: [2023-05-09 16:44:25,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 3: [2023-05-09 16:44:25,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +10: [2023-05-09 16:44:25,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +10: [2023-05-09 16:44:25,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 4: [2023-05-09 16:44:25,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +13: [2023-05-09 16:44:25,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 8: [2023-05-09 16:44:25,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt. + 4: [2023-05-09 16:44:25,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 8: [2023-05-09 16:44:25,204] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 68 + 4: [2023-05-09 16:44:25,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +14: [2023-05-09 16:44:25,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt. +14: [2023-05-09 16:44:25,206] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 116 + 4: [2023-05-09 16:44:25,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 9: [2023-05-09 16:44:25,208] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 78 + 9: [2023-05-09 16:44:25,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 9: [2023-05-09 16:44:25,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 8: [2023-05-09 16:44:25,209] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 68 +31: [2023-05-09 16:44:25,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +14: [2023-05-09 16:44:25,209] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 116 +26: [2023-05-09 16:44:25,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt. +26: [2023-05-09 16:44:25,211] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 208 +31: [2023-05-09 16:44:25,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 9: [2023-05-09 16:44:25,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... + 0: [2023-05-09 16:44:25,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. + 0: [2023-05-09 16:44:25,211] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 2 + 0: [2023-05-09 16:44:25,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. + 0: [2023-05-09 16:44:25,213] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 0 + 9: [2023-05-09 16:44:25,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt... +26: [2023-05-09 16:44:25,215] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 208 +19: [2023-05-09 16:44:25,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt. +19: [2023-05-09 16:44:25,216] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 158 + 0: [2023-05-09 16:44:25,216] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 2 + 0: [2023-05-09 16:44:25,217] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 0 +27: [2023-05-09 16:44:25,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt. +27: [2023-05-09 16:44:25,218] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 218 +21: [2023-05-09 16:44:25,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt. +21: [2023-05-09 16:44:25,219] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 170 +13: [2023-05-09 16:44:25,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 3: [2023-05-09 16:44:25,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 3: [2023-05-09 16:44:25,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 3: [2023-05-09 16:44:25,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 3: [2023-05-09 16:44:25,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +13: [2023-05-09 16:44:25,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 3: [2023-05-09 16:44:25,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 3: [2023-05-09 16:44:25,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 3: [2023-05-09 16:44:25,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 3: [2023-05-09 16:44:25,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +19: [2023-05-09 16:44:25,220] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 158 + 9: [2023-05-09 16:44:25,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +13: [2023-05-09 16:44:25,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +13: [2023-05-09 16:44:25,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +13: [2023-05-09 16:44:25,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +13: [2023-05-09 16:44:25,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 0: could not find arguments in the checkpoint ... + 0: checkpoint version 3.0 + 1: [2023-05-09 16:44:25,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt. +27: [2023-05-09 16:44:25,223] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 218 + 1: [2023-05-09 16:44:25,223] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 12 +21: [2023-05-09 16:44:25,223] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 170 + 9: [2023-05-09 16:44:25,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 9: [2023-05-09 16:44:25,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 9: [2023-05-09 16:44:25,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 9: [2023-05-09 16:44:25,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 9: [2023-05-09 16:44:25,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +19: [2023-05-09 16:44:25,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt. +19: [2023-05-09 16:44:25,226] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 152 +31: [2023-05-09 16:44:25,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt. +31: [2023-05-09 16:44:25,227] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 250 + 1: [2023-05-09 16:44:25,227] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 12 +13: [2023-05-09 16:44:25,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 4: [2023-05-09 16:44:25,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 4: [2023-05-09 16:44:25,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 4: [2023-05-09 16:44:25,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 4: [2023-05-09 16:44:25,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 4: [2023-05-09 16:44:25,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 4: [2023-05-09 16:44:25,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +13: [2023-05-09 16:44:25,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... +31: [2023-05-09 16:44:25,231] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 250 +19: [2023-05-09 16:44:25,231] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 152 +26: [2023-05-09 16:44:25,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt. +26: [2023-05-09 16:44:25,232] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 212 + 4: [2023-05-09 16:44:25,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt. +29: [2023-05-09 16:44:25,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt. + 4: [2023-05-09 16:44:25,233] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 34 +29: [2023-05-09 16:44:25,233] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 232 +13: [2023-05-09 16:44:25,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt. +13: [2023-05-09 16:44:25,234] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 108 +31: [2023-05-09 16:44:25,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt. +31: [2023-05-09 16:44:25,235] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 252 + 3: [2023-05-09 16:44:25,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt. + 3: [2023-05-09 16:44:25,235] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 26 +26: [2023-05-09 16:44:25,236] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 212 +19: [2023-05-09 16:44:25,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt. +19: [2023-05-09 16:44:25,237] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 154 +29: [2023-05-09 16:44:25,238] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 232 + 4: [2023-05-09 16:44:25,238] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 34 +13: [2023-05-09 16:44:25,239] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 108 +31: [2023-05-09 16:44:25,239] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 252 + 9: [2023-05-09 16:44:25,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. + 3: [2023-05-09 16:44:25,240] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 26 + 9: [2023-05-09 16:44:25,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 9: [2023-05-09 16:44:25,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt. + 9: [2023-05-09 16:44:25,241] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 74 +19: [2023-05-09 16:44:25,241] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 154 + 4: [2023-05-09 16:44:25,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt. +20: [2023-05-09 16:44:25,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt. +20: [2023-05-09 16:44:25,244] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 162 + 4: [2023-05-09 16:44:25,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt... + 1: [2023-05-09 16:44:25,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt. +13: [2023-05-09 16:44:25,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt. + 1: [2023-05-09 16:44:25,245] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 8 +13: [2023-05-09 16:44:25,245] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 110 +18: [2023-05-09 16:44:25,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt. +18: [2023-05-09 16:44:25,246] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 148 + 9: [2023-05-09 16:44:25,246] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 74 +20: [2023-05-09 16:44:25,248] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 162 + 1: [2023-05-09 16:44:25,250] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 8 +18: [2023-05-09 16:44:25,250] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 148 +13: [2023-05-09 16:44:25,250] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 110 +22: [2023-05-09 16:44:25,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt. +22: [2023-05-09 16:44:25,251] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 180 +23: [2023-05-09 16:44:25,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt. +23: [2023-05-09 16:44:25,256] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 184 +26: [2023-05-09 16:44:25,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +26: [2023-05-09 16:44:25,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +26: [2023-05-09 16:44:25,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +21: [2023-05-09 16:44:25,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt. +22: [2023-05-09 16:44:25,258] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 180 +26: [2023-05-09 16:44:25,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +21: [2023-05-09 16:44:25,258] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 172 +23: [2023-05-09 16:44:25,260] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 184 +20: [2023-05-09 16:44:25,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +20: [2023-05-09 16:44:25,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +20: [2023-05-09 16:44:25,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +20: [2023-05-09 16:44:25,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +22: [2023-05-09 16:44:25,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt. +22: [2023-05-09 16:44:25,262] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 176 +21: [2023-05-09 16:44:25,262] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 172 + 2: [2023-05-09 16:44:25,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt. + 2: [2023-05-09 16:44:25,266] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 22 +21: [2023-05-09 16:44:25,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt. +21: [2023-05-09 16:44:25,266] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 168 +22: [2023-05-09 16:44:25,267] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 176 +26: [2023-05-09 16:44:25,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +26: [2023-05-09 16:44:25,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +26: [2023-05-09 16:44:25,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +26: [2023-05-09 16:44:25,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 2: [2023-05-09 16:44:25,270] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 22 +17: [2023-05-09 16:44:25,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +21: [2023-05-09 16:44:25,271] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 168 +30: [2023-05-09 16:44:25,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt. +30: [2023-05-09 16:44:25,271] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 246 +17: [2023-05-09 16:44:25,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +17: [2023-05-09 16:44:25,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +20: [2023-05-09 16:44:25,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +20: [2023-05-09 16:44:25,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +20: [2023-05-09 16:44:25,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +20: [2023-05-09 16:44:25,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +17: [2023-05-09 16:44:25,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +12: [2023-05-09 16:44:25,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +12: [2023-05-09 16:44:25,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +12: [2023-05-09 16:44:25,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +16: [2023-05-09 16:44:25,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt. + 7: [2023-05-09 16:44:25,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt. +12: [2023-05-09 16:44:25,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +16: [2023-05-09 16:44:25,275] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 132 + 7: [2023-05-09 16:44:25,275] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 58 +30: [2023-05-09 16:44:25,275] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 246 + 2: [2023-05-09 16:44:25,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt. +22: [2023-05-09 16:44:25,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt. +22: [2023-05-09 16:44:25,276] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 182 + 2: [2023-05-09 16:44:25,276] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 18 + 7: [2023-05-09 16:44:25,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt. + 7: [2023-05-09 16:44:25,276] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 60 +28: [2023-05-09 16:44:25,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +28: [2023-05-09 16:44:25,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +28: [2023-05-09 16:44:25,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +28: [2023-05-09 16:44:25,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +18: [2023-05-09 16:44:25,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt. +18: [2023-05-09 16:44:25,278] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 146 + 7: [2023-05-09 16:44:25,279] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 58 +16: [2023-05-09 16:44:25,279] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 132 +22: [2023-05-09 16:44:25,280] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 182 + 8: [2023-05-09 16:44:25,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt. + 2: [2023-05-09 16:44:25,280] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 18 + 8: [2023-05-09 16:44:25,280] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 66 + 7: [2023-05-09 16:44:25,281] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 60 + 2: [2023-05-09 16:44:25,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt. +17: [2023-05-09 16:44:25,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 2: [2023-05-09 16:44:25,281] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 16 +14: [2023-05-09 16:44:25,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +14: [2023-05-09 16:44:25,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +17: [2023-05-09 16:44:25,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +14: [2023-05-09 16:44:25,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +14: [2023-05-09 16:44:25,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +18: [2023-05-09 16:44:25,282] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 146 +12: [2023-05-09 16:44:25,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +11: [2023-05-09 16:44:25,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt. +24: [2023-05-09 16:44:25,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt. +11: [2023-05-09 16:44:25,284] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 94 +12: [2023-05-09 16:44:25,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +12: [2023-05-09 16:44:25,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +24: [2023-05-09 16:44:25,284] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 198 +17: [2023-05-09 16:44:25,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +12: [2023-05-09 16:44:25,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +17: [2023-05-09 16:44:25,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 8: [2023-05-09 16:44:25,285] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 66 + 2: [2023-05-09 16:44:25,285] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 16 +28: [2023-05-09 16:44:25,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +28: [2023-05-09 16:44:25,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt. +28: [2023-05-09 16:44:25,285] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 228 +31: [2023-05-09 16:44:25,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt. +31: [2023-05-09 16:44:25,286] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 248 + 0: [2023-05-09 16:44:25,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 0: [2023-05-09 16:44:25,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 0: [2023-05-09 16:44:25,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +28: [2023-05-09 16:44:25,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +26: [2023-05-09 16:44:25,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 0: [2023-05-09 16:44:25,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +26: [2023-05-09 16:44:25,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +17: [2023-05-09 16:44:25,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt. +20: [2023-05-09 16:44:25,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +20: [2023-05-09 16:44:25,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +26: [2023-05-09 16:44:25,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +26: [2023-05-09 16:44:25,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +17: [2023-05-09 16:44:25,287] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 136 +28: [2023-05-09 16:44:25,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +28: [2023-05-09 16:44:25,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +20: [2023-05-09 16:44:25,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +20: [2023-05-09 16:44:25,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +20: [2023-05-09 16:44:25,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +20: [2023-05-09 16:44:25,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +20: [2023-05-09 16:44:25,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +20: [2023-05-09 16:44:25,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +24: [2023-05-09 16:44:25,288] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 198 +11: [2023-05-09 16:44:25,289] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 94 +28: [2023-05-09 16:44:25,290] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 228 +16: [2023-05-09 16:44:25,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt. +16: [2023-05-09 16:44:25,290] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 128 +31: [2023-05-09 16:44:25,290] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 248 +14: [2023-05-09 16:44:25,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +26: [2023-05-09 16:44:25,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +14: [2023-05-09 16:44:25,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +14: [2023-05-09 16:44:25,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +16: [2023-05-09 16:44:25,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt. +14: [2023-05-09 16:44:25,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +16: [2023-05-09 16:44:25,292] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 130 +17: [2023-05-09 16:44:25,292] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 136 +26: [2023-05-09 16:44:25,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +26: [2023-05-09 16:44:25,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +12: [2023-05-09 16:44:25,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +16: [2023-05-09 16:44:25,295] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 128 +12: [2023-05-09 16:44:25,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +26: [2023-05-09 16:44:25,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 7: [2023-05-09 16:44:25,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt. + 7: [2023-05-09 16:44:25,295] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 56 +12: [2023-05-09 16:44:25,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +12: [2023-05-09 16:44:25,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +16: [2023-05-09 16:44:25,296] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 130 + 0: [2023-05-09 16:44:25,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 0: [2023-05-09 16:44:25,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +28: [2023-05-09 16:44:25,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +18: [2023-05-09 16:44:25,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt. +18: [2023-05-09 16:44:25,298] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 144 +28: [2023-05-09 16:44:25,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +12: [2023-05-09 16:44:25,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +12: [2023-05-09 16:44:25,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +12: [2023-05-09 16:44:25,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +12: [2023-05-09 16:44:25,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 0: [2023-05-09 16:44:25,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 0: [2023-05-09 16:44:25,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 7: [2023-05-09 16:44:25,300] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 56 +17: [2023-05-09 16:44:25,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +17: [2023-05-09 16:44:25,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 5: [2023-05-09 16:44:25,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt. +17: [2023-05-09 16:44:25,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt. + 5: [2023-05-09 16:44:25,301] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 42 +17: [2023-05-09 16:44:25,301] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 140 +17: [2023-05-09 16:44:25,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +18: [2023-05-09 16:44:25,303] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 144 +17: [2023-05-09 16:44:25,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +11: [2023-05-09 16:44:25,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt. +11: [2023-05-09 16:44:25,305] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 90 + 5: [2023-05-09 16:44:25,305] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 42 +17: [2023-05-09 16:44:25,306] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 140 +13: [2023-05-09 16:44:25,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt. +13: [2023-05-09 16:44:25,306] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 106 +18: [2023-05-09 16:44:25,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt. +18: [2023-05-09 16:44:25,307] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 150 +30: [2023-05-09 16:44:25,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt. +30: [2023-05-09 16:44:25,307] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 242 +28: [2023-05-09 16:44:25,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +28: [2023-05-09 16:44:25,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +28: [2023-05-09 16:44:25,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +28: [2023-05-09 16:44:25,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +28: [2023-05-09 16:44:25,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +28: [2023-05-09 16:44:25,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +14: [2023-05-09 16:44:25,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +17: [2023-05-09 16:44:25,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +14: [2023-05-09 16:44:25,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +11: [2023-05-09 16:44:25,310] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 90 +13: [2023-05-09 16:44:25,310] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 106 +14: [2023-05-09 16:44:25,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +14: [2023-05-09 16:44:25,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +14: [2023-05-09 16:44:25,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +14: [2023-05-09 16:44:25,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +17: [2023-05-09 16:44:25,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +14: [2023-05-09 16:44:25,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +14: [2023-05-09 16:44:25,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +18: [2023-05-09 16:44:25,311] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 150 +30: [2023-05-09 16:44:25,311] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 242 +10: [2023-05-09 16:44:25,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt. +10: [2023-05-09 16:44:25,312] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 86 +17: [2023-05-09 16:44:25,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +24: [2023-05-09 16:44:25,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt. +24: [2023-05-09 16:44:25,315] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 196 +10: [2023-05-09 16:44:25,316] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 86 +17: [2023-05-09 16:44:25,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +21: [2023-05-09 16:44:25,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt. +21: [2023-05-09 16:44:25,317] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 174 + 1: [2023-05-09 16:44:25,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt. + 1: [2023-05-09 16:44:25,317] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 14 + 0: [2023-05-09 16:44:25,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 0: [2023-05-09 16:44:25,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 0: [2023-05-09 16:44:25,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 0: [2023-05-09 16:44:25,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 4: [2023-05-09 16:44:25,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt. + 4: [2023-05-09 16:44:25,319] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 38 +24: [2023-05-09 16:44:25,319] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 196 +21: [2023-05-09 16:44:25,321] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 174 + 1: [2023-05-09 16:44:25,321] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 14 +17: [2023-05-09 16:44:25,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt. +28: [2023-05-09 16:44:25,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt. +28: [2023-05-09 16:44:25,323] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 224 +17: [2023-05-09 16:44:25,323] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 142 + 4: [2023-05-09 16:44:25,323] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 38 +28: [2023-05-09 16:44:25,327] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 224 +17: [2023-05-09 16:44:25,327] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 142 + 5: [2023-05-09 16:44:25,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt. + 5: [2023-05-09 16:44:25,328] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 40 +17: [2023-05-09 16:44:25,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt. +17: [2023-05-09 16:44:25,329] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 138 + 0: [2023-05-09 16:44:25,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +10: [2023-05-09 16:44:25,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt. +10: [2023-05-09 16:44:25,330] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 84 + 0: [2023-05-09 16:44:25,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 0: [2023-05-09 16:44:25,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 0: [2023-05-09 16:44:25,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 5: [2023-05-09 16:44:25,333] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 40 +28: [2023-05-09 16:44:25,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt. +17: [2023-05-09 16:44:25,334] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 138 +28: [2023-05-09 16:44:25,334] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 230 +10: [2023-05-09 16:44:25,334] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 84 +28: [2023-05-09 16:44:25,338] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 230 +24: [2023-05-09 16:44:25,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt. +24: [2023-05-09 16:44:25,341] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 194 +24: [2023-05-09 16:44:25,345] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 194 +26: [2023-05-09 16:44:25,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt. +26: [2023-05-09 16:44:25,347] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 210 +26: [2023-05-09 16:44:25,351] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 210 +30: [2023-05-09 16:44:25,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt. +30: [2023-05-09 16:44:25,360] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 240 + 4: [2023-05-09 16:44:25,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt. + 4: [2023-05-09 16:44:25,362] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 36 + 7: [2023-05-09 16:44:25,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt. + 7: [2023-05-09 16:44:25,362] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 62 +30: [2023-05-09 16:44:25,364] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 240 + 5: [2023-05-09 16:44:25,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 5: [2023-05-09 16:44:25,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 5: [2023-05-09 16:44:25,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 4: [2023-05-09 16:44:25,366] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 36 + 7: [2023-05-09 16:44:25,367] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 62 +10: [2023-05-09 16:44:25,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 7: [2023-05-09 16:44:25,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 7: [2023-05-09 16:44:25,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 5: [2023-05-09 16:44:25,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +10: [2023-05-09 16:44:25,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +10: [2023-05-09 16:44:25,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 7: [2023-05-09 16:44:25,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +10: [2023-05-09 16:44:25,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 7: [2023-05-09 16:44:25,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +15: [2023-05-09 16:44:25,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +15: [2023-05-09 16:44:25,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +15: [2023-05-09 16:44:25,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +15: [2023-05-09 16:44:25,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +22: [2023-05-09 16:44:25,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt. +22: [2023-05-09 16:44:25,369] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 178 +30: [2023-05-09 16:44:25,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +30: [2023-05-09 16:44:25,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +30: [2023-05-09 16:44:25,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +30: [2023-05-09 16:44:25,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +22: [2023-05-09 16:44:25,374] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 178 +28: [2023-05-09 16:44:25,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt. +28: [2023-05-09 16:44:25,375] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 226 + 5: [2023-05-09 16:44:25,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +10: [2023-05-09 16:44:25,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +10: [2023-05-09 16:44:25,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 5: [2023-05-09 16:44:25,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +10: [2023-05-09 16:44:25,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +15: [2023-05-09 16:44:25,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 5: [2023-05-09 16:44:25,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 5: [2023-05-09 16:44:25,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +15: [2023-05-09 16:44:25,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 7: [2023-05-09 16:44:25,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +28: [2023-05-09 16:44:25,379] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 226 +15: [2023-05-09 16:44:25,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +10: [2023-05-09 16:44:25,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +30: [2023-05-09 16:44:25,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 7: [2023-05-09 16:44:25,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 7: [2023-05-09 16:44:25,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 7: [2023-05-09 16:44:25,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +15: [2023-05-09 16:44:25,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +30: [2023-05-09 16:44:25,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +30: [2023-05-09 16:44:25,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +30: [2023-05-09 16:44:25,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +29: [2023-05-09 16:44:25,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +29: [2023-05-09 16:44:25,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +29: [2023-05-09 16:44:25,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +29: [2023-05-09 16:44:25,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +22: [2023-05-09 16:44:25,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +22: [2023-05-09 16:44:25,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +22: [2023-05-09 16:44:25,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +22: [2023-05-09 16:44:25,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 5: [2023-05-09 16:44:25,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt. +10: [2023-05-09 16:44:25,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 5: [2023-05-09 16:44:25,389] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 44 +10: [2023-05-09 16:44:25,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +30: [2023-05-09 16:44:25,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +30: [2023-05-09 16:44:25,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 5: [2023-05-09 16:44:25,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +30: [2023-05-09 16:44:25,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +30: [2023-05-09 16:44:25,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 5: [2023-05-09 16:44:25,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +15: [2023-05-09 16:44:25,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +15: [2023-05-09 16:44:25,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +30: [2023-05-09 16:44:25,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +30: [2023-05-09 16:44:25,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 5: [2023-05-09 16:44:25,394] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 44 + 5: [2023-05-09 16:44:25,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 5: [2023-05-09 16:44:25,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 5: [2023-05-09 16:44:25,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 5: [2023-05-09 16:44:25,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 7: [2023-05-09 16:44:25,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 7: [2023-05-09 16:44:25,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 7: [2023-05-09 16:44:25,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +10: [2023-05-09 16:44:25,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +10: [2023-05-09 16:44:25,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +10: [2023-05-09 16:44:25,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +10: [2023-05-09 16:44:25,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 2: [2023-05-09 16:44:25,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 2: [2023-05-09 16:44:25,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +29: [2023-05-09 16:44:25,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +22: [2023-05-09 16:44:25,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +22: [2023-05-09 16:44:25,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 2: [2023-05-09 16:44:25,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +29: [2023-05-09 16:44:25,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +16: [2023-05-09 16:44:25,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +16: [2023-05-09 16:44:25,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +22: [2023-05-09 16:44:25,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +16: [2023-05-09 16:44:25,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +16: [2023-05-09 16:44:25,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 7: [2023-05-09 16:44:25,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 7: [2023-05-09 16:44:25,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 7: [2023-05-09 16:44:25,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +29: [2023-05-09 16:44:25,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +15: [2023-05-09 16:44:25,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +15: [2023-05-09 16:44:25,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 1: [2023-05-09 16:44:25,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +15: [2023-05-09 16:44:25,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +29: [2023-05-09 16:44:25,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 1: [2023-05-09 16:44:25,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 1: [2023-05-09 16:44:25,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +15: [2023-05-09 16:44:25,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +22: [2023-05-09 16:44:25,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 5: [2023-05-09 16:44:25,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 1: [2023-05-09 16:44:25,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 5: [2023-05-09 16:44:25,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +15: [2023-05-09 16:44:25,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 8: [2023-05-09 16:44:25,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 8: [2023-05-09 16:44:25,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 2: [2023-05-09 16:44:25,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 7: [2023-05-09 16:44:25,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +15: [2023-05-09 16:44:25,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +16: [2023-05-09 16:44:25,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 7: [2023-05-09 16:44:25,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 8: [2023-05-09 16:44:25,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +16: [2023-05-09 16:44:25,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +16: [2023-05-09 16:44:25,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +16: [2023-05-09 16:44:25,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 8: [2023-05-09 16:44:25,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +30: [2023-05-09 16:44:25,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 2: [2023-05-09 16:44:25,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 2: [2023-05-09 16:44:25,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 2: [2023-05-09 16:44:25,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 1: [2023-05-09 16:44:25,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +30: [2023-05-09 16:44:25,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 1: [2023-05-09 16:44:25,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 1: [2023-05-09 16:44:25,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +24: [2023-05-09 16:44:25,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt. +10: [2023-05-09 16:44:25,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +29: [2023-05-09 16:44:25,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +24: [2023-05-09 16:44:25,410] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 192 + 1: [2023-05-09 16:44:25,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +29: [2023-05-09 16:44:25,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +22: [2023-05-09 16:44:25,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +10: [2023-05-09 16:44:25,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +22: [2023-05-09 16:44:25,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +29: [2023-05-09 16:44:25,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +29: [2023-05-09 16:44:25,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 8: [2023-05-09 16:44:25,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 8: [2023-05-09 16:44:25,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +24: [2023-05-09 16:44:25,416] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 192 +22: [2023-05-09 16:44:25,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 8: [2023-05-09 16:44:25,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +22: [2023-05-09 16:44:25,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +22: [2023-05-09 16:44:25,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +22: [2023-05-09 16:44:25,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 8: [2023-05-09 16:44:25,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 2: [2023-05-09 16:44:25,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +16: [2023-05-09 16:44:25,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +16: [2023-05-09 16:44:25,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +29: [2023-05-09 16:44:25,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +16: [2023-05-09 16:44:25,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +16: [2023-05-09 16:44:25,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +16: [2023-05-09 16:44:25,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +16: [2023-05-09 16:44:25,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +29: [2023-05-09 16:44:25,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +29: [2023-05-09 16:44:25,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt. +29: [2023-05-09 16:44:25,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,423] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 20 +23: [2023-05-09 16:44:25,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt. +23: [2023-05-09 16:44:25,426] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 190 +22: [2023-05-09 16:44:25,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 2: [2023-05-09 16:44:25,427] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 20 + 8: [2023-05-09 16:44:25,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 8: [2023-05-09 16:44:25,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +22: [2023-05-09 16:44:25,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 1: [2023-05-09 16:44:25,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +23: [2023-05-09 16:44:25,431] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 190 + 1: [2023-05-09 16:44:25,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +16: [2023-05-09 16:44:25,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 2: [2023-05-09 16:44:25,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +16: [2023-05-09 16:44:25,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +26: [2023-05-09 16:44:25,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt. +26: [2023-05-09 16:44:25,433] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 214 + 8: [2023-05-09 16:44:25,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +24: [2023-05-09 16:44:25,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +24: [2023-05-09 16:44:25,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 8: [2023-05-09 16:44:25,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 1: [2023-05-09 16:44:25,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +24: [2023-05-09 16:44:25,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 8: [2023-05-09 16:44:25,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 8: [2023-05-09 16:44:25,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 8: [2023-05-09 16:44:25,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 8: [2023-05-09 16:44:25,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +24: [2023-05-09 16:44:25,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 1: [2023-05-09 16:44:25,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 2: [2023-05-09 16:44:25,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 1: [2023-05-09 16:44:25,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 1: [2023-05-09 16:44:25,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 2: [2023-05-09 16:44:25,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +26: [2023-05-09 16:44:25,437] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 214 + 1: [2023-05-09 16:44:25,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 1: [2023-05-09 16:44:25,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +27: [2023-05-09 16:44:25,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +27: [2023-05-09 16:44:25,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +27: [2023-05-09 16:44:25,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +19: [2023-05-09 16:44:25,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +19: [2023-05-09 16:44:25,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +27: [2023-05-09 16:44:25,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +25: [2023-05-09 16:44:25,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +25: [2023-05-09 16:44:25,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +25: [2023-05-09 16:44:25,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +24: [2023-05-09 16:44:25,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +19: [2023-05-09 16:44:25,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +25: [2023-05-09 16:44:25,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +19: [2023-05-09 16:44:25,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +24: [2023-05-09 16:44:25,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +31: [2023-05-09 16:44:25,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +31: [2023-05-09 16:44:25,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +31: [2023-05-09 16:44:25,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +31: [2023-05-09 16:44:25,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +24: [2023-05-09 16:44:25,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +24: [2023-05-09 16:44:25,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 3: [2023-05-09 16:44:25,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 3: [2023-05-09 16:44:25,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 3: [2023-05-09 16:44:25,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 3: [2023-05-09 16:44:25,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +19: [2023-05-09 16:44:25,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +27: [2023-05-09 16:44:25,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +27: [2023-05-09 16:44:25,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +27: [2023-05-09 16:44:25,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 6: [2023-05-09 16:44:25,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 6: [2023-05-09 16:44:25,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +25: [2023-05-09 16:44:25,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +25: [2023-05-09 16:44:25,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +25: [2023-05-09 16:44:25,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +31: [2023-05-09 16:44:25,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +19: [2023-05-09 16:44:25,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +19: [2023-05-09 16:44:25,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +14: [2023-05-09 16:44:25,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +14: [2023-05-09 16:44:25,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +14: [2023-05-09 16:44:25,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +27: [2023-05-09 16:44:25,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +14: [2023-05-09 16:44:25,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +31: [2023-05-09 16:44:25,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +19: [2023-05-09 16:44:25,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +13: [2023-05-09 16:44:25,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +25: [2023-05-09 16:44:25,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 9: [2023-05-09 16:44:25,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 9: [2023-05-09 16:44:25,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 9: [2023-05-09 16:44:25,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +13: [2023-05-09 16:44:25,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +13: [2023-05-09 16:44:25,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 9: [2023-05-09 16:44:25,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +24: [2023-05-09 16:44:25,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +31: [2023-05-09 16:44:25,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +13: [2023-05-09 16:44:25,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +24: [2023-05-09 16:44:25,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 6: [2023-05-09 16:44:25,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 6: [2023-05-09 16:44:25,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +24: [2023-05-09 16:44:25,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +24: [2023-05-09 16:44:25,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +31: [2023-05-09 16:44:25,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +11: [2023-05-09 16:44:25,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +11: [2023-05-09 16:44:25,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +11: [2023-05-09 16:44:25,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +11: [2023-05-09 16:44:25,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +18: [2023-05-09 16:44:25,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 6: [2023-05-09 16:44:25,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +18: [2023-05-09 16:44:25,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +18: [2023-05-09 16:44:25,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +18: [2023-05-09 16:44:25,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 6: [2023-05-09 16:44:25,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +14: [2023-05-09 16:44:25,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +13: [2023-05-09 16:44:25,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 3: [2023-05-09 16:44:25,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 3: [2023-05-09 16:44:25,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 3: [2023-05-09 16:44:25,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +24: [2023-05-09 16:44:25,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +24: [2023-05-09 16:44:25,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 3: [2023-05-09 16:44:25,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +14: [2023-05-09 16:44:25,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +14: [2023-05-09 16:44:25,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +14: [2023-05-09 16:44:25,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 9: [2023-05-09 16:44:25,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 9: [2023-05-09 16:44:25,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 9: [2023-05-09 16:44:25,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 9: [2023-05-09 16:44:25,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 6: [2023-05-09 16:44:25,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +13: [2023-05-09 16:44:25,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 6: [2023-05-09 16:44:25,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +13: [2023-05-09 16:44:25,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +13: [2023-05-09 16:44:25,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +31: [2023-05-09 16:44:25,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +19: [2023-05-09 16:44:25,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +31: [2023-05-09 16:44:25,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +19: [2023-05-09 16:44:25,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +11: [2023-05-09 16:44:25,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +27: [2023-05-09 16:44:25,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +11: [2023-05-09 16:44:25,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +27: [2023-05-09 16:44:25,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +19: [2023-05-09 16:44:25,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +19: [2023-05-09 16:44:25,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +19: [2023-05-09 16:44:25,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +19: [2023-05-09 16:44:25,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +11: [2023-05-09 16:44:25,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +27: [2023-05-09 16:44:25,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +27: [2023-05-09 16:44:25,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +18: [2023-05-09 16:44:25,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +18: [2023-05-09 16:44:25,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +31: [2023-05-09 16:44:25,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +11: [2023-05-09 16:44:25,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +31: [2023-05-09 16:44:25,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +27: [2023-05-09 16:44:25,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +18: [2023-05-09 16:44:25,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +27: [2023-05-09 16:44:25,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +24: [2023-05-09 16:44:25,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +14: [2023-05-09 16:44:25,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +14: [2023-05-09 16:44:25,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +13: [2023-05-09 16:44:25,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +13: [2023-05-09 16:44:25,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +24: [2023-05-09 16:44:25,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +18: [2023-05-09 16:44:25,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +25: [2023-05-09 16:44:25,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +25: [2023-05-09 16:44:25,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +25: [2023-05-09 16:44:25,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +25: [2023-05-09 16:44:25,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +25: [2023-05-09 16:44:25,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +25: [2023-05-09 16:44:25,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 6: [2023-05-09 16:44:25,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +31: [2023-05-09 16:44:25,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 3: [2023-05-09 16:44:25,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 6: [2023-05-09 16:44:25,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +31: [2023-05-09 16:44:25,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +25: [2023-05-09 16:44:25,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +25: [2023-05-09 16:44:25,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 3: [2023-05-09 16:44:25,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +11: [2023-05-09 16:44:25,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 9: [2023-05-09 16:44:25,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 6: [2023-05-09 16:44:25,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 6: [2023-05-09 16:44:25,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +11: [2023-05-09 16:44:25,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 9: [2023-05-09 16:44:25,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 3: [2023-05-09 16:44:25,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 3: [2023-05-09 16:44:25,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 9: [2023-05-09 16:44:25,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 6: [2023-05-09 16:44:25,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 6: [2023-05-09 16:44:25,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 6: [2023-05-09 16:44:25,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 6: [2023-05-09 16:44:25,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 9: [2023-05-09 16:44:25,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +19: [2023-05-09 16:44:25,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 3: [2023-05-09 16:44:25,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +19: [2023-05-09 16:44:25,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +11: [2023-05-09 16:44:25,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 3: [2023-05-09 16:44:25,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +11: [2023-05-09 16:44:25,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +13: [2023-05-09 16:44:25,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +18: [2023-05-09 16:44:25,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +13: [2023-05-09 16:44:25,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +18: [2023-05-09 16:44:25,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +11: [2023-05-09 16:44:25,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +14: [2023-05-09 16:44:25,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +14: [2023-05-09 16:44:25,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +27: [2023-05-09 16:44:25,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 9: [2023-05-09 16:44:25,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 9: [2023-05-09 16:44:25,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +11: [2023-05-09 16:44:25,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +18: [2023-05-09 16:44:25,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +18: [2023-05-09 16:44:25,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +27: [2023-05-09 16:44:25,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +13: [2023-05-09 16:44:25,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +13: [2023-05-09 16:44:25,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +13: [2023-05-09 16:44:25,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +13: [2023-05-09 16:44:25,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +11: [2023-05-09 16:44:25,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +11: [2023-05-09 16:44:25,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +14: [2023-05-09 16:44:25,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +14: [2023-05-09 16:44:25,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 3: [2023-05-09 16:44:25,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +31: [2023-05-09 16:44:25,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +18: [2023-05-09 16:44:25,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 9: [2023-05-09 16:44:25,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +20: [2023-05-09 16:44:25,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +20: [2023-05-09 16:44:25,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +20: [2023-05-09 16:44:25,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +20: [2023-05-09 16:44:25,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 3: [2023-05-09 16:44:25,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +18: [2023-05-09 16:44:25,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +31: [2023-05-09 16:44:25,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 9: [2023-05-09 16:44:25,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +12: [2023-05-09 16:44:25,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +12: [2023-05-09 16:44:25,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +12: [2023-05-09 16:44:25,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +14: [2023-05-09 16:44:25,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +12: [2023-05-09 16:44:25,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +23: [2023-05-09 16:44:25,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +23: [2023-05-09 16:44:25,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +23: [2023-05-09 16:44:25,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 4: [2023-05-09 16:44:25,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 4: [2023-05-09 16:44:25,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. + 4: [2023-05-09 16:44:25,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +21: [2023-05-09 16:44:25,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +21: [2023-05-09 16:44:25,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +21: [2023-05-09 16:44:25,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +23: [2023-05-09 16:44:25,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +21: [2023-05-09 16:44:25,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +14: [2023-05-09 16:44:25,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 4: [2023-05-09 16:44:25,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt. +17: [2023-05-09 16:44:25,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +17: [2023-05-09 16:44:25,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +17: [2023-05-09 16:44:25,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 0: [2023-05-09 16:44:25,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +17: [2023-05-09 16:44:25,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 0: [2023-05-09 16:44:25,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 0: [2023-05-09 16:44:25,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 0: [2023-05-09 16:44:25,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +20: [2023-05-09 16:44:25,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +20: [2023-05-09 16:44:25,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +20: [2023-05-09 16:44:25,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +20: [2023-05-09 16:44:25,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +12: [2023-05-09 16:44:25,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +18: [2023-05-09 16:44:25,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +12: [2023-05-09 16:44:25,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +12: [2023-05-09 16:44:25,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +21: [2023-05-09 16:44:25,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 4: [2023-05-09 16:44:25,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +18: [2023-05-09 16:44:25,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 4: [2023-05-09 16:44:25,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +21: [2023-05-09 16:44:25,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +12: [2023-05-09 16:44:25,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +21: [2023-05-09 16:44:25,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 4: [2023-05-09 16:44:25,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 4: [2023-05-09 16:44:25,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +21: [2023-05-09 16:44:25,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +23: [2023-05-09 16:44:25,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +23: [2023-05-09 16:44:25,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +23: [2023-05-09 16:44:25,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... +23: [2023-05-09 16:44:25,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt... + 0: [2023-05-09 16:44:25,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +17: [2023-05-09 16:44:25,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 0: [2023-05-09 16:44:25,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 0: [2023-05-09 16:44:25,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 0: [2023-05-09 16:44:25,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +17: [2023-05-09 16:44:25,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +17: [2023-05-09 16:44:25,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +17: [2023-05-09 16:44:25,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +12: [2023-05-09 16:44:25,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +12: [2023-05-09 16:44:25,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +12: [2023-05-09 16:44:25,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +12: [2023-05-09 16:44:25,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +20: [2023-05-09 16:44:25,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +20: [2023-05-09 16:44:25,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +20: [2023-05-09 16:44:25,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +20: [2023-05-09 16:44:25,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +20: [2023-05-09 16:44:25,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +20: [2023-05-09 16:44:25,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +21: [2023-05-09 16:44:25,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +12: [2023-05-09 16:44:25,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +12: [2023-05-09 16:44:25,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +21: [2023-05-09 16:44:25,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +20: [2023-05-09 16:44:25,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +20: [2023-05-09 16:44:25,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +23: [2023-05-09 16:44:25,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +23: [2023-05-09 16:44:25,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +23: [2023-05-09 16:44:25,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +23: [2023-05-09 16:44:25,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +17: [2023-05-09 16:44:25,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +17: [2023-05-09 16:44:25,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +17: [2023-05-09 16:44:25,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +17: [2023-05-09 16:44:25,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 4: [2023-05-09 16:44:25,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 4: [2023-05-09 16:44:25,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 4: [2023-05-09 16:44:25,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 4: [2023-05-09 16:44:25,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +17: [2023-05-09 16:44:25,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +17: [2023-05-09 16:44:25,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +21: [2023-05-09 16:44:25,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +21: [2023-05-09 16:44:25,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 4: [2023-05-09 16:44:25,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 4: [2023-05-09 16:44:25,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +12: [2023-05-09 16:44:25,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +21: [2023-05-09 16:44:25,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +23: [2023-05-09 16:44:25,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 0: [2023-05-09 16:44:25,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 0: [2023-05-09 16:44:25,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +21: [2023-05-09 16:44:25,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +21: [2023-05-09 16:44:25,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +12: [2023-05-09 16:44:25,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 0: [2023-05-09 16:44:25,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 0: [2023-05-09 16:44:25,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +21: [2023-05-09 16:44:25,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +23: [2023-05-09 16:44:25,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +23: [2023-05-09 16:44:25,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. +23: [2023-05-09 16:44:25,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... + 4: [2023-05-09 16:44:25,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt. + 0: [2023-05-09 16:44:25,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 0: [2023-05-09 16:44:25,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 4: [2023-05-09 16:44:25,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt... +17: [2023-05-09 16:44:25,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +17: [2023-05-09 16:44:25,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 0: [2023-05-09 16:44:25,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 0: [2023-05-09 16:44:25,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +26: [2023-05-09 16:44:25,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +26: [2023-05-09 16:44:25,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +26: [2023-05-09 16:44:25,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +26: [2023-05-09 16:44:25,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +26: [2023-05-09 16:44:25,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +26: [2023-05-09 16:44:25,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +26: [2023-05-09 16:44:25,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +26: [2023-05-09 16:44:25,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +28: [2023-05-09 16:44:25,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +28: [2023-05-09 16:44:25,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +28: [2023-05-09 16:44:25,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +28: [2023-05-09 16:44:25,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +28: [2023-05-09 16:44:25,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +28: [2023-05-09 16:44:25,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +28: [2023-05-09 16:44:25,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +28: [2023-05-09 16:44:25,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +26: [2023-05-09 16:44:25,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +26: [2023-05-09 16:44:25,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +26: [2023-05-09 16:44:25,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +26: [2023-05-09 16:44:25,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 7: [2023-05-09 16:44:25,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 7: [2023-05-09 16:44:25,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 7: [2023-05-09 16:44:25,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 7: [2023-05-09 16:44:25,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +26: [2023-05-09 16:44:25,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +15: [2023-05-09 16:44:25,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +15: [2023-05-09 16:44:25,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +15: [2023-05-09 16:44:25,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +26: [2023-05-09 16:44:25,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +15: [2023-05-09 16:44:25,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +26: [2023-05-09 16:44:25,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +26: [2023-05-09 16:44:25,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 7: [2023-05-09 16:44:25,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 7: [2023-05-09 16:44:25,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +15: [2023-05-09 16:44:25,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +15: [2023-05-09 16:44:25,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +28: [2023-05-09 16:44:25,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +15: [2023-05-09 16:44:25,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 7: [2023-05-09 16:44:25,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 7: [2023-05-09 16:44:25,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +28: [2023-05-09 16:44:25,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +15: [2023-05-09 16:44:25,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +28: [2023-05-09 16:44:25,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +28: [2023-05-09 16:44:25,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +28: [2023-05-09 16:44:25,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +28: [2023-05-09 16:44:25,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +28: [2023-05-09 16:44:25,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +28: [2023-05-09 16:44:25,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 7: [2023-05-09 16:44:25,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 7: [2023-05-09 16:44:25,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 7: [2023-05-09 16:44:25,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 7: [2023-05-09 16:44:25,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +15: [2023-05-09 16:44:25,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +15: [2023-05-09 16:44:25,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +15: [2023-05-09 16:44:25,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +15: [2023-05-09 16:44:25,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +15: [2023-05-09 16:44:25,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +15: [2023-05-09 16:44:25,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 7: [2023-05-09 16:44:25,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 7: [2023-05-09 16:44:25,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +30: [2023-05-09 16:44:25,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +15: [2023-05-09 16:44:25,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +30: [2023-05-09 16:44:25,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +30: [2023-05-09 16:44:25,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +30: [2023-05-09 16:44:25,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 7: [2023-05-09 16:44:25,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 8: [2023-05-09 16:44:25,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 8: [2023-05-09 16:44:25,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 8: [2023-05-09 16:44:25,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +22: [2023-05-09 16:44:25,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +22: [2023-05-09 16:44:25,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +22: [2023-05-09 16:44:25,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +22: [2023-05-09 16:44:25,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 8: [2023-05-09 16:44:25,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +15: [2023-05-09 16:44:25,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 7: [2023-05-09 16:44:25,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 2: [2023-05-09 16:44:25,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 2: [2023-05-09 16:44:25,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +30: [2023-05-09 16:44:25,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +30: [2023-05-09 16:44:25,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +30: [2023-05-09 16:44:25,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +30: [2023-05-09 16:44:25,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 8: [2023-05-09 16:44:25,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 2: [2023-05-09 16:44:25,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 8: [2023-05-09 16:44:25,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +22: [2023-05-09 16:44:25,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +22: [2023-05-09 16:44:25,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +22: [2023-05-09 16:44:25,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 8: [2023-05-09 16:44:25,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 2: [2023-05-09 16:44:25,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 2: [2023-05-09 16:44:25,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 2: [2023-05-09 16:44:25,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +22: [2023-05-09 16:44:25,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 8: [2023-05-09 16:44:25,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 5: [2023-05-09 16:44:25,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 5: [2023-05-09 16:44:25,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 5: [2023-05-09 16:44:25,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 5: [2023-05-09 16:44:25,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +30: [2023-05-09 16:44:25,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +30: [2023-05-09 16:44:25,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +30: [2023-05-09 16:44:25,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +30: [2023-05-09 16:44:25,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 2: [2023-05-09 16:44:25,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +22: [2023-05-09 16:44:25,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 8: [2023-05-09 16:44:25,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +22: [2023-05-09 16:44:25,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +22: [2023-05-09 16:44:25,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +22: [2023-05-09 16:44:25,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 2: [2023-05-09 16:44:25,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +30: [2023-05-09 16:44:25,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +30: [2023-05-09 16:44:25,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +30: [2023-05-09 16:44:25,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +30: [2023-05-09 16:44:25,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 8: [2023-05-09 16:44:25,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +29: [2023-05-09 16:44:25,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +29: [2023-05-09 16:44:25,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +29: [2023-05-09 16:44:25,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +29: [2023-05-09 16:44:25,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 2: [2023-05-09 16:44:25,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 2: [2023-05-09 16:44:25,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +22: [2023-05-09 16:44:25,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +22: [2023-05-09 16:44:25,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 8: [2023-05-09 16:44:25,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 5: [2023-05-09 16:44:25,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 8: [2023-05-09 16:44:25,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 5: [2023-05-09 16:44:25,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 5: [2023-05-09 16:44:25,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 5: [2023-05-09 16:44:25,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 2: [2023-05-09 16:44:25,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 2: [2023-05-09 16:44:25,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 8: [2023-05-09 16:44:25,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +22: [2023-05-09 16:44:25,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 8: [2023-05-09 16:44:25,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +22: [2023-05-09 16:44:25,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +29: [2023-05-09 16:44:25,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +29: [2023-05-09 16:44:25,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +29: [2023-05-09 16:44:25,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +29: [2023-05-09 16:44:25,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 8: [2023-05-09 16:44:25,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 8: [2023-05-09 16:44:25,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 5: [2023-05-09 16:44:25,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 5: [2023-05-09 16:44:25,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 5: [2023-05-09 16:44:25,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 5: [2023-05-09 16:44:25,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 5: [2023-05-09 16:44:25,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 5: [2023-05-09 16:44:25,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +29: [2023-05-09 16:44:25,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +29: [2023-05-09 16:44:25,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +29: [2023-05-09 16:44:25,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +29: [2023-05-09 16:44:25,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +29: [2023-05-09 16:44:25,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +29: [2023-05-09 16:44:25,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +29: [2023-05-09 16:44:25,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +29: [2023-05-09 16:44:25,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 5: [2023-05-09 16:44:25,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 5: [2023-05-09 16:44:25,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +24: [2023-05-09 16:44:25,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +24: [2023-05-09 16:44:25,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +24: [2023-05-09 16:44:25,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +24: [2023-05-09 16:44:25,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +31: [2023-05-09 16:44:25,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +31: [2023-05-09 16:44:25,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +31: [2023-05-09 16:44:25,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +31: [2023-05-09 16:44:25,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +24: [2023-05-09 16:44:25,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +31: [2023-05-09 16:44:25,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +24: [2023-05-09 16:44:25,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +31: [2023-05-09 16:44:25,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +31: [2023-05-09 16:44:25,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +24: [2023-05-09 16:44:25,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +24: [2023-05-09 16:44:25,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +31: [2023-05-09 16:44:25,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +13: [2023-05-09 16:44:25,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +13: [2023-05-09 16:44:25,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +21: [2023-05-09 16:44:25,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +21: [2023-05-09 16:44:25,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +13: [2023-05-09 16:44:25,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +21: [2023-05-09 16:44:25,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +13: [2023-05-09 16:44:25,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +21: [2023-05-09 16:44:25,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +24: [2023-05-09 16:44:25,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +24: [2023-05-09 16:44:25,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +31: [2023-05-09 16:44:25,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +31: [2023-05-09 16:44:25,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +24: [2023-05-09 16:44:25,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +24: [2023-05-09 16:44:25,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +21: [2023-05-09 16:44:25,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +21: [2023-05-09 16:44:25,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +13: [2023-05-09 16:44:25,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +13: [2023-05-09 16:44:25,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +21: [2023-05-09 16:44:25,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +13: [2023-05-09 16:44:25,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +13: [2023-05-09 16:44:25,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +21: [2023-05-09 16:44:25,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +24: [2023-05-09 16:44:25,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +24: [2023-05-09 16:44:25,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +31: [2023-05-09 16:44:25,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +31: [2023-05-09 16:44:25,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +31: [2023-05-09 16:44:25,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +31: [2023-05-09 16:44:25,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +31: [2023-05-09 16:44:25,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +16: [2023-05-09 16:44:25,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +16: [2023-05-09 16:44:25,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +16: [2023-05-09 16:44:25,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +16: [2023-05-09 16:44:25,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +31: [2023-05-09 16:44:25,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +24: [2023-05-09 16:44:25,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +24: [2023-05-09 16:44:25,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +13: [2023-05-09 16:44:25,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +13: [2023-05-09 16:44:25,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +13: [2023-05-09 16:44:25,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +13: [2023-05-09 16:44:25,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +21: [2023-05-09 16:44:25,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +21: [2023-05-09 16:44:25,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +13: [2023-05-09 16:44:25,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +13: [2023-05-09 16:44:25,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +21: [2023-05-09 16:44:25,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +21: [2023-05-09 16:44:25,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +16: [2023-05-09 16:44:25,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +16: [2023-05-09 16:44:25,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +16: [2023-05-09 16:44:25,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +16: [2023-05-09 16:44:25,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +13: [2023-05-09 16:44:25,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +13: [2023-05-09 16:44:25,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +21: [2023-05-09 16:44:25,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +21: [2023-05-09 16:44:25,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +16: [2023-05-09 16:44:25,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +16: [2023-05-09 16:44:25,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +21: [2023-05-09 16:44:25,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +16: [2023-05-09 16:44:25,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +16: [2023-05-09 16:44:25,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +16: [2023-05-09 16:44:25,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +16: [2023-05-09 16:44:25,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +21: [2023-05-09 16:44:25,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +19: [2023-05-09 16:44:25,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +19: [2023-05-09 16:44:25,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +19: [2023-05-09 16:44:25,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +19: [2023-05-09 16:44:25,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 9: [2023-05-09 16:44:25,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 9: [2023-05-09 16:44:25,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 9: [2023-05-09 16:44:25,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 9: [2023-05-09 16:44:25,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 3: [2023-05-09 16:44:25,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 3: [2023-05-09 16:44:25,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 3: [2023-05-09 16:44:25,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 3: [2023-05-09 16:44:25,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +16: [2023-05-09 16:44:25,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +25: [2023-05-09 16:44:25,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +25: [2023-05-09 16:44:25,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +16: [2023-05-09 16:44:25,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +25: [2023-05-09 16:44:25,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +25: [2023-05-09 16:44:25,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +19: [2023-05-09 16:44:25,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +19: [2023-05-09 16:44:25,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +19: [2023-05-09 16:44:25,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +19: [2023-05-09 16:44:25,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 9: [2023-05-09 16:44:25,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 3: [2023-05-09 16:44:25,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 6: [2023-05-09 16:44:25,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +27: [2023-05-09 16:44:25,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +27: [2023-05-09 16:44:25,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +27: [2023-05-09 16:44:25,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 9: [2023-05-09 16:44:25,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 6: [2023-05-09 16:44:25,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 6: [2023-05-09 16:44:25,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +27: [2023-05-09 16:44:25,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 6: [2023-05-09 16:44:25,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 3: [2023-05-09 16:44:25,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 3: [2023-05-09 16:44:25,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +11: [2023-05-09 16:44:25,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +11: [2023-05-09 16:44:25,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +11: [2023-05-09 16:44:25,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +11: [2023-05-09 16:44:25,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +18: [2023-05-09 16:44:25,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +18: [2023-05-09 16:44:25,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +18: [2023-05-09 16:44:25,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +18: [2023-05-09 16:44:25,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +23: [2023-05-09 16:44:25,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +23: [2023-05-09 16:44:25,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 3: [2023-05-09 16:44:25,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 9: [2023-05-09 16:44:25,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 9: [2023-05-09 16:44:25,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +23: [2023-05-09 16:44:25,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +23: [2023-05-09 16:44:25,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +25: [2023-05-09 16:44:25,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +25: [2023-05-09 16:44:25,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +25: [2023-05-09 16:44:25,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +25: [2023-05-09 16:44:25,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 4: [2023-05-09 16:44:25,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. +27: [2023-05-09 16:44:25,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +27: [2023-05-09 16:44:25,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +27: [2023-05-09 16:44:25,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 4: [2023-05-09 16:44:25,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 4: [2023-05-09 16:44:25,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 6: [2023-05-09 16:44:25,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +11: [2023-05-09 16:44:25,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 4: [2023-05-09 16:44:25,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt. + 6: [2023-05-09 16:44:25,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +27: [2023-05-09 16:44:25,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +11: [2023-05-09 16:44:25,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +11: [2023-05-09 16:44:25,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +19: [2023-05-09 16:44:25,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +18: [2023-05-09 16:44:25,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +19: [2023-05-09 16:44:25,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +18: [2023-05-09 16:44:25,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +23: [2023-05-09 16:44:25,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +19: [2023-05-09 16:44:25,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 9: [2023-05-09 16:44:25,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +18: [2023-05-09 16:44:25,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +19: [2023-05-09 16:44:25,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +19: [2023-05-09 16:44:25,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +19: [2023-05-09 16:44:25,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 6: [2023-05-09 16:44:25,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 9: [2023-05-09 16:44:25,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 6: [2023-05-09 16:44:25,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +23: [2023-05-09 16:44:25,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +23: [2023-05-09 16:44:25,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 9: [2023-05-09 16:44:25,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 9: [2023-05-09 16:44:25,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +18: [2023-05-09 16:44:25,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 3: [2023-05-09 16:44:25,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +11: [2023-05-09 16:44:25,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 3: [2023-05-09 16:44:25,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +23: [2023-05-09 16:44:25,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 3: [2023-05-09 16:44:25,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 3: [2023-05-09 16:44:25,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 3: [2023-05-09 16:44:25,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 3: [2023-05-09 16:44:25,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 4: [2023-05-09 16:44:25,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 4: [2023-05-09 16:44:25,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... + 9: [2023-05-09 16:44:25,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 9: [2023-05-09 16:44:25,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 4: [2023-05-09 16:44:25,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +25: [2023-05-09 16:44:25,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +27: [2023-05-09 16:44:25,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +25: [2023-05-09 16:44:25,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 3: [2023-05-09 16:44:25,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 3: [2023-05-09 16:44:25,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +27: [2023-05-09 16:44:25,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +11: [2023-05-09 16:44:25,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +19: [2023-05-09 16:44:25,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +11: [2023-05-09 16:44:25,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 4: [2023-05-09 16:44:25,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt... +11: [2023-05-09 16:44:25,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +11: [2023-05-09 16:44:25,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +25: [2023-05-09 16:44:25,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +25: [2023-05-09 16:44:25,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +25: [2023-05-09 16:44:25,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +25: [2023-05-09 16:44:25,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +19: [2023-05-09 16:44:25,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +27: [2023-05-09 16:44:25,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +27: [2023-05-09 16:44:25,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 6: [2023-05-09 16:44:25,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +27: [2023-05-09 16:44:25,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +27: [2023-05-09 16:44:25,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 6: [2023-05-09 16:44:25,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +25: [2023-05-09 16:44:25,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +23: [2023-05-09 16:44:25,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +25: [2023-05-09 16:44:25,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +18: [2023-05-09 16:44:25,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +23: [2023-05-09 16:44:25,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +18: [2023-05-09 16:44:25,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +18: [2023-05-09 16:44:25,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +18: [2023-05-09 16:44:25,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 9: [2023-05-09 16:44:25,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 6: [2023-05-09 16:44:25,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 6: [2023-05-09 16:44:25,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +18: [2023-05-09 16:44:25,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +18: [2023-05-09 16:44:25,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 9: [2023-05-09 16:44:25,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +23: [2023-05-09 16:44:25,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +23: [2023-05-09 16:44:25,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +27: [2023-05-09 16:44:25,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 6: [2023-05-09 16:44:25,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +27: [2023-05-09 16:44:25,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 6: [2023-05-09 16:44:25,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +14: [2023-05-09 16:44:25,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +26: [2023-05-09 16:44:25,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 6: [2023-05-09 16:44:25,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +26: [2023-05-09 16:44:25,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +26: [2023-05-09 16:44:25,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +18: [2023-05-09 16:44:25,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +18: [2023-05-09 16:44:25,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +26: [2023-05-09 16:44:25,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +23: [2023-05-09 16:44:25,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +14: [2023-05-09 16:44:25,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +11: [2023-05-09 16:44:25,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 6: [2023-05-09 16:44:25,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +23: [2023-05-09 16:44:25,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +11: [2023-05-09 16:44:25,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +14: [2023-05-09 16:44:25,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 4: [2023-05-09 16:44:25,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 4: [2023-05-09 16:44:25,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +14: [2023-05-09 16:44:25,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 4: [2023-05-09 16:44:25,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... + 4: [2023-05-09 16:44:25,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +23: [2023-05-09 16:44:25,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. +20: [2023-05-09 16:44:25,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +20: [2023-05-09 16:44:25,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +20: [2023-05-09 16:44:25,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +20: [2023-05-09 16:44:25,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +23: [2023-05-09 16:44:25,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +11: [2023-05-09 16:44:25,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 4: [2023-05-09 16:44:25,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 4: [2023-05-09 16:44:25,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +10: [2023-05-09 16:44:25,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +10: [2023-05-09 16:44:25,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +10: [2023-05-09 16:44:25,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +26: [2023-05-09 16:44:25,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +26: [2023-05-09 16:44:25,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +26: [2023-05-09 16:44:25,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +26: [2023-05-09 16:44:25,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +10: [2023-05-09 16:44:25,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +11: [2023-05-09 16:44:25,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +14: [2023-05-09 16:44:25,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +14: [2023-05-09 16:44:25,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +14: [2023-05-09 16:44:25,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +14: [2023-05-09 16:44:25,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +20: [2023-05-09 16:44:25,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +20: [2023-05-09 16:44:25,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +20: [2023-05-09 16:44:25,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +20: [2023-05-09 16:44:25,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 4: [2023-05-09 16:44:25,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt. + 4: [2023-05-09 16:44:25,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt... +26: [2023-05-09 16:44:25,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +10: [2023-05-09 16:44:25,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +26: [2023-05-09 16:44:25,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +10: [2023-05-09 16:44:25,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +10: [2023-05-09 16:44:25,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +10: [2023-05-09 16:44:25,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +14: [2023-05-09 16:44:25,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +14: [2023-05-09 16:44:25,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +26: [2023-05-09 16:44:25,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +26: [2023-05-09 16:44:25,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +26: [2023-05-09 16:44:25,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +26: [2023-05-09 16:44:25,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +20: [2023-05-09 16:44:25,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +20: [2023-05-09 16:44:25,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +20: [2023-05-09 16:44:25,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +20: [2023-05-09 16:44:25,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +14: [2023-05-09 16:44:25,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +14: [2023-05-09 16:44:25,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +14: [2023-05-09 16:44:25,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +14: [2023-05-09 16:44:25,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +26: [2023-05-09 16:44:25,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 1: [2023-05-09 16:44:25,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 1: [2023-05-09 16:44:25,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 1: [2023-05-09 16:44:25,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 1: [2023-05-09 16:44:25,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +26: [2023-05-09 16:44:25,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +10: [2023-05-09 16:44:25,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +10: [2023-05-09 16:44:25,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +14: [2023-05-09 16:44:25,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +20: [2023-05-09 16:44:25,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +20: [2023-05-09 16:44:25,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +14: [2023-05-09 16:44:25,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +20: [2023-05-09 16:44:25,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +20: [2023-05-09 16:44:25,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +10: [2023-05-09 16:44:25,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +10: [2023-05-09 16:44:25,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +10: [2023-05-09 16:44:25,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +10: [2023-05-09 16:44:25,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 1: [2023-05-09 16:44:25,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +10: [2023-05-09 16:44:25,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 1: [2023-05-09 16:44:25,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 1: [2023-05-09 16:44:25,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 1: [2023-05-09 16:44:25,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +10: [2023-05-09 16:44:25,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 1: [2023-05-09 16:44:25,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 1: [2023-05-09 16:44:25,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 1: [2023-05-09 16:44:25,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 1: [2023-05-09 16:44:25,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 1: [2023-05-09 16:44:25,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 1: [2023-05-09 16:44:25,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 1: [2023-05-09 16:44:25,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 1: [2023-05-09 16:44:25,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +15: [2023-05-09 16:44:25,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +15: [2023-05-09 16:44:25,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +15: [2023-05-09 16:44:25,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +15: [2023-05-09 16:44:25,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +15: [2023-05-09 16:44:25,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +15: [2023-05-09 16:44:25,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +15: [2023-05-09 16:44:25,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +15: [2023-05-09 16:44:25,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +15: [2023-05-09 16:44:25,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +15: [2023-05-09 16:44:25,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +15: [2023-05-09 16:44:25,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +15: [2023-05-09 16:44:25,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +15: [2023-05-09 16:44:25,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +15: [2023-05-09 16:44:25,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +12: [2023-05-09 16:44:25,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +12: [2023-05-09 16:44:25,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +12: [2023-05-09 16:44:25,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +12: [2023-05-09 16:44:25,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +15: [2023-05-09 16:44:25,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +15: [2023-05-09 16:44:25,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +12: [2023-05-09 16:44:25,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +12: [2023-05-09 16:44:25,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +12: [2023-05-09 16:44:25,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +12: [2023-05-09 16:44:25,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +12: [2023-05-09 16:44:25,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +12: [2023-05-09 16:44:25,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +12: [2023-05-09 16:44:25,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +12: [2023-05-09 16:44:25,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +22: [2023-05-09 16:44:25,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +22: [2023-05-09 16:44:25,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +22: [2023-05-09 16:44:25,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +22: [2023-05-09 16:44:25,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +12: [2023-05-09 16:44:25,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +12: [2023-05-09 16:44:25,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +12: [2023-05-09 16:44:25,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +12: [2023-05-09 16:44:25,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +22: [2023-05-09 16:44:25,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +22: [2023-05-09 16:44:25,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +22: [2023-05-09 16:44:25,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +22: [2023-05-09 16:44:25,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 7: [2023-05-09 16:44:25,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 7: [2023-05-09 16:44:25,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 8: [2023-05-09 16:44:25,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 7: [2023-05-09 16:44:25,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 8: [2023-05-09 16:44:25,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 7: [2023-05-09 16:44:25,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 8: [2023-05-09 16:44:25,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 0: [2023-05-09 16:44:25,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 8: [2023-05-09 16:44:25,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 0: [2023-05-09 16:44:25,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 0: [2023-05-09 16:44:25,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +28: [2023-05-09 16:44:25,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +28: [2023-05-09 16:44:25,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +28: [2023-05-09 16:44:25,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 0: [2023-05-09 16:44:25,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +28: [2023-05-09 16:44:25,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +17: [2023-05-09 16:44:25,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +17: [2023-05-09 16:44:25,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +17: [2023-05-09 16:44:25,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +17: [2023-05-09 16:44:25,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 8: [2023-05-09 16:44:25,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +22: [2023-05-09 16:44:25,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +22: [2023-05-09 16:44:25,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +22: [2023-05-09 16:44:25,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +22: [2023-05-09 16:44:25,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 7: [2023-05-09 16:44:25,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 7: [2023-05-09 16:44:25,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 8: [2023-05-09 16:44:25,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +28: [2023-05-09 16:44:25,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 0: [2023-05-09 16:44:25,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 8: [2023-05-09 16:44:25,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +28: [2023-05-09 16:44:25,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 0: [2023-05-09 16:44:25,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 8: [2023-05-09 16:44:25,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +28: [2023-05-09 16:44:25,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 7: [2023-05-09 16:44:25,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 7: [2023-05-09 16:44:25,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 0: [2023-05-09 16:44:25,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 0: [2023-05-09 16:44:25,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +28: [2023-05-09 16:44:25,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +29: [2023-05-09 16:44:25,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +29: [2023-05-09 16:44:25,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +29: [2023-05-09 16:44:25,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +29: [2023-05-09 16:44:25,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +22: [2023-05-09 16:44:25,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +22: [2023-05-09 16:44:25,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +17: [2023-05-09 16:44:25,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 2: [2023-05-09 16:44:25,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +17: [2023-05-09 16:44:25,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +24: [2023-05-09 16:44:25,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +24: [2023-05-09 16:44:25,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +24: [2023-05-09 16:44:25,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 2: [2023-05-09 16:44:25,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 2: [2023-05-09 16:44:25,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +24: [2023-05-09 16:44:25,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +17: [2023-05-09 16:44:25,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +17: [2023-05-09 16:44:25,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 8: [2023-05-09 16:44:25,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 8: [2023-05-09 16:44:25,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +30: [2023-05-09 16:44:25,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +30: [2023-05-09 16:44:25,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +30: [2023-05-09 16:44:25,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +21: [2023-05-09 16:44:25,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +21: [2023-05-09 16:44:25,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +30: [2023-05-09 16:44:25,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +21: [2023-05-09 16:44:25,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +21: [2023-05-09 16:44:25,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +28: [2023-05-09 16:44:25,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +29: [2023-05-09 16:44:25,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +28: [2023-05-09 16:44:25,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +29: [2023-05-09 16:44:25,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +22: [2023-05-09 16:44:25,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +29: [2023-05-09 16:44:25,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 2: [2023-05-09 16:44:25,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 7: [2023-05-09 16:44:25,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +29: [2023-05-09 16:44:25,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +22: [2023-05-09 16:44:25,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 7: [2023-05-09 16:44:25,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 2: [2023-05-09 16:44:25,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 7: [2023-05-09 16:44:25,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +17: [2023-05-09 16:44:25,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 7: [2023-05-09 16:44:25,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 7: [2023-05-09 16:44:25,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 7: [2023-05-09 16:44:25,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +17: [2023-05-09 16:44:25,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 8: [2023-05-09 16:44:25,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 8: [2023-05-09 16:44:25,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +24: [2023-05-09 16:44:25,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +30: [2023-05-09 16:44:25,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +24: [2023-05-09 16:44:25,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +30: [2023-05-09 16:44:25,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +30: [2023-05-09 16:44:25,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +24: [2023-05-09 16:44:25,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +24: [2023-05-09 16:44:25,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +21: [2023-05-09 16:44:25,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +21: [2023-05-09 16:44:25,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +21: [2023-05-09 16:44:25,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +28: [2023-05-09 16:44:25,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +28: [2023-05-09 16:44:25,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +28: [2023-05-09 16:44:25,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +30: [2023-05-09 16:44:25,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +21: [2023-05-09 16:44:25,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +28: [2023-05-09 16:44:25,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 0: [2023-05-09 16:44:25,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 0: [2023-05-09 16:44:25,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +17: [2023-05-09 16:44:25,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 8: [2023-05-09 16:44:25,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +17: [2023-05-09 16:44:25,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 8: [2023-05-09 16:44:25,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 8: [2023-05-09 16:44:25,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 8: [2023-05-09 16:44:25,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 7: [2023-05-09 16:44:25,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +29: [2023-05-09 16:44:25,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +29: [2023-05-09 16:44:25,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 7: [2023-05-09 16:44:25,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +29: [2023-05-09 16:44:25,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +29: [2023-05-09 16:44:25,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 0: [2023-05-09 16:44:25,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 0: [2023-05-09 16:44:25,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 0: [2023-05-09 16:44:25,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 0: [2023-05-09 16:44:25,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 2: [2023-05-09 16:44:25,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 2: [2023-05-09 16:44:25,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +24: [2023-05-09 16:44:25,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +30: [2023-05-09 16:44:25,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +24: [2023-05-09 16:44:25,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +30: [2023-05-09 16:44:25,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +24: [2023-05-09 16:44:25,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +24: [2023-05-09 16:44:25,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +28: [2023-05-09 16:44:25,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +24: [2023-05-09 16:44:25,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +24: [2023-05-09 16:44:25,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +28: [2023-05-09 16:44:25,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +30: [2023-05-09 16:44:25,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +30: [2023-05-09 16:44:25,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 0: [2023-05-09 16:44:25,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +30: [2023-05-09 16:44:25,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +30: [2023-05-09 16:44:25,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +21: [2023-05-09 16:44:25,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +21: [2023-05-09 16:44:25,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +17: [2023-05-09 16:44:25,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +17: [2023-05-09 16:44:25,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +21: [2023-05-09 16:44:25,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +21: [2023-05-09 16:44:25,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 0: [2023-05-09 16:44:25,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +24: [2023-05-09 16:44:25,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +24: [2023-05-09 16:44:25,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +30: [2023-05-09 16:44:25,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +30: [2023-05-09 16:44:25,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +17: [2023-05-09 16:44:25,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +17: [2023-05-09 16:44:25,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +29: [2023-05-09 16:44:25,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +29: [2023-05-09 16:44:25,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 2: [2023-05-09 16:44:25,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +29: [2023-05-09 16:44:26,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 2: [2023-05-09 16:44:26,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +21: [2023-05-09 16:44:26,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +29: [2023-05-09 16:44:26,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +21: [2023-05-09 16:44:26,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 2: [2023-05-09 16:44:26,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 2: [2023-05-09 16:44:26,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +31: [2023-05-09 16:44:26,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +31: [2023-05-09 16:44:26,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +31: [2023-05-09 16:44:26,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +31: [2023-05-09 16:44:26,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +21: [2023-05-09 16:44:26,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +21: [2023-05-09 16:44:26,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +31: [2023-05-09 16:44:26,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +31: [2023-05-09 16:44:26,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +31: [2023-05-09 16:44:26,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +31: [2023-05-09 16:44:26,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 9: [2023-05-09 16:44:26,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 9: [2023-05-09 16:44:26,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 9: [2023-05-09 16:44:26,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 9: [2023-05-09 16:44:26,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +13: [2023-05-09 16:44:26,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +13: [2023-05-09 16:44:26,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +13: [2023-05-09 16:44:26,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +13: [2023-05-09 16:44:26,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +31: [2023-05-09 16:44:26,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +31: [2023-05-09 16:44:26,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 9: [2023-05-09 16:44:26,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 9: [2023-05-09 16:44:26,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 9: [2023-05-09 16:44:26,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 9: [2023-05-09 16:44:26,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 5: [2023-05-09 16:44:26,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 5: [2023-05-09 16:44:26,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 5: [2023-05-09 16:44:26,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +11: [2023-05-09 16:44:26,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 5: [2023-05-09 16:44:26,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +11: [2023-05-09 16:44:26,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +13: [2023-05-09 16:44:26,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +13: [2023-05-09 16:44:26,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +13: [2023-05-09 16:44:26,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +11: [2023-05-09 16:44:26,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +19: [2023-05-09 16:44:26,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +19: [2023-05-09 16:44:26,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +19: [2023-05-09 16:44:26,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +13: [2023-05-09 16:44:26,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +19: [2023-05-09 16:44:26,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +11: [2023-05-09 16:44:26,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +31: [2023-05-09 16:44:26,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +31: [2023-05-09 16:44:26,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +23: [2023-05-09 16:44:26,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +23: [2023-05-09 16:44:26,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +23: [2023-05-09 16:44:26,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +23: [2023-05-09 16:44:26,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +25: [2023-05-09 16:44:26,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 4: [2023-05-09 16:44:26,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 4: [2023-05-09 16:44:26,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +25: [2023-05-09 16:44:26,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +25: [2023-05-09 16:44:26,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 4: [2023-05-09 16:44:26,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +25: [2023-05-09 16:44:26,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 4: [2023-05-09 16:44:26,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +11: [2023-05-09 16:44:26,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +11: [2023-05-09 16:44:26,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 5: [2023-05-09 16:44:26,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +31: [2023-05-09 16:44:26,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +31: [2023-05-09 16:44:26,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +27: [2023-05-09 16:44:26,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 5: [2023-05-09 16:44:26,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 5: [2023-05-09 16:44:26,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +27: [2023-05-09 16:44:26,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +27: [2023-05-09 16:44:26,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 5: [2023-05-09 16:44:26,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +27: [2023-05-09 16:44:26,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +11: [2023-05-09 16:44:26,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +19: [2023-05-09 16:44:26,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +19: [2023-05-09 16:44:26,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +19: [2023-05-09 16:44:26,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 9: [2023-05-09 16:44:26,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +19: [2023-05-09 16:44:26,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 9: [2023-05-09 16:44:26,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +25: [2023-05-09 16:44:26,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +11: [2023-05-09 16:44:26,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +13: [2023-05-09 16:44:26,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 9: [2023-05-09 16:44:26,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 9: [2023-05-09 16:44:26,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +23: [2023-05-09 16:44:26,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +23: [2023-05-09 16:44:26,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +23: [2023-05-09 16:44:26,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +13: [2023-05-09 16:44:26,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +13: [2023-05-09 16:44:26,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +13: [2023-05-09 16:44:26,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +31: [2023-05-09 16:44:26,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 9: [2023-05-09 16:44:26,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 4: [2023-05-09 16:44:26,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 9: [2023-05-09 16:44:26,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 4: [2023-05-09 16:44:26,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 4: [2023-05-09 16:44:26,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +25: [2023-05-09 16:44:26,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +23: [2023-05-09 16:44:26,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +31: [2023-05-09 16:44:26,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +25: [2023-05-09 16:44:26,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +25: [2023-05-09 16:44:26,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +18: [2023-05-09 16:44:26,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +18: [2023-05-09 16:44:26,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 4: [2023-05-09 16:44:26,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +27: [2023-05-09 16:44:26,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +27: [2023-05-09 16:44:26,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 6: [2023-05-09 16:44:26,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 9: [2023-05-09 16:44:26,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +11: [2023-05-09 16:44:26,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +13: [2023-05-09 16:44:26,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +27: [2023-05-09 16:44:26,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +11: [2023-05-09 16:44:26,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +27: [2023-05-09 16:44:26,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +18: [2023-05-09 16:44:26,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 6: [2023-05-09 16:44:26,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 9: [2023-05-09 16:44:26,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +13: [2023-05-09 16:44:26,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +13: [2023-05-09 16:44:26,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +18: [2023-05-09 16:44:26,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +13: [2023-05-09 16:44:26,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 6: [2023-05-09 16:44:26,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 6: [2023-05-09 16:44:26,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +11: [2023-05-09 16:44:26,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +11: [2023-05-09 16:44:26,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +11: [2023-05-09 16:44:26,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +11: [2023-05-09 16:44:26,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 5: [2023-05-09 16:44:26,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 5: [2023-05-09 16:44:26,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +25: [2023-05-09 16:44:26,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +23: [2023-05-09 16:44:26,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +25: [2023-05-09 16:44:26,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 5: [2023-05-09 16:44:26,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +19: [2023-05-09 16:44:26,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 5: [2023-05-09 16:44:26,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +23: [2023-05-09 16:44:26,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +19: [2023-05-09 16:44:26,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +18: [2023-05-09 16:44:26,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +19: [2023-05-09 16:44:26,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +19: [2023-05-09 16:44:26,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +19: [2023-05-09 16:44:26,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +19: [2023-05-09 16:44:26,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +23: [2023-05-09 16:44:26,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +23: [2023-05-09 16:44:26,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +23: [2023-05-09 16:44:26,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +23: [2023-05-09 16:44:26,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 6: [2023-05-09 16:44:26,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +18: [2023-05-09 16:44:26,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +18: [2023-05-09 16:44:26,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 6: [2023-05-09 16:44:26,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 6: [2023-05-09 16:44:26,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +18: [2023-05-09 16:44:26,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 6: [2023-05-09 16:44:26,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 5: [2023-05-09 16:44:26,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +11: [2023-05-09 16:44:26,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +11: [2023-05-09 16:44:26,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 5: [2023-05-09 16:44:26,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +27: [2023-05-09 16:44:26,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 5: [2023-05-09 16:44:26,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +27: [2023-05-09 16:44:26,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 5: [2023-05-09 16:44:26,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +19: [2023-05-09 16:44:26,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 4: [2023-05-09 16:44:26,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +27: [2023-05-09 16:44:26,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 4: [2023-05-09 16:44:26,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 4: [2023-05-09 16:44:26,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 4: [2023-05-09 16:44:26,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 4: [2023-05-09 16:44:26,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +25: [2023-05-09 16:44:26,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 4: [2023-05-09 16:44:26,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +25: [2023-05-09 16:44:26,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +19: [2023-05-09 16:44:26,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +25: [2023-05-09 16:44:26,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +25: [2023-05-09 16:44:26,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +27: [2023-05-09 16:44:26,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +23: [2023-05-09 16:44:26,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +23: [2023-05-09 16:44:26,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +18: [2023-05-09 16:44:26,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +18: [2023-05-09 16:44:26,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +14: [2023-05-09 16:44:26,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 4: [2023-05-09 16:44:26,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 4: [2023-05-09 16:44:26,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +27: [2023-05-09 16:44:26,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +14: [2023-05-09 16:44:26,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +14: [2023-05-09 16:44:26,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +18: [2023-05-09 16:44:26,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +25: [2023-05-09 16:44:26,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +18: [2023-05-09 16:44:26,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +14: [2023-05-09 16:44:26,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +27: [2023-05-09 16:44:26,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +25: [2023-05-09 16:44:26,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 6: [2023-05-09 16:44:26,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 6: [2023-05-09 16:44:26,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +27: [2023-05-09 16:44:26,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 6: [2023-05-09 16:44:26,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 6: [2023-05-09 16:44:26,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 6: [2023-05-09 16:44:26,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 6: [2023-05-09 16:44:26,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +27: [2023-05-09 16:44:26,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 6: [2023-05-09 16:44:26,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 6: [2023-05-09 16:44:26,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +18: [2023-05-09 16:44:26,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +18: [2023-05-09 16:44:26,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... + 3: [2023-05-09 16:44:26,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 3: [2023-05-09 16:44:26,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 3: [2023-05-09 16:44:26,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. + 3: [2023-05-09 16:44:26,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt. +20: [2023-05-09 16:44:26,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +20: [2023-05-09 16:44:26,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +20: [2023-05-09 16:44:26,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +20: [2023-05-09 16:44:26,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 1: [2023-05-09 16:44:26,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +14: [2023-05-09 16:44:26,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 1: [2023-05-09 16:44:26,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 1: [2023-05-09 16:44:26,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +14: [2023-05-09 16:44:26,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 1: [2023-05-09 16:44:26,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +18: [2023-05-09 16:44:26,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +26: [2023-05-09 16:44:26,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +26: [2023-05-09 16:44:26,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +10: [2023-05-09 16:44:26,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +26: [2023-05-09 16:44:26,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +10: [2023-05-09 16:44:26,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +26: [2023-05-09 16:44:26,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +18: [2023-05-09 16:44:26,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +10: [2023-05-09 16:44:26,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +10: [2023-05-09 16:44:26,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +14: [2023-05-09 16:44:26,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +14: [2023-05-09 16:44:26,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +15: [2023-05-09 16:44:26,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +15: [2023-05-09 16:44:26,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +15: [2023-05-09 16:44:26,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +15: [2023-05-09 16:44:26,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +16: [2023-05-09 16:44:26,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +16: [2023-05-09 16:44:26,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 3: [2023-05-09 16:44:26,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +16: [2023-05-09 16:44:26,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +16: [2023-05-09 16:44:26,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 3: [2023-05-09 16:44:26,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... + 3: [2023-05-09 16:44:26,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +20: [2023-05-09 16:44:26,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +20: [2023-05-09 16:44:26,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 3: [2023-05-09 16:44:26,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt... +20: [2023-05-09 16:44:26,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +20: [2023-05-09 16:44:26,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 1: [2023-05-09 16:44:26,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +26: [2023-05-09 16:44:26,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +26: [2023-05-09 16:44:26,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +26: [2023-05-09 16:44:26,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +14: [2023-05-09 16:44:26,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +26: [2023-05-09 16:44:26,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 1: [2023-05-09 16:44:26,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 1: [2023-05-09 16:44:26,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +14: [2023-05-09 16:44:26,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 1: [2023-05-09 16:44:26,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +10: [2023-05-09 16:44:26,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +10: [2023-05-09 16:44:26,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +10: [2023-05-09 16:44:26,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +15: [2023-05-09 16:44:26,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +15: [2023-05-09 16:44:26,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +15: [2023-05-09 16:44:26,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +16: [2023-05-09 16:44:26,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +15: [2023-05-09 16:44:26,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +14: [2023-05-09 16:44:26,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +14: [2023-05-09 16:44:26,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +16: [2023-05-09 16:44:26,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +16: [2023-05-09 16:44:26,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +16: [2023-05-09 16:44:26,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +10: [2023-05-09 16:44:26,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 3: [2023-05-09 16:44:26,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 3: [2023-05-09 16:44:26,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +26: [2023-05-09 16:44:26,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +26: [2023-05-09 16:44:26,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +20: [2023-05-09 16:44:26,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +20: [2023-05-09 16:44:26,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +20: [2023-05-09 16:44:26,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 8: [2023-05-09 16:44:26,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 8: [2023-05-09 16:44:26,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 8: [2023-05-09 16:44:26,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +20: [2023-05-09 16:44:26,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +20: [2023-05-09 16:44:26,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +20: [2023-05-09 16:44:26,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 8: [2023-05-09 16:44:26,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +26: [2023-05-09 16:44:26,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +26: [2023-05-09 16:44:26,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +10: [2023-05-09 16:44:26,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +10: [2023-05-09 16:44:26,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +26: [2023-05-09 16:44:26,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +26: [2023-05-09 16:44:26,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 1: [2023-05-09 16:44:26,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 3: [2023-05-09 16:44:26,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. +15: [2023-05-09 16:44:26,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 3: [2023-05-09 16:44:26,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +16: [2023-05-09 16:44:26,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +14: [2023-05-09 16:44:26,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +16: [2023-05-09 16:44:26,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +15: [2023-05-09 16:44:26,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 1: [2023-05-09 16:44:26,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +26: [2023-05-09 16:44:26,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +14: [2023-05-09 16:44:26,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +10: [2023-05-09 16:44:26,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +10: [2023-05-09 16:44:26,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +26: [2023-05-09 16:44:26,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +16: [2023-05-09 16:44:26,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +16: [2023-05-09 16:44:26,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +10: [2023-05-09 16:44:26,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +10: [2023-05-09 16:44:26,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +16: [2023-05-09 16:44:26,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +20: [2023-05-09 16:44:26,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +16: [2023-05-09 16:44:26,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 3: [2023-05-09 16:44:26,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 3: [2023-05-09 16:44:26,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt. + 1: [2023-05-09 16:44:26,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +14: [2023-05-09 16:44:26,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 8: [2023-05-09 16:44:26,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +20: [2023-05-09 16:44:26,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 8: [2023-05-09 16:44:26,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +15: [2023-05-09 16:44:26,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +15: [2023-05-09 16:44:26,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 3: [2023-05-09 16:44:26,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +15: [2023-05-09 16:44:26,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +15: [2023-05-09 16:44:26,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 1: [2023-05-09 16:44:26,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +14: [2023-05-09 16:44:26,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 3: [2023-05-09 16:44:26,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt... +17: [2023-05-09 16:44:26,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +17: [2023-05-09 16:44:26,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 8: [2023-05-09 16:44:26,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 8: [2023-05-09 16:44:26,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 1: [2023-05-09 16:44:26,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 1: [2023-05-09 16:44:26,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +17: [2023-05-09 16:44:26,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +17: [2023-05-09 16:44:26,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 1: [2023-05-09 16:44:26,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 1: [2023-05-09 16:44:26,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +15: [2023-05-09 16:44:26,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +16: [2023-05-09 16:44:26,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +15: [2023-05-09 16:44:26,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +16: [2023-05-09 16:44:26,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +10: [2023-05-09 16:44:26,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +10: [2023-05-09 16:44:26,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +17: [2023-05-09 16:44:26,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +12: [2023-05-09 16:44:26,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +12: [2023-05-09 16:44:26,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +12: [2023-05-09 16:44:26,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +12: [2023-05-09 16:44:26,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +17: [2023-05-09 16:44:26,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 8: [2023-05-09 16:44:26,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 8: [2023-05-09 16:44:26,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +17: [2023-05-09 16:44:26,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +17: [2023-05-09 16:44:26,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +12: [2023-05-09 16:44:26,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +12: [2023-05-09 16:44:26,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +12: [2023-05-09 16:44:26,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +12: [2023-05-09 16:44:26,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +17: [2023-05-09 16:44:26,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 8: [2023-05-09 16:44:26,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +17: [2023-05-09 16:44:26,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 8: [2023-05-09 16:44:26,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 8: [2023-05-09 16:44:26,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 8: [2023-05-09 16:44:26,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +17: [2023-05-09 16:44:26,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +17: [2023-05-09 16:44:26,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +17: [2023-05-09 16:44:26,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +17: [2023-05-09 16:44:26,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 8: [2023-05-09 16:44:26,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +12: [2023-05-09 16:44:26,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +12: [2023-05-09 16:44:26,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 8: [2023-05-09 16:44:26,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +12: [2023-05-09 16:44:26,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +12: [2023-05-09 16:44:26,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +12: [2023-05-09 16:44:26,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +12: [2023-05-09 16:44:26,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +17: [2023-05-09 16:44:26,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +17: [2023-05-09 16:44:26,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +12: [2023-05-09 16:44:26,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +12: [2023-05-09 16:44:26,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +22: [2023-05-09 16:44:26,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +22: [2023-05-09 16:44:26,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +22: [2023-05-09 16:44:26,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +22: [2023-05-09 16:44:26,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +22: [2023-05-09 16:44:26,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +22: [2023-05-09 16:44:26,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +22: [2023-05-09 16:44:26,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +22: [2023-05-09 16:44:26,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 0: [2023-05-09 16:44:26,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 0: [2023-05-09 16:44:26,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 0: [2023-05-09 16:44:26,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 0: [2023-05-09 16:44:26,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 7: [2023-05-09 16:44:26,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 7: [2023-05-09 16:44:26,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 7: [2023-05-09 16:44:26,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 7: [2023-05-09 16:44:26,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +22: [2023-05-09 16:44:26,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +22: [2023-05-09 16:44:26,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +22: [2023-05-09 16:44:26,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +22: [2023-05-09 16:44:26,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 7: [2023-05-09 16:44:26,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 7: [2023-05-09 16:44:26,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 0: [2023-05-09 16:44:26,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 0: [2023-05-09 16:44:26,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 0: [2023-05-09 16:44:26,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 0: [2023-05-09 16:44:26,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +22: [2023-05-09 16:44:26,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +22: [2023-05-09 16:44:26,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 7: [2023-05-09 16:44:26,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 7: [2023-05-09 16:44:26,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +28: [2023-05-09 16:44:26,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +28: [2023-05-09 16:44:26,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +28: [2023-05-09 16:44:26,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +28: [2023-05-09 16:44:26,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +29: [2023-05-09 16:44:26,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +29: [2023-05-09 16:44:26,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +29: [2023-05-09 16:44:26,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +29: [2023-05-09 16:44:26,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +22: [2023-05-09 16:44:26,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +22: [2023-05-09 16:44:26,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +28: [2023-05-09 16:44:26,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 7: [2023-05-09 16:44:26,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 7: [2023-05-09 16:44:26,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +28: [2023-05-09 16:44:26,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 7: [2023-05-09 16:44:26,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +28: [2023-05-09 16:44:26,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 7: [2023-05-09 16:44:26,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +28: [2023-05-09 16:44:26,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +29: [2023-05-09 16:44:26,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +29: [2023-05-09 16:44:26,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +29: [2023-05-09 16:44:26,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +29: [2023-05-09 16:44:26,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 0: [2023-05-09 16:44:26,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 0: [2023-05-09 16:44:26,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 0: [2023-05-09 16:44:26,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 0: [2023-05-09 16:44:26,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 0: [2023-05-09 16:44:26,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 0: [2023-05-09 16:44:26,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +28: [2023-05-09 16:44:26,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +28: [2023-05-09 16:44:26,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 7: [2023-05-09 16:44:26,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +29: [2023-05-09 16:44:26,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +29: [2023-05-09 16:44:26,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 7: [2023-05-09 16:44:26,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +29: [2023-05-09 16:44:26,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +29: [2023-05-09 16:44:26,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +29: [2023-05-09 16:44:26,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +29: [2023-05-09 16:44:26,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 7: [2023-05-09 16:44:26,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 7: [2023-05-09 16:44:26,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +28: [2023-05-09 16:44:26,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +28: [2023-05-09 16:44:26,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +28: [2023-05-09 16:44:26,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 0: [2023-05-09 16:44:26,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +28: [2023-05-09 16:44:26,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +28: [2023-05-09 16:44:26,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +29: [2023-05-09 16:44:26,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +29: [2023-05-09 16:44:26,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +28: [2023-05-09 16:44:26,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 0: [2023-05-09 16:44:26,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +24: [2023-05-09 16:44:26,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +24: [2023-05-09 16:44:26,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +24: [2023-05-09 16:44:26,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +24: [2023-05-09 16:44:26,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 2: [2023-05-09 16:44:26,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 2: [2023-05-09 16:44:26,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 2: [2023-05-09 16:44:26,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 2: [2023-05-09 16:44:26,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +21: [2023-05-09 16:44:26,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +21: [2023-05-09 16:44:26,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +21: [2023-05-09 16:44:26,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +21: [2023-05-09 16:44:26,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +24: [2023-05-09 16:44:26,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 2: [2023-05-09 16:44:26,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 2: [2023-05-09 16:44:26,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 2: [2023-05-09 16:44:26,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +24: [2023-05-09 16:44:26,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +24: [2023-05-09 16:44:26,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +21: [2023-05-09 16:44:26,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 2: [2023-05-09 16:44:26,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +21: [2023-05-09 16:44:26,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +21: [2023-05-09 16:44:26,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +24: [2023-05-09 16:44:26,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +21: [2023-05-09 16:44:26,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +24: [2023-05-09 16:44:26,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +24: [2023-05-09 16:44:26,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +24: [2023-05-09 16:44:26,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +24: [2023-05-09 16:44:26,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 2: [2023-05-09 16:44:26,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 2: [2023-05-09 16:44:26,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 2: [2023-05-09 16:44:26,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 2: [2023-05-09 16:44:26,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +21: [2023-05-09 16:44:26,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +21: [2023-05-09 16:44:26,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +24: [2023-05-09 16:44:26,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 2: [2023-05-09 16:44:26,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +21: [2023-05-09 16:44:26,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +21: [2023-05-09 16:44:26,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 2: [2023-05-09 16:44:26,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +24: [2023-05-09 16:44:26,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 2: [2023-05-09 16:44:26,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +21: [2023-05-09 16:44:26,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +21: [2023-05-09 16:44:26,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 2: [2023-05-09 16:44:26,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +21: [2023-05-09 16:44:26,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +21: [2023-05-09 16:44:26,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +24: [2023-05-09 16:44:26,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +24: [2023-05-09 16:44:26,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 4: [2023-05-09 16:44:26,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 4: [2023-05-09 16:44:26,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 4: [2023-05-09 16:44:26,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 4: [2023-05-09 16:44:26,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 4: [2023-05-09 16:44:26,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 4: [2023-05-09 16:44:26,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 4: [2023-05-09 16:44:26,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 4: [2023-05-09 16:44:26,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 4: [2023-05-09 16:44:26,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 4: [2023-05-09 16:44:26,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 4: [2023-05-09 16:44:26,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 4: [2023-05-09 16:44:26,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 4: [2023-05-09 16:44:26,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 4: [2023-05-09 16:44:26,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 4: [2023-05-09 16:44:26,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 4: [2023-05-09 16:44:26,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +14: [2023-05-09 16:44:26,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +14: [2023-05-09 16:44:26,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +14: [2023-05-09 16:44:26,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +14: [2023-05-09 16:44:26,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +14: [2023-05-09 16:44:26,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +17: [2023-05-09 16:44:26,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +17: [2023-05-09 16:44:26,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +17: [2023-05-09 16:44:26,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +14: [2023-05-09 16:44:26,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +17: [2023-05-09 16:44:26,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +14: [2023-05-09 16:44:26,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +14: [2023-05-09 16:44:26,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +14: [2023-05-09 16:44:26,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +14: [2023-05-09 16:44:26,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +14: [2023-05-09 16:44:26,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +14: [2023-05-09 16:44:26,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +14: [2023-05-09 16:44:26,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +14: [2023-05-09 16:44:26,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +14: [2023-05-09 16:44:26,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +14: [2023-05-09 16:44:26,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 8: [2023-05-09 16:44:26,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 8: [2023-05-09 16:44:26,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 8: [2023-05-09 16:44:26,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 8: [2023-05-09 16:44:26,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +17: [2023-05-09 16:44:26,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +10: [2023-05-09 16:44:26,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +10: [2023-05-09 16:44:26,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +10: [2023-05-09 16:44:26,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +17: [2023-05-09 16:44:26,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +10: [2023-05-09 16:44:26,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +17: [2023-05-09 16:44:26,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +17: [2023-05-09 16:44:26,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +17: [2023-05-09 16:44:26,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +17: [2023-05-09 16:44:26,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +23: [2023-05-09 16:44:26,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +23: [2023-05-09 16:44:26,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +23: [2023-05-09 16:44:26,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +26: [2023-05-09 16:44:26,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +26: [2023-05-09 16:44:26,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +26: [2023-05-09 16:44:26,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +26: [2023-05-09 16:44:26,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +23: [2023-05-09 16:44:26,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +17: [2023-05-09 16:44:26,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +17: [2023-05-09 16:44:26,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +17: [2023-05-09 16:44:26,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +17: [2023-05-09 16:44:26,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +17: [2023-05-09 16:44:26,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +17: [2023-05-09 16:44:26,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +30: [2023-05-09 16:44:26,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +30: [2023-05-09 16:44:26,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +30: [2023-05-09 16:44:26,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +30: [2023-05-09 16:44:26,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +20: [2023-05-09 16:44:26,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +20: [2023-05-09 16:44:26,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +20: [2023-05-09 16:44:26,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +20: [2023-05-09 16:44:26,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +27: [2023-05-09 16:44:26,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +27: [2023-05-09 16:44:26,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +31: [2023-05-09 16:44:26,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +31: [2023-05-09 16:44:26,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +31: [2023-05-09 16:44:26,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 8: [2023-05-09 16:44:26,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 8: [2023-05-09 16:44:26,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +31: [2023-05-09 16:44:26,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 8: [2023-05-09 16:44:26,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 8: [2023-05-09 16:44:26,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +27: [2023-05-09 16:44:26,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 8: [2023-05-09 16:44:26,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 8: [2023-05-09 16:44:26,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 8: [2023-05-09 16:44:26,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 8: [2023-05-09 16:44:26,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 8: [2023-05-09 16:44:26,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 8: [2023-05-09 16:44:26,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 8: [2023-05-09 16:44:26,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 8: [2023-05-09 16:44:26,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +27: [2023-05-09 16:44:26,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +14: [2023-05-09 16:44:26,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +14: [2023-05-09 16:44:26,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +14: [2023-05-09 16:44:26,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +14: [2023-05-09 16:44:26,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +17: [2023-05-09 16:44:26,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +17: [2023-05-09 16:44:26,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +17: [2023-05-09 16:44:26,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 8: [2023-05-09 16:44:26,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 8: [2023-05-09 16:44:26,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 8: [2023-05-09 16:44:26,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +17: [2023-05-09 16:44:26,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 8: [2023-05-09 16:44:26,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +26: [2023-05-09 16:44:26,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +26: [2023-05-09 16:44:26,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +23: [2023-05-09 16:44:26,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +26: [2023-05-09 16:44:26,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +26: [2023-05-09 16:44:26,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +26: [2023-05-09 16:44:26,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +10: [2023-05-09 16:44:26,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +26: [2023-05-09 16:44:26,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +26: [2023-05-09 16:44:26,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +26: [2023-05-09 16:44:26,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +26: [2023-05-09 16:44:26,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +26: [2023-05-09 16:44:26,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +10: [2023-05-09 16:44:26,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +26: [2023-05-09 16:44:26,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +26: [2023-05-09 16:44:26,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +23: [2023-05-09 16:44:26,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +23: [2023-05-09 16:44:26,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +30: [2023-05-09 16:44:26,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +23: [2023-05-09 16:44:26,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +30: [2023-05-09 16:44:26,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +30: [2023-05-09 16:44:26,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +10: [2023-05-09 16:44:26,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +10: [2023-05-09 16:44:26,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +20: [2023-05-09 16:44:26,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +10: [2023-05-09 16:44:26,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +20: [2023-05-09 16:44:26,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +20: [2023-05-09 16:44:26,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +10: [2023-05-09 16:44:26,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +10: [2023-05-09 16:44:26,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +10: [2023-05-09 16:44:26,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +26: [2023-05-09 16:44:26,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +26: [2023-05-09 16:44:26,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +26: [2023-05-09 16:44:26,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +10: [2023-05-09 16:44:26,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +30: [2023-05-09 16:44:26,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +26: [2023-05-09 16:44:26,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +20: [2023-05-09 16:44:26,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +20: [2023-05-09 16:44:26,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +27: [2023-05-09 16:44:26,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +20: [2023-05-09 16:44:26,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 9: [2023-05-09 16:44:26,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +10: [2023-05-09 16:44:26,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 9: [2023-05-09 16:44:26,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +10: [2023-05-09 16:44:26,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +10: [2023-05-09 16:44:26,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +20: [2023-05-09 16:44:26,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +20: [2023-05-09 16:44:26,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +27: [2023-05-09 16:44:26,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 9: [2023-05-09 16:44:26,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +31: [2023-05-09 16:44:26,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +31: [2023-05-09 16:44:26,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +10: [2023-05-09 16:44:26,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +10: [2023-05-09 16:44:26,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +10: [2023-05-09 16:44:26,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 9: [2023-05-09 16:44:26,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +20: [2023-05-09 16:44:26,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +20: [2023-05-09 16:44:26,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +20: [2023-05-09 16:44:26,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +10: [2023-05-09 16:44:26,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +20: [2023-05-09 16:44:26,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +27: [2023-05-09 16:44:26,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +20: [2023-05-09 16:44:26,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +27: [2023-05-09 16:44:26,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +20: [2023-05-09 16:44:26,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +20: [2023-05-09 16:44:26,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +20: [2023-05-09 16:44:26,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +31: [2023-05-09 16:44:26,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +31: [2023-05-09 16:44:26,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 5: [2023-05-09 16:44:26,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 5: [2023-05-09 16:44:26,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 3: [2023-05-09 16:44:26,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 3: [2023-05-09 16:44:26,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 5: [2023-05-09 16:44:26,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 5: [2023-05-09 16:44:26,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +23: [2023-05-09 16:44:26,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 3: [2023-05-09 16:44:26,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +23: [2023-05-09 16:44:26,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 3: [2023-05-09 16:44:26,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +30: [2023-05-09 16:44:26,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 9: [2023-05-09 16:44:26,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +30: [2023-05-09 16:44:26,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 9: [2023-05-09 16:44:26,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 9: [2023-05-09 16:44:26,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 9: [2023-05-09 16:44:26,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +30: [2023-05-09 16:44:26,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +30: [2023-05-09 16:44:26,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +30: [2023-05-09 16:44:26,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +30: [2023-05-09 16:44:26,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 5: [2023-05-09 16:44:26,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +11: [2023-05-09 16:44:26,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +11: [2023-05-09 16:44:26,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +11: [2023-05-09 16:44:26,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 3: [2023-05-09 16:44:26,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 3: [2023-05-09 16:44:26,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +27: [2023-05-09 16:44:26,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 5: [2023-05-09 16:44:26,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +11: [2023-05-09 16:44:26,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +27: [2023-05-09 16:44:26,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +27: [2023-05-09 16:44:26,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +23: [2023-05-09 16:44:26,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +23: [2023-05-09 16:44:26,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 5: [2023-05-09 16:44:26,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +27: [2023-05-09 16:44:26,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 5: [2023-05-09 16:44:26,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 3: [2023-05-09 16:44:26,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 3: [2023-05-09 16:44:26,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +30: [2023-05-09 16:44:26,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +31: [2023-05-09 16:44:26,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +30: [2023-05-09 16:44:26,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +31: [2023-05-09 16:44:26,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +27: [2023-05-09 16:44:26,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +31: [2023-05-09 16:44:26,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +27: [2023-05-09 16:44:26,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +31: [2023-05-09 16:44:26,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +23: [2023-05-09 16:44:26,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +31: [2023-05-09 16:44:26,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +31: [2023-05-09 16:44:26,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +23: [2023-05-09 16:44:26,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +23: [2023-05-09 16:44:26,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +15: [2023-05-09 16:44:26,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +15: [2023-05-09 16:44:26,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +15: [2023-05-09 16:44:26,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 9: [2023-05-09 16:44:26,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +15: [2023-05-09 16:44:26,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +23: [2023-05-09 16:44:26,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 9: [2023-05-09 16:44:26,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +27: [2023-05-09 16:44:26,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +11: [2023-05-09 16:44:26,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +11: [2023-05-09 16:44:26,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +11: [2023-05-09 16:44:26,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +11: [2023-05-09 16:44:26,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +27: [2023-05-09 16:44:26,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 9: [2023-05-09 16:44:26,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 9: [2023-05-09 16:44:26,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 9: [2023-05-09 16:44:26,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 9: [2023-05-09 16:44:26,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +17: [2023-05-09 16:44:26,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt... +14: [2023-05-09 16:44:26,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt... +14: [2023-05-09 16:44:26,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt... +14: [2023-05-09 16:44:26,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt... +14: [2023-05-09 16:44:26,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt... +18: [2023-05-09 16:44:26,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +17: [2023-05-09 16:44:26,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt... +18: [2023-05-09 16:44:26,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +18: [2023-05-09 16:44:26,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +17: [2023-05-09 16:44:26,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt... + 8: [2023-05-09 16:44:26,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt... + 8: [2023-05-09 16:44:26,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt... + 8: [2023-05-09 16:44:26,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt... + 8: [2023-05-09 16:44:26,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt... +17: [2023-05-09 16:44:26,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt... + 9: [2023-05-09 16:44:26,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 9: [2023-05-09 16:44:26,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +19: [2023-05-09 16:44:26,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +19: [2023-05-09 16:44:26,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +19: [2023-05-09 16:44:26,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +19: [2023-05-09 16:44:26,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +18: [2023-05-09 16:44:26,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 5: [2023-05-09 16:44:26,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 5: [2023-05-09 16:44:26,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 5: [2023-05-09 16:44:26,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 5: [2023-05-09 16:44:26,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +31: [2023-05-09 16:44:26,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +15: [2023-05-09 16:44:26,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +15: [2023-05-09 16:44:26,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 3: [2023-05-09 16:44:26,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +15: [2023-05-09 16:44:26,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 3: [2023-05-09 16:44:26,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 5: [2023-05-09 16:44:26,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 5: [2023-05-09 16:44:26,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +15: [2023-05-09 16:44:26,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 5: [2023-05-09 16:44:26,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 5: [2023-05-09 16:44:26,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +10: [2023-05-09 16:44:26,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt... +31: [2023-05-09 16:44:26,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +15: [2023-05-09 16:44:26,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +26: [2023-05-09 16:44:26,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt... +26: [2023-05-09 16:44:26,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt... +26: [2023-05-09 16:44:26,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt... +26: [2023-05-09 16:44:26,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt... + 3: [2023-05-09 16:44:26,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +10: [2023-05-09 16:44:26,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt... +10: [2023-05-09 16:44:26,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt... +15: [2023-05-09 16:44:26,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 3: [2023-05-09 16:44:26,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +10: [2023-05-09 16:44:26,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt... + 3: [2023-05-09 16:44:26,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +15: [2023-05-09 16:44:26,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +15: [2023-05-09 16:44:26,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +15: [2023-05-09 16:44:26,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +15: [2023-05-09 16:44:26,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +15: [2023-05-09 16:44:26,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +15: [2023-05-09 16:44:26,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 3: [2023-05-09 16:44:26,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +15: [2023-05-09 16:44:26,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +15: [2023-05-09 16:44:26,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +15: [2023-05-09 16:44:26,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +15: [2023-05-09 16:44:26,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +11: [2023-05-09 16:44:26,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +11: [2023-05-09 16:44:26,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +18: [2023-05-09 16:44:26,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +18: [2023-05-09 16:44:26,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +19: [2023-05-09 16:44:26,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +18: [2023-05-09 16:44:26,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 3: [2023-05-09 16:44:26,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +11: [2023-05-09 16:44:26,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 3: [2023-05-09 16:44:26,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +11: [2023-05-09 16:44:26,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +18: [2023-05-09 16:44:26,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +19: [2023-05-09 16:44:26,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +19: [2023-05-09 16:44:26,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +19: [2023-05-09 16:44:26,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +20: [2023-05-09 16:44:26,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt... +20: [2023-05-09 16:44:26,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt... +20: [2023-05-09 16:44:26,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt... +20: [2023-05-09 16:44:26,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt... +11: [2023-05-09 16:44:26,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +11: [2023-05-09 16:44:26,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +18: [2023-05-09 16:44:26,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +11: [2023-05-09 16:44:26,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +18: [2023-05-09 16:44:26,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +18: [2023-05-09 16:44:26,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +18: [2023-05-09 16:44:26,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +11: [2023-05-09 16:44:26,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +18: [2023-05-09 16:44:26,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +18: [2023-05-09 16:44:26,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +19: [2023-05-09 16:44:26,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +19: [2023-05-09 16:44:26,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +18: [2023-05-09 16:44:26,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +18: [2023-05-09 16:44:26,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +13: [2023-05-09 16:44:26,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 6: [2023-05-09 16:44:26,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +13: [2023-05-09 16:44:26,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +25: [2023-05-09 16:44:26,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 6: [2023-05-09 16:44:26,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 6: [2023-05-09 16:44:26,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +13: [2023-05-09 16:44:26,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +25: [2023-05-09 16:44:26,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +25: [2023-05-09 16:44:26,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. + 6: [2023-05-09 16:44:26,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +13: [2023-05-09 16:44:26,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +25: [2023-05-09 16:44:26,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt. +19: [2023-05-09 16:44:26,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +19: [2023-05-09 16:44:26,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +19: [2023-05-09 16:44:26,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +19: [2023-05-09 16:44:26,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +13: [2023-05-09 16:44:26,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +13: [2023-05-09 16:44:26,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +13: [2023-05-09 16:44:26,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +13: [2023-05-09 16:44:26,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +25: [2023-05-09 16:44:26,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +25: [2023-05-09 16:44:26,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +19: [2023-05-09 16:44:26,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +25: [2023-05-09 16:44:26,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 6: [2023-05-09 16:44:26,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +15: [2023-05-09 16:44:26,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt... +15: [2023-05-09 16:44:26,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt... + 6: [2023-05-09 16:44:26,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 6: [2023-05-09 16:44:26,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +15: [2023-05-09 16:44:26,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt... +15: [2023-05-09 16:44:26,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt... +19: [2023-05-09 16:44:26,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +25: [2023-05-09 16:44:26,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... + 6: [2023-05-09 16:44:26,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt... +16: [2023-05-09 16:44:26,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +16: [2023-05-09 16:44:26,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +16: [2023-05-09 16:44:26,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +16: [2023-05-09 16:44:26,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +12: [2023-05-09 16:44:26,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +12: [2023-05-09 16:44:26,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +12: [2023-05-09 16:44:26,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +12: [2023-05-09 16:44:26,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +25: [2023-05-09 16:44:26,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +25: [2023-05-09 16:44:26,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +13: [2023-05-09 16:44:26,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +13: [2023-05-09 16:44:26,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +13: [2023-05-09 16:44:26,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +13: [2023-05-09 16:44:26,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +13: [2023-05-09 16:44:26,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +13: [2023-05-09 16:44:26,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +16: [2023-05-09 16:44:26,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 6: [2023-05-09 16:44:26,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +16: [2023-05-09 16:44:26,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +13: [2023-05-09 16:44:26,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +13: [2023-05-09 16:44:26,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 6: [2023-05-09 16:44:26,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +16: [2023-05-09 16:44:26,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +16: [2023-05-09 16:44:26,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +16: [2023-05-09 16:44:26,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +16: [2023-05-09 16:44:26,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +16: [2023-05-09 16:44:26,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +16: [2023-05-09 16:44:26,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +16: [2023-05-09 16:44:26,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +16: [2023-05-09 16:44:26,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +16: [2023-05-09 16:44:26,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +16: [2023-05-09 16:44:26,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +12: [2023-05-09 16:44:26,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +16: [2023-05-09 16:44:26,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +16: [2023-05-09 16:44:26,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +16: [2023-05-09 16:44:26,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +16: [2023-05-09 16:44:26,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +12: [2023-05-09 16:44:26,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +12: [2023-05-09 16:44:26,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +29: [2023-05-09 16:44:26,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +29: [2023-05-09 16:44:26,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +29: [2023-05-09 16:44:26,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +29: [2023-05-09 16:44:26,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +25: [2023-05-09 16:44:26,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 6: [2023-05-09 16:44:26,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +25: [2023-05-09 16:44:26,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 6: [2023-05-09 16:44:26,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +22: [2023-05-09 16:44:26,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +22: [2023-05-09 16:44:26,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +22: [2023-05-09 16:44:26,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +22: [2023-05-09 16:44:26,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +12: [2023-05-09 16:44:26,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 6: [2023-05-09 16:44:26,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +25: [2023-05-09 16:44:26,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. + 6: [2023-05-09 16:44:26,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +25: [2023-05-09 16:44:26,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 6: [2023-05-09 16:44:26,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +25: [2023-05-09 16:44:26,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt. +25: [2023-05-09 16:44:26,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... + 6: [2023-05-09 16:44:26,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt... +12: [2023-05-09 16:44:26,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +12: [2023-05-09 16:44:26,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +12: [2023-05-09 16:44:26,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +12: [2023-05-09 16:44:26,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +12: [2023-05-09 16:44:26,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +12: [2023-05-09 16:44:26,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +12: [2023-05-09 16:44:26,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +12: [2023-05-09 16:44:26,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +12: [2023-05-09 16:44:26,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +12: [2023-05-09 16:44:26,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +22: [2023-05-09 16:44:26,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +12: [2023-05-09 16:44:26,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +12: [2023-05-09 16:44:26,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +22: [2023-05-09 16:44:26,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +22: [2023-05-09 16:44:26,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +22: [2023-05-09 16:44:26,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +22: [2023-05-09 16:44:26,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +22: [2023-05-09 16:44:26,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +22: [2023-05-09 16:44:26,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +22: [2023-05-09 16:44:26,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +22: [2023-05-09 16:44:26,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +29: [2023-05-09 16:44:26,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +29: [2023-05-09 16:44:26,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +29: [2023-05-09 16:44:26,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +29: [2023-05-09 16:44:26,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +29: [2023-05-09 16:44:26,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +29: [2023-05-09 16:44:26,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +29: [2023-05-09 16:44:26,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +29: [2023-05-09 16:44:26,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +29: [2023-05-09 16:44:26,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +29: [2023-05-09 16:44:26,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +29: [2023-05-09 16:44:26,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +29: [2023-05-09 16:44:26,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +22: [2023-05-09 16:44:26,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +22: [2023-05-09 16:44:26,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +22: [2023-05-09 16:44:26,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +22: [2023-05-09 16:44:26,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +22: [2023-05-09 16:44:26,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +22: [2023-05-09 16:44:26,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +29: [2023-05-09 16:44:26,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +22: [2023-05-09 16:44:26,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +29: [2023-05-09 16:44:26,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +29: [2023-05-09 16:44:26,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +29: [2023-05-09 16:44:26,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +16: [2023-05-09 16:44:26,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt... +16: [2023-05-09 16:44:26,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt... +16: [2023-05-09 16:44:26,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt... +16: [2023-05-09 16:44:26,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt... +12: [2023-05-09 16:44:26,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt... +12: [2023-05-09 16:44:26,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt... +12: [2023-05-09 16:44:26,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt... +12: [2023-05-09 16:44:26,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt... +28: [2023-05-09 16:44:26,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +28: [2023-05-09 16:44:26,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +28: [2023-05-09 16:44:26,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +28: [2023-05-09 16:44:26,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 1: [2023-05-09 16:44:26,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 1: [2023-05-09 16:44:26,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 1: [2023-05-09 16:44:26,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +24: [2023-05-09 16:44:26,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +24: [2023-05-09 16:44:26,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 1: [2023-05-09 16:44:26,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 7: [2023-05-09 16:44:26,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 7: [2023-05-09 16:44:26,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 7: [2023-05-09 16:44:26,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +24: [2023-05-09 16:44:26,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 7: [2023-05-09 16:44:26,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +24: [2023-05-09 16:44:26,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +22: [2023-05-09 16:44:26,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt... +22: [2023-05-09 16:44:26,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt... +22: [2023-05-09 16:44:26,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt... +22: [2023-05-09 16:44:26,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt... +29: [2023-05-09 16:44:26,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt... +29: [2023-05-09 16:44:26,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt... +29: [2023-05-09 16:44:26,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt... +29: [2023-05-09 16:44:26,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt... +28: [2023-05-09 16:44:26,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 1: [2023-05-09 16:44:26,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 7: [2023-05-09 16:44:26,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +28: [2023-05-09 16:44:26,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 1: [2023-05-09 16:44:26,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +28: [2023-05-09 16:44:26,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 1: [2023-05-09 16:44:26,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 7: [2023-05-09 16:44:26,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +28: [2023-05-09 16:44:26,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +28: [2023-05-09 16:44:26,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 1: [2023-05-09 16:44:26,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +28: [2023-05-09 16:44:26,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 1: [2023-05-09 16:44:26,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 1: [2023-05-09 16:44:26,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +28: [2023-05-09 16:44:26,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +28: [2023-05-09 16:44:26,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +28: [2023-05-09 16:44:26,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +28: [2023-05-09 16:44:26,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +28: [2023-05-09 16:44:26,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 1: [2023-05-09 16:44:26,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 1: [2023-05-09 16:44:26,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +28: [2023-05-09 16:44:26,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 7: [2023-05-09 16:44:26,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 7: [2023-05-09 16:44:26,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 1: [2023-05-09 16:44:26,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 1: [2023-05-09 16:44:26,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 1: [2023-05-09 16:44:26,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 7: [2023-05-09 16:44:26,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 1: [2023-05-09 16:44:26,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 7: [2023-05-09 16:44:26,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 7: [2023-05-09 16:44:26,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 7: [2023-05-09 16:44:26,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 7: [2023-05-09 16:44:26,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 7: [2023-05-09 16:44:26,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 7: [2023-05-09 16:44:26,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 7: [2023-05-09 16:44:26,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +28: [2023-05-09 16:44:26,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +28: [2023-05-09 16:44:26,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +28: [2023-05-09 16:44:26,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +28: [2023-05-09 16:44:26,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +24: [2023-05-09 16:44:26,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +24: [2023-05-09 16:44:26,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 1: [2023-05-09 16:44:26,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 1: [2023-05-09 16:44:26,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 1: [2023-05-09 16:44:26,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +24: [2023-05-09 16:44:26,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 7: [2023-05-09 16:44:26,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 7: [2023-05-09 16:44:26,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 1: [2023-05-09 16:44:26,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +24: [2023-05-09 16:44:26,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 7: [2023-05-09 16:44:26,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 7: [2023-05-09 16:44:26,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +24: [2023-05-09 16:44:26,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +24: [2023-05-09 16:44:26,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +24: [2023-05-09 16:44:26,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +24: [2023-05-09 16:44:26,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +24: [2023-05-09 16:44:26,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +24: [2023-05-09 16:44:26,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +24: [2023-05-09 16:44:26,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +24: [2023-05-09 16:44:26,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +24: [2023-05-09 16:44:26,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +24: [2023-05-09 16:44:26,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +24: [2023-05-09 16:44:26,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +24: [2023-05-09 16:44:26,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 4: [2023-05-09 16:44:26,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt. + 4: [2023-05-09 16:44:26,521] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 32 + 4: [2023-05-09 16:44:26,525] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 32 + 2: [2023-05-09 16:44:26,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 2: [2023-05-09 16:44:26,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +21: [2023-05-09 16:44:26,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +21: [2023-05-09 16:44:26,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +21: [2023-05-09 16:44:26,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +21: [2023-05-09 16:44:26,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 2: [2023-05-09 16:44:26,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 2: [2023-05-09 16:44:26,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 7: [2023-05-09 16:44:26,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt... + 7: [2023-05-09 16:44:26,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt... + 7: [2023-05-09 16:44:26,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt... + 7: [2023-05-09 16:44:26,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt... +28: [2023-05-09 16:44:26,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt... +28: [2023-05-09 16:44:26,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt... +28: [2023-05-09 16:44:26,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt... +28: [2023-05-09 16:44:26,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt... + 1: [2023-05-09 16:44:26,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt... + 1: [2023-05-09 16:44:26,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt... +21: [2023-05-09 16:44:26,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 1: [2023-05-09 16:44:26,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt... + 1: [2023-05-09 16:44:26,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt... +21: [2023-05-09 16:44:26,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +21: [2023-05-09 16:44:26,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +21: [2023-05-09 16:44:26,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +21: [2023-05-09 16:44:26,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +21: [2023-05-09 16:44:26,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +21: [2023-05-09 16:44:26,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +21: [2023-05-09 16:44:26,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +21: [2023-05-09 16:44:26,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +21: [2023-05-09 16:44:26,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +21: [2023-05-09 16:44:26,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +21: [2023-05-09 16:44:26,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 2: [2023-05-09 16:44:26,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 4: [2023-05-09 16:44:26,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +23: [2023-05-09 16:44:26,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +21: [2023-05-09 16:44:26,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +21: [2023-05-09 16:44:26,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 2: [2023-05-09 16:44:26,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +21: [2023-05-09 16:44:26,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 2: [2023-05-09 16:44:26,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 4: [2023-05-09 16:44:26,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 4: [2023-05-09 16:44:26,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +21: [2023-05-09 16:44:26,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 2: [2023-05-09 16:44:26,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 4: [2023-05-09 16:44:26,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 2: [2023-05-09 16:44:26,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 2: [2023-05-09 16:44:26,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 2: [2023-05-09 16:44:26,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 2: [2023-05-09 16:44:26,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 2: [2023-05-09 16:44:26,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 2: [2023-05-09 16:44:26,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 2: [2023-05-09 16:44:26,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 2: [2023-05-09 16:44:26,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 2: [2023-05-09 16:44:26,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 2: [2023-05-09 16:44:26,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 2: [2023-05-09 16:44:26,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 2: [2023-05-09 16:44:26,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +23: [2023-05-09 16:44:26,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +24: [2023-05-09 16:44:26,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt... +24: [2023-05-09 16:44:26,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt... +24: [2023-05-09 16:44:26,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt... +24: [2023-05-09 16:44:26,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt... +23: [2023-05-09 16:44:26,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +23: [2023-05-09 16:44:26,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 4: [2023-05-09 16:44:26,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 4: [2023-05-09 16:44:26,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 4: [2023-05-09 16:44:26,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 4: [2023-05-09 16:44:26,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 4: [2023-05-09 16:44:26,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 4: [2023-05-09 16:44:26,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 4: [2023-05-09 16:44:26,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 4: [2023-05-09 16:44:26,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 4: [2023-05-09 16:44:26,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +23: [2023-05-09 16:44:26,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 4: [2023-05-09 16:44:26,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 4: [2023-05-09 16:44:26,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +23: [2023-05-09 16:44:26,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 4: [2023-05-09 16:44:26,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +23: [2023-05-09 16:44:26,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +23: [2023-05-09 16:44:26,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +23: [2023-05-09 16:44:26,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +23: [2023-05-09 16:44:26,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 4: [2023-05-09 16:44:26,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 4: [2023-05-09 16:44:26,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 4: [2023-05-09 16:44:26,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 4: [2023-05-09 16:44:26,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +23: [2023-05-09 16:44:26,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +23: [2023-05-09 16:44:26,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +23: [2023-05-09 16:44:26,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +23: [2023-05-09 16:44:26,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +23: [2023-05-09 16:44:26,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +23: [2023-05-09 16:44:26,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +23: [2023-05-09 16:44:26,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +23: [2023-05-09 16:44:26,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +23: [2023-05-09 16:44:26,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +23: [2023-05-09 16:44:26,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 3: [2023-05-09 16:44:26,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 3: [2023-05-09 16:44:26,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 3: [2023-05-09 16:44:26,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 3: [2023-05-09 16:44:26,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +19: [2023-05-09 16:44:26,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +19: [2023-05-09 16:44:26,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +19: [2023-05-09 16:44:26,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +19: [2023-05-09 16:44:26,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 2: [2023-05-09 16:44:26,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt... + 2: [2023-05-09 16:44:26,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt... + 2: [2023-05-09 16:44:26,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt... + 2: [2023-05-09 16:44:26,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt... + 3: [2023-05-09 16:44:26,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 3: [2023-05-09 16:44:26,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 3: [2023-05-09 16:44:26,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 3: [2023-05-09 16:44:26,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +19: [2023-05-09 16:44:26,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 3: [2023-05-09 16:44:26,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 3: [2023-05-09 16:44:26,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +19: [2023-05-09 16:44:26,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +19: [2023-05-09 16:44:26,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +19: [2023-05-09 16:44:26,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 3: [2023-05-09 16:44:26,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +19: [2023-05-09 16:44:26,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 3: [2023-05-09 16:44:26,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +19: [2023-05-09 16:44:26,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 3: [2023-05-09 16:44:26,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 3: [2023-05-09 16:44:26,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 3: [2023-05-09 16:44:26,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 3: [2023-05-09 16:44:26,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +19: [2023-05-09 16:44:26,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +19: [2023-05-09 16:44:26,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 3: [2023-05-09 16:44:26,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 3: [2023-05-09 16:44:26,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 3: [2023-05-09 16:44:26,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +19: [2023-05-09 16:44:26,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +19: [2023-05-09 16:44:26,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 3: [2023-05-09 16:44:26,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +19: [2023-05-09 16:44:26,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +19: [2023-05-09 16:44:26,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +19: [2023-05-09 16:44:26,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +19: [2023-05-09 16:44:26,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +19: [2023-05-09 16:44:26,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +19: [2023-05-09 16:44:26,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +27: [2023-05-09 16:44:26,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +21: [2023-05-09 16:44:26,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt... +21: [2023-05-09 16:44:26,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt... +21: [2023-05-09 16:44:26,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt... +21: [2023-05-09 16:44:26,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt... +27: [2023-05-09 16:44:26,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 9: [2023-05-09 16:44:26,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 9: [2023-05-09 16:44:26,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 9: [2023-05-09 16:44:26,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 9: [2023-05-09 16:44:26,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +27: [2023-05-09 16:44:26,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +27: [2023-05-09 16:44:26,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 0: [2023-05-09 16:44:26,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 0: [2023-05-09 16:44:26,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 0: [2023-05-09 16:44:26,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +30: [2023-05-09 16:44:26,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +30: [2023-05-09 16:44:26,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +30: [2023-05-09 16:44:26,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +30: [2023-05-09 16:44:26,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 9: [2023-05-09 16:44:26,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 9: [2023-05-09 16:44:26,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 9: [2023-05-09 16:44:26,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 9: [2023-05-09 16:44:26,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +27: [2023-05-09 16:44:26,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +27: [2023-05-09 16:44:26,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 9: [2023-05-09 16:44:26,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +27: [2023-05-09 16:44:26,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 9: [2023-05-09 16:44:26,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 4: [2023-05-09 16:44:26,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt... + 4: [2023-05-09 16:44:26,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt... + 4: [2023-05-09 16:44:26,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt... + 4: [2023-05-09 16:44:26,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt... +27: [2023-05-09 16:44:26,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +27: [2023-05-09 16:44:26,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 9: [2023-05-09 16:44:26,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +27: [2023-05-09 16:44:26,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 9: [2023-05-09 16:44:26,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 9: [2023-05-09 16:44:26,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 9: [2023-05-09 16:44:26,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 9: [2023-05-09 16:44:26,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 9: [2023-05-09 16:44:26,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 9: [2023-05-09 16:44:26,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 9: [2023-05-09 16:44:26,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 9: [2023-05-09 16:44:26,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +27: [2023-05-09 16:44:26,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 9: [2023-05-09 16:44:26,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +27: [2023-05-09 16:44:26,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +27: [2023-05-09 16:44:26,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +27: [2023-05-09 16:44:26,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +27: [2023-05-09 16:44:26,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +27: [2023-05-09 16:44:26,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +27: [2023-05-09 16:44:26,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +27: [2023-05-09 16:44:26,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +27: [2023-05-09 16:44:26,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +27: [2023-05-09 16:44:26,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +30: [2023-05-09 16:44:26,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +30: [2023-05-09 16:44:26,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +30: [2023-05-09 16:44:26,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +30: [2023-05-09 16:44:26,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +30: [2023-05-09 16:44:26,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +30: [2023-05-09 16:44:26,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +23: [2023-05-09 16:44:26,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt... +30: [2023-05-09 16:44:26,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +30: [2023-05-09 16:44:26,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +30: [2023-05-09 16:44:26,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +23: [2023-05-09 16:44:26,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt... +30: [2023-05-09 16:44:26,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +23: [2023-05-09 16:44:26,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt... +23: [2023-05-09 16:44:26,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt... +30: [2023-05-09 16:44:26,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +30: [2023-05-09 16:44:26,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +30: [2023-05-09 16:44:26,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +30: [2023-05-09 16:44:26,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +30: [2023-05-09 16:44:26,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +30: [2023-05-09 16:44:26,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +31: [2023-05-09 16:44:26,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +31: [2023-05-09 16:44:26,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +31: [2023-05-09 16:44:26,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +31: [2023-05-09 16:44:26,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +19: [2023-05-09 16:44:26,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt... +19: [2023-05-09 16:44:26,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt... +19: [2023-05-09 16:44:26,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt... +19: [2023-05-09 16:44:26,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt... + 0: [2023-05-09 16:44:26,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +18: [2023-05-09 16:44:26,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +11: [2023-05-09 16:44:26,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +18: [2023-05-09 16:44:26,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +18: [2023-05-09 16:44:26,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +11: [2023-05-09 16:44:26,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +11: [2023-05-09 16:44:26,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 3: [2023-05-09 16:44:26,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt... + 3: [2023-05-09 16:44:26,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt... + 3: [2023-05-09 16:44:26,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt... + 3: [2023-05-09 16:44:26,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt... +25: [2023-05-09 16:44:26,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +18: [2023-05-09 16:44:26,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +25: [2023-05-09 16:44:26,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +25: [2023-05-09 16:44:26,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +11: [2023-05-09 16:44:26,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +25: [2023-05-09 16:44:26,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +31: [2023-05-09 16:44:26,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 5: [2023-05-09 16:44:26,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +31: [2023-05-09 16:44:26,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 5: [2023-05-09 16:44:26,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 5: [2023-05-09 16:44:26,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 5: [2023-05-09 16:44:26,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +31: [2023-05-09 16:44:26,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +31: [2023-05-09 16:44:26,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +31: [2023-05-09 16:44:26,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +31: [2023-05-09 16:44:26,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +31: [2023-05-09 16:44:26,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +31: [2023-05-09 16:44:26,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +31: [2023-05-09 16:44:26,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +31: [2023-05-09 16:44:26,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +31: [2023-05-09 16:44:26,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +31: [2023-05-09 16:44:26,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +31: [2023-05-09 16:44:26,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +31: [2023-05-09 16:44:26,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +31: [2023-05-09 16:44:26,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +31: [2023-05-09 16:44:26,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +11: [2023-05-09 16:44:26,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +18: [2023-05-09 16:44:26,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +11: [2023-05-09 16:44:26,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +11: [2023-05-09 16:44:26,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +18: [2023-05-09 16:44:26,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 0: [2023-05-09 16:44:26,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +11: [2023-05-09 16:44:26,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +25: [2023-05-09 16:44:26,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +18: [2023-05-09 16:44:26,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +18: [2023-05-09 16:44:26,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +11: [2023-05-09 16:44:26,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +11: [2023-05-09 16:44:26,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +25: [2023-05-09 16:44:26,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +18: [2023-05-09 16:44:26,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +11: [2023-05-09 16:44:26,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +18: [2023-05-09 16:44:26,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +18: [2023-05-09 16:44:26,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +18: [2023-05-09 16:44:26,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +11: [2023-05-09 16:44:26,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +11: [2023-05-09 16:44:26,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +18: [2023-05-09 16:44:26,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +18: [2023-05-09 16:44:26,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +25: [2023-05-09 16:44:26,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +25: [2023-05-09 16:44:26,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +18: [2023-05-09 16:44:26,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +25: [2023-05-09 16:44:26,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +18: [2023-05-09 16:44:26,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +25: [2023-05-09 16:44:26,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +25: [2023-05-09 16:44:26,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +25: [2023-05-09 16:44:26,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +25: [2023-05-09 16:44:26,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +18: [2023-05-09 16:44:26,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +18: [2023-05-09 16:44:26,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +18: [2023-05-09 16:44:26,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +11: [2023-05-09 16:44:26,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +11: [2023-05-09 16:44:26,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 0: [2023-05-09 16:44:26,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +18: [2023-05-09 16:44:26,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +11: [2023-05-09 16:44:26,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +25: [2023-05-09 16:44:26,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +11: [2023-05-09 16:44:26,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 9: [2023-05-09 16:44:26,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt... + 9: [2023-05-09 16:44:26,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt... + 9: [2023-05-09 16:44:26,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt... + 9: [2023-05-09 16:44:26,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt... + 5: [2023-05-09 16:44:26,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +25: [2023-05-09 16:44:26,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +11: [2023-05-09 16:44:26,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +11: [2023-05-09 16:44:26,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +25: [2023-05-09 16:44:26,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 5: [2023-05-09 16:44:26,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 5: [2023-05-09 16:44:26,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +25: [2023-05-09 16:44:26,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +25: [2023-05-09 16:44:26,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +25: [2023-05-09 16:44:26,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +11: [2023-05-09 16:44:26,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +25: [2023-05-09 16:44:26,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 5: [2023-05-09 16:44:26,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 5: [2023-05-09 16:44:26,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 5: [2023-05-09 16:44:26,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 5: [2023-05-09 16:44:26,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 5: [2023-05-09 16:44:26,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 5: [2023-05-09 16:44:26,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 5: [2023-05-09 16:44:26,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 5: [2023-05-09 16:44:26,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 5: [2023-05-09 16:44:26,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 5: [2023-05-09 16:44:26,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 5: [2023-05-09 16:44:26,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 5: [2023-05-09 16:44:26,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 5: [2023-05-09 16:44:26,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 0: [2023-05-09 16:44:26,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +27: [2023-05-09 16:44:26,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt... +27: [2023-05-09 16:44:26,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt... +27: [2023-05-09 16:44:26,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt... +27: [2023-05-09 16:44:26,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt... +30: [2023-05-09 16:44:26,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt... +30: [2023-05-09 16:44:26,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt... +30: [2023-05-09 16:44:26,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt... +30: [2023-05-09 16:44:26,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt... + 0: [2023-05-09 16:44:26,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 0: [2023-05-09 16:44:26,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 0: [2023-05-09 16:44:26,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 0: [2023-05-09 16:44:26,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 0: [2023-05-09 16:44:26,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 0: [2023-05-09 16:44:26,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 0: [2023-05-09 16:44:26,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 0: [2023-05-09 16:44:26,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 0: [2023-05-09 16:44:26,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 0: [2023-05-09 16:44:26,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 0: [2023-05-09 16:44:26,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 0: [2023-05-09 16:44:26,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 0: [2023-05-09 16:44:26,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +13: [2023-05-09 16:44:26,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +13: [2023-05-09 16:44:26,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +13: [2023-05-09 16:44:26,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +13: [2023-05-09 16:44:26,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +20: [2023-05-09 16:44:26,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt. +20: [2023-05-09 16:44:26,650] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 164 +31: [2023-05-09 16:44:26,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt... +31: [2023-05-09 16:44:26,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt... +31: [2023-05-09 16:44:26,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt... +31: [2023-05-09 16:44:26,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt... + 6: [2023-05-09 16:44:26,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 6: [2023-05-09 16:44:26,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +20: [2023-05-09 16:44:26,655] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 164 + 6: [2023-05-09 16:44:26,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. + 6: [2023-05-09 16:44:26,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt. +13: [2023-05-09 16:44:26,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +13: [2023-05-09 16:44:26,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +13: [2023-05-09 16:44:26,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +13: [2023-05-09 16:44:26,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +13: [2023-05-09 16:44:26,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... +13: [2023-05-09 16:44:26,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +13: [2023-05-09 16:44:26,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +13: [2023-05-09 16:44:26,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +13: [2023-05-09 16:44:26,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +13: [2023-05-09 16:44:26,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +11: [2023-05-09 16:44:26,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt... +11: [2023-05-09 16:44:26,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt... +11: [2023-05-09 16:44:26,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt... +13: [2023-05-09 16:44:26,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. +11: [2023-05-09 16:44:26,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt... +13: [2023-05-09 16:44:26,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... +18: [2023-05-09 16:44:26,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt... +18: [2023-05-09 16:44:26,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt... +18: [2023-05-09 16:44:26,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt... +18: [2023-05-09 16:44:26,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt... +13: [2023-05-09 16:44:26,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +13: [2023-05-09 16:44:26,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +13: [2023-05-09 16:44:26,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. +13: [2023-05-09 16:44:26,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 6: [2023-05-09 16:44:26,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 6: [2023-05-09 16:44:26,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 5: [2023-05-09 16:44:26,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt... + 6: [2023-05-09 16:44:26,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 5: [2023-05-09 16:44:26,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt... + 5: [2023-05-09 16:44:26,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt... + 5: [2023-05-09 16:44:26,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt... +25: [2023-05-09 16:44:26,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt... +25: [2023-05-09 16:44:26,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt... +25: [2023-05-09 16:44:26,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt... +25: [2023-05-09 16:44:26,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt... + 6: [2023-05-09 16:44:26,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 6: [2023-05-09 16:44:26,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 6: [2023-05-09 16:44:26,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 6: [2023-05-09 16:44:26,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt... + 6: [2023-05-09 16:44:26,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 6: [2023-05-09 16:44:26,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 6: [2023-05-09 16:44:26,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt. + 6: [2023-05-09 16:44:26,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 6: [2023-05-09 16:44:26,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 6: [2023-05-09 16:44:26,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt... + 6: [2023-05-09 16:44:26,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 6: [2023-05-09 16:44:26,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 6: [2023-05-09 16:44:26,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt. + 0: [2023-05-09 16:44:26,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt... + 0: [2023-05-09 16:44:26,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt... + 0: [2023-05-09 16:44:26,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt... + 0: [2023-05-09 16:44:26,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt... +13: [2023-05-09 16:44:26,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt... +13: [2023-05-09 16:44:26,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt... +13: [2023-05-09 16:44:26,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt... +13: [2023-05-09 16:44:26,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt... + 6: [2023-05-09 16:44:26,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt... + 6: [2023-05-09 16:44:26,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt... + 6: [2023-05-09 16:44:26,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt... + 6: [2023-05-09 16:44:26,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt... +10: [2023-05-09 16:44:26,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt. +10: [2023-05-09 16:44:26,733] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 83 +10: [2023-05-09 16:44:26,738] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 83 + 8: [2023-05-09 16:44:26,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt. + 8: [2023-05-09 16:44:26,742] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 67 + 8: [2023-05-09 16:44:26,747] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 67 +25: [2023-05-09 16:44:26,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt. +25: [2023-05-09 16:44:26,756] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 202 +14: [2023-05-09 16:44:26,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt. +14: [2023-05-09 16:44:26,759] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 117 +25: [2023-05-09 16:44:26,761] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 202 +14: [2023-05-09 16:44:26,764] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 117 +20: [2023-05-09 16:44:26,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt. +20: [2023-05-09 16:44:26,769] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 163 +14: [2023-05-09 16:44:26,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt. +14: [2023-05-09 16:44:26,769] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 119 +15: [2023-05-09 16:44:26,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt. +15: [2023-05-09 16:44:26,772] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 125 +14: [2023-05-09 16:44:26,774] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 119 +17: [2023-05-09 16:44:26,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt. +17: [2023-05-09 16:44:26,774] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 141 + 8: [2023-05-09 16:44:26,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt. + 8: [2023-05-09 16:44:26,775] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 65 +20: [2023-05-09 16:44:26,775] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 163 +15: [2023-05-09 16:44:26,776] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 125 + 8: [2023-05-09 16:44:26,780] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 65 +17: [2023-05-09 16:44:26,780] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 141 +26: [2023-05-09 16:44:26,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt. +26: [2023-05-09 16:44:26,804] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 211 +26: [2023-05-09 16:44:26,809] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 211 +12: [2023-05-09 16:44:26,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt. +12: [2023-05-09 16:44:26,814] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 97 +12: [2023-05-09 16:44:26,818] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 97 +14: [2023-05-09 16:44:26,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt. +14: [2023-05-09 16:44:26,819] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 115 +17: [2023-05-09 16:44:26,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt. +17: [2023-05-09 16:44:26,822] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 139 +26: [2023-05-09 16:44:26,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt. +26: [2023-05-09 16:44:26,822] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 209 +14: [2023-05-09 16:44:26,823] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 115 +26: [2023-05-09 16:44:26,826] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 209 +17: [2023-05-09 16:44:26,828] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 139 +17: [2023-05-09 16:44:26,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt. +17: [2023-05-09 16:44:26,830] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 137 +14: [2023-05-09 16:44:26,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt. +14: [2023-05-09 16:44:26,830] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 113 + 5: [2023-05-09 16:44:26,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt. + 5: [2023-05-09 16:44:26,831] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 46 +14: [2023-05-09 16:44:26,834] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 113 + 5: [2023-05-09 16:44:26,835] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 46 +17: [2023-05-09 16:44:26,836] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 137 +15: [2023-05-09 16:44:26,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt. +15: [2023-05-09 16:44:26,839] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 127 +15: [2023-05-09 16:44:26,845] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 127 +10: [2023-05-09 16:44:26,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt. +10: [2023-05-09 16:44:26,849] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 87 + 8: [2023-05-09 16:44:26,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt. + 8: [2023-05-09 16:44:26,849] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 69 +20: [2023-05-09 16:44:26,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt. +20: [2023-05-09 16:44:26,853] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 167 +10: [2023-05-09 16:44:26,853] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 87 + 8: [2023-05-09 16:44:26,854] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 69 +20: [2023-05-09 16:44:26,857] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 167 +22: [2023-05-09 16:44:26,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt. +22: [2023-05-09 16:44:26,858] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 177 +26: [2023-05-09 16:44:26,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt. +26: [2023-05-09 16:44:26,859] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 213 +20: [2023-05-09 16:44:26,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt. +20: [2023-05-09 16:44:26,860] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 161 +15: [2023-05-09 16:44:26,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt. +15: [2023-05-09 16:44:26,862] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 121 +22: [2023-05-09 16:44:26,862] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 177 +26: [2023-05-09 16:44:26,864] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 213 +20: [2023-05-09 16:44:26,864] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 161 +10: [2023-05-09 16:44:26,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt. +10: [2023-05-09 16:44:26,865] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 81 +15: [2023-05-09 16:44:26,867] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 121 +10: [2023-05-09 16:44:26,870] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 81 +22: [2023-05-09 16:44:26,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt. +22: [2023-05-09 16:44:26,875] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 181 +22: [2023-05-09 16:44:26,879] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 181 +17: [2023-05-09 16:44:26,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt. +17: [2023-05-09 16:44:26,880] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 143 +12: [2023-05-09 16:44:26,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt. +12: [2023-05-09 16:44:26,882] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 101 +17: [2023-05-09 16:44:26,886] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 143 +10: [2023-05-09 16:44:26,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt. +26: [2023-05-09 16:44:26,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt. +10: [2023-05-09 16:44:26,889] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 85 +26: [2023-05-09 16:44:26,889] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 215 +12: [2023-05-09 16:44:26,886] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 101 +26: [2023-05-09 16:44:26,893] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 215 +10: [2023-05-09 16:44:26,894] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 85 +29: [2023-05-09 16:44:26,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt. +29: [2023-05-09 16:44:26,897] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 235 +29: [2023-05-09 16:44:26,901] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 235 +29: [2023-05-09 16:44:26,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt. +29: [2023-05-09 16:44:26,902] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 237 +29: [2023-05-09 16:44:26,906] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 237 +20: [2023-05-09 16:44:26,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt. +20: [2023-05-09 16:44:26,909] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 165 + 1: [2023-05-09 16:44:26,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt. + 1: [2023-05-09 16:44:26,909] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 11 +20: [2023-05-09 16:44:26,913] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 165 + 1: [2023-05-09 16:44:26,913] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 11 +15: [2023-05-09 16:44:26,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt. +15: [2023-05-09 16:44:26,915] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 123 +15: [2023-05-09 16:44:26,919] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 123 + 8: [2023-05-09 16:44:26,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt. + 8: [2023-05-09 16:44:26,920] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 71 + 3: [2023-05-09 16:44:26,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt. + 3: [2023-05-09 16:44:26,921] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 29 + 8: [2023-05-09 16:44:26,925] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 71 + 3: [2023-05-09 16:44:26,925] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 29 +21: [2023-05-09 16:44:26,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt. +21: [2023-05-09 16:44:26,925] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 175 +23: [2023-05-09 16:44:26,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt. +23: [2023-05-09 16:44:26,927] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 185 +21: [2023-05-09 16:44:26,930] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 175 +16: [2023-05-09 16:44:26,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt. +23: [2023-05-09 16:44:26,931] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 185 +16: [2023-05-09 16:44:26,931] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 135 +16: [2023-05-09 16:44:26,935] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 135 + 3: [2023-05-09 16:44:26,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt. + 3: [2023-05-09 16:44:26,940] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 25 + 3: [2023-05-09 16:44:26,944] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 25 + 1: [2023-05-09 16:44:26,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt. + 1: [2023-05-09 16:44:26,948] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 9 +29: [2023-05-09 16:44:26,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt. +29: [2023-05-09 16:44:26,951] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 239 + 1: [2023-05-09 16:44:26,953] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 9 + 2: [2023-05-09 16:44:26,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt. + 2: [2023-05-09 16:44:26,954] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 19 + 7: [2023-05-09 16:44:26,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt. + 7: [2023-05-09 16:44:26,954] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 63 +29: [2023-05-09 16:44:26,956] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 239 +16: [2023-05-09 16:44:26,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt. +16: [2023-05-09 16:44:26,956] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 131 + 2: [2023-05-09 16:44:26,958] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 19 + 7: [2023-05-09 16:44:26,958] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 63 +16: [2023-05-09 16:44:26,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt. +16: [2023-05-09 16:44:26,960] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 129 +16: [2023-05-09 16:44:26,960] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 131 +28: [2023-05-09 16:44:26,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt. +28: [2023-05-09 16:44:26,963] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 229 +29: [2023-05-09 16:44:26,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt. +29: [2023-05-09 16:44:26,964] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 233 +16: [2023-05-09 16:44:26,964] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 129 + 2: [2023-05-09 16:44:26,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt. + 2: [2023-05-09 16:44:26,966] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 21 +24: [2023-05-09 16:44:26,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt. +24: [2023-05-09 16:44:26,966] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 197 +28: [2023-05-09 16:44:26,967] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 229 +29: [2023-05-09 16:44:26,968] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 233 + 2: [2023-05-09 16:44:26,970] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 21 +24: [2023-05-09 16:44:26,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt. +24: [2023-05-09 16:44:26,972] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 195 +24: [2023-05-09 16:44:26,973] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 197 + 2: [2023-05-09 16:44:26,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt. + 2: [2023-05-09 16:44:26,973] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 23 +16: [2023-05-09 16:44:26,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt. +16: [2023-05-09 16:44:26,974] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 133 +19: [2023-05-09 16:44:26,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt. +19: [2023-05-09 16:44:26,975] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 157 +24: [2023-05-09 16:44:26,977] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 195 + 2: [2023-05-09 16:44:26,978] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 23 +16: [2023-05-09 16:44:26,978] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 133 +19: [2023-05-09 16:44:26,980] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 157 +25: [2023-05-09 16:44:26,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt. +25: [2023-05-09 16:44:26,980] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 201 +25: [2023-05-09 16:44:26,984] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 201 +28: [2023-05-09 16:44:26,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt. +28: [2023-05-09 16:44:26,986] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 227 + 7: [2023-05-09 16:44:26,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt. + 7: [2023-05-09 16:44:26,987] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 61 + 4: [2023-05-09 16:44:26,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt. + 4: [2023-05-09 16:44:26,988] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 39 +28: [2023-05-09 16:44:26,990] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 227 + 7: [2023-05-09 16:44:26,991] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 61 + 4: [2023-05-09 16:44:26,992] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 39 + 7: [2023-05-09 16:44:26,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt. + 7: [2023-05-09 16:44:26,993] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 57 +27: [2023-05-09 16:44:26,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt. +27: [2023-05-09 16:44:26,994] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 219 +25: [2023-05-09 16:44:26,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt. +25: [2023-05-09 16:44:26,994] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 205 + 9: [2023-05-09 16:44:26,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt. + 4: [2023-05-09 16:44:26,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt. + 9: [2023-05-09 16:44:26,996] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 73 + 4: [2023-05-09 16:44:26,997] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 33 + 7: [2023-05-09 16:44:26,997] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 57 +27: [2023-05-09 16:44:26,998] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 219 + 9: [2023-05-09 16:44:27,001] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 73 + 4: [2023-05-09 16:44:27,001] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 33 +25: [2023-05-09 16:44:27,001] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 205 +21: [2023-05-09 16:44:27,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt. +21: [2023-05-09 16:44:27,001] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 169 +21: [2023-05-09 16:44:27,006] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 169 + 3: [2023-05-09 16:44:27,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt. + 3: [2023-05-09 16:44:27,010] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 31 +28: [2023-05-09 16:44:27,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt. +28: [2023-05-09 16:44:27,010] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 231 + 1: [2023-05-09 16:44:27,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt. + 1: [2023-05-09 16:44:27,011] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 15 + 3: [2023-05-09 16:44:27,014] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 31 +28: [2023-05-09 16:44:27,015] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 231 + 1: [2023-05-09 16:44:27,015] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 15 +22: [2023-05-09 16:44:27,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt. +22: [2023-05-09 16:44:27,017] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 179 +22: [2023-05-09 16:44:27,023] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 179 +27: [2023-05-09 16:44:27,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt. +11: [2023-05-09 16:44:27,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt. +27: [2023-05-09 16:44:27,027] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 223 +12: [2023-05-09 16:44:27,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt. +11: [2023-05-09 16:44:27,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt. + 9: [2023-05-09 16:44:27,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt. +12: [2023-05-09 16:44:27,028] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 103 +11: [2023-05-09 16:44:27,028] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 89 +11: [2023-05-09 16:44:27,028] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 91 + 9: [2023-05-09 16:44:27,028] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 75 +31: [2023-05-09 16:44:27,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt. +31: [2023-05-09 16:44:27,030] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 249 + 7: [2023-05-09 16:44:27,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt. +27: [2023-05-09 16:44:27,031] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 223 + 7: [2023-05-09 16:44:27,031] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 59 +13: [2023-05-09 16:44:27,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt. +22: [2023-05-09 16:44:27,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt. +21: [2023-05-09 16:44:27,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt. + 9: [2023-05-09 16:44:27,032] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 75 +13: [2023-05-09 16:44:27,032] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 105 +22: [2023-05-09 16:44:27,032] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 183 +21: [2023-05-09 16:44:27,032] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 173 +11: [2023-05-09 16:44:27,032] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 89 +12: [2023-05-09 16:44:27,032] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 103 +11: [2023-05-09 16:44:27,033] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 91 +23: [2023-05-09 16:44:27,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt. +23: [2023-05-09 16:44:27,033] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 187 +11: [2023-05-09 16:44:27,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt. +11: [2023-05-09 16:44:27,034] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 95 +31: [2023-05-09 16:44:27,035] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 249 + 7: [2023-05-09 16:44:27,036] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 59 +22: [2023-05-09 16:44:27,036] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 183 +13: [2023-05-09 16:44:27,036] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 105 +21: [2023-05-09 16:44:27,036] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 173 +23: [2023-05-09 16:44:27,038] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 187 + 2: [2023-05-09 16:44:27,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt. + 2: [2023-05-09 16:44:27,039] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 17 +11: [2023-05-09 16:44:27,040] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 95 +23: [2023-05-09 16:44:27,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt. +23: [2023-05-09 16:44:27,040] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 191 +12: [2023-05-09 16:44:27,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt. +12: [2023-05-09 16:44:27,043] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 99 + 2: [2023-05-09 16:44:27,044] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 17 + 1: [2023-05-09 16:44:27,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt. + 5: [2023-05-09 16:44:27,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt. + 1: [2023-05-09 16:44:27,045] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 13 +23: [2023-05-09 16:44:27,045] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 191 + 5: [2023-05-09 16:44:27,045] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 45 +19: [2023-05-09 16:44:27,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt. +19: [2023-05-09 16:44:27,047] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 159 +12: [2023-05-09 16:44:27,048] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 99 + 9: [2023-05-09 16:44:27,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt. + 9: [2023-05-09 16:44:27,048] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 79 + 5: [2023-05-09 16:44:27,049] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 45 + 1: [2023-05-09 16:44:27,049] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 13 +24: [2023-05-09 16:44:27,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt. +24: [2023-05-09 16:44:27,050] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 193 +24: [2023-05-09 16:44:27,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt. +24: [2023-05-09 16:44:27,051] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 199 +19: [2023-05-09 16:44:27,051] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 159 + 4: [2023-05-09 16:44:27,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt. + 4: [2023-05-09 16:44:27,050] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 35 + 9: [2023-05-09 16:44:27,052] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 79 +31: [2023-05-09 16:44:27,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt. +31: [2023-05-09 16:44:27,052] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 253 +27: [2023-05-09 16:44:27,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt. +27: [2023-05-09 16:44:27,054] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 217 + 4: [2023-05-09 16:44:27,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt. + 4: [2023-05-09 16:44:27,054] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 37 +24: [2023-05-09 16:44:27,056] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 199 +24: [2023-05-09 16:44:27,057] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 193 + 6: [2023-05-09 16:44:27,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt. +31: [2023-05-09 16:44:27,057] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 253 + 4: [2023-05-09 16:44:27,057] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 35 + 6: [2023-05-09 16:44:27,057] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 51 +27: [2023-05-09 16:44:27,058] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 217 +18: [2023-05-09 16:44:27,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt. +18: [2023-05-09 16:44:27,059] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 145 + 4: [2023-05-09 16:44:27,059] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 37 + 6: [2023-05-09 16:44:27,061] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 51 +18: [2023-05-09 16:44:27,063] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 145 +21: [2023-05-09 16:44:27,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt. +21: [2023-05-09 16:44:27,065] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 171 +30: [2023-05-09 16:44:27,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt. +30: [2023-05-09 16:44:27,067] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 245 +21: [2023-05-09 16:44:27,069] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 171 +27: [2023-05-09 16:44:27,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt. +27: [2023-05-09 16:44:27,071] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 221 +30: [2023-05-09 16:44:27,071] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 245 +30: [2023-05-09 16:44:27,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt. +30: [2023-05-09 16:44:27,072] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 247 +27: [2023-05-09 16:44:27,075] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 221 +30: [2023-05-09 16:44:27,076] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 247 +11: [2023-05-09 16:44:27,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt. +11: [2023-05-09 16:44:27,077] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 93 +19: [2023-05-09 16:44:27,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt. +19: [2023-05-09 16:44:27,077] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 153 +18: [2023-05-09 16:44:27,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt. +18: [2023-05-09 16:44:27,079] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 149 + 3: [2023-05-09 16:44:27,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt. +25: [2023-05-09 16:44:27,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt. +18: [2023-05-09 16:44:27,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt. +25: [2023-05-09 16:44:27,081] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 207 + 3: [2023-05-09 16:44:27,081] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 27 +18: [2023-05-09 16:44:27,081] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 151 +19: [2023-05-09 16:44:27,081] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 153 +11: [2023-05-09 16:44:27,082] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 93 + 9: [2023-05-09 16:44:27,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt. +18: [2023-05-09 16:44:27,083] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 149 + 9: [2023-05-09 16:44:27,083] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 77 +30: [2023-05-09 16:44:27,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt. +30: [2023-05-09 16:44:27,085] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 241 + 3: [2023-05-09 16:44:27,085] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 27 + 5: [2023-05-09 16:44:27,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt. +25: [2023-05-09 16:44:27,085] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 207 + 5: [2023-05-09 16:44:27,085] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 41 +18: [2023-05-09 16:44:27,086] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 151 + 9: [2023-05-09 16:44:27,088] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 77 +30: [2023-05-09 16:44:27,089] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 241 + 5: [2023-05-09 16:44:27,090] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 41 +31: [2023-05-09 16:44:27,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt. +31: [2023-05-09 16:44:27,094] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 251 + 0: [2023-05-09 16:44:27,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt. + 0: [2023-05-09 16:44:27,094] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 7 + 0: [2023-05-09 16:44:27,099] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 7 +31: [2023-05-09 16:44:27,099] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 251 + 0: [2023-05-09 16:44:27,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt. + 6: [2023-05-09 16:44:27,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt. + 6: [2023-05-09 16:44:27,105] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 55 +31: [2023-05-09 16:44:27,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt. + 0: [2023-05-09 16:44:27,105] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 1 +31: [2023-05-09 16:44:27,105] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 255 +19: [2023-05-09 16:44:27,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt. +19: [2023-05-09 16:44:27,107] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 155 +13: [2023-05-09 16:44:27,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt. +13: [2023-05-09 16:44:27,109] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 107 + 6: [2023-05-09 16:44:27,109] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 55 + 0: [2023-05-09 16:44:27,110] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 1 +31: [2023-05-09 16:44:27,109] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 255 +19: [2023-05-09 16:44:27,112] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 155 +13: [2023-05-09 16:44:27,113] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 107 + 5: [2023-05-09 16:44:27,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt. + 5: [2023-05-09 16:44:27,114] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 43 +23: [2023-05-09 16:44:27,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt. +23: [2023-05-09 16:44:27,116] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 189 + 5: [2023-05-09 16:44:27,118] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 43 + 0: [2023-05-09 16:44:27,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt. + 0: [2023-05-09 16:44:27,119] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 3 +23: [2023-05-09 16:44:27,120] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 189 +18: [2023-05-09 16:44:27,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt. +18: [2023-05-09 16:44:27,120] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 147 +18: [2023-05-09 16:44:27,124] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 147 + 0: [2023-05-09 16:44:27,125] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 3 +25: [2023-05-09 16:44:27,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt. +25: [2023-05-09 16:44:27,126] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 203 +30: [2023-05-09 16:44:27,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt. +30: [2023-05-09 16:44:27,129] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 243 +25: [2023-05-09 16:44:27,130] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 203 + 0: [2023-05-09 16:44:27,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt. +30: [2023-05-09 16:44:27,133] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 243 + 0: [2023-05-09 16:44:27,133] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 5 +13: [2023-05-09 16:44:27,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt. +13: [2023-05-09 16:44:27,136] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 111 + 6: [2023-05-09 16:44:27,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt. + 6: [2023-05-09 16:44:27,137] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 49 + 0: [2023-05-09 16:44:27,139] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 5 +13: [2023-05-09 16:44:27,140] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 111 + 6: [2023-05-09 16:44:27,142] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 49 +28: [2023-05-09 16:44:27,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt. +28: [2023-05-09 16:44:27,146] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 225 +28: [2023-05-09 16:44:27,151] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 225 +13: [2023-05-09 16:44:27,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt. +13: [2023-05-09 16:44:27,181] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 109 +13: [2023-05-09 16:44:27,185] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 109 + 5: [2023-05-09 16:44:27,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt. + 5: [2023-05-09 16:44:27,189] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 47 + 5: [2023-05-09 16:44:27,193] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 47 + 6: [2023-05-09 16:44:27,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt. + 6: [2023-05-09 16:44:27,220] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 53 + 6: [2023-05-09 16:44:27,224] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 53 + 0: successfully loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2 at iteration 0 +31: time (ms) | load-checkpoint: 14461.52 + 0: estimated model parameters: 2.81512448 + 0: estimated model parameters without embeddings: 2.6755328 + 0: [after model, optimizer, and learning rate scheduler are built] datetime: 2023-05-09 16:44:28 + 0: > building train, validation, and test datasets ... + 0: > datasets target sizes (minimum size): + 0: train: 1 + 0: validation: 102400 + 0: test: 102400 + 0: > building train, validation, and test datasets for GPT ... + 0: > building dataset index ... + 0: reading sizes... + 0: reading pointers... + 0: reading document index... + 0: creating numpy buffer of mmap... + 0: creating memory view of numpy buffer... + 0: > finished creating indexed dataset in 0.031616 seconds + 0: number of documents: 3133972 + 0: > dataset split: + 0: train: + 0: document indices in [0, 3133972) total of 3133972 documents + 0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_doc_idx.npy + 0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_sample_idx.npy + 0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_shuffle_idx.npy + 0: loaded indexed file in 0.092 seconds + 0: total number of samples: 731002 + 0: total number of epochs: 1 + 0: > building dataset index ... + 0: reading sizes... + 0: reading pointers... + 0: reading document index... + 0: creating numpy buffer of mmap... + 0: creating memory view of numpy buffer... + 0: > finished creating indexed dataset in 0.030097 seconds + 0: number of documents: 364608 + 0: > dataset split: + 0: validation: + 0: document indices in [0, 364608) total of 364608 documents + 0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_102400ns_2048sl_1234s_doc_idx.npy + 0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_102400ns_2048sl_1234s_sample_idx.npy + 0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_102400ns_2048sl_1234s_shuffle_idx.npy + 0: loaded indexed file in 0.081 seconds + 0: total number of samples: 169955 + 0: total number of epochs: 2 + 0: > finished creating GPT datasets ... + 0: [after dataloaders are built] datetime: 2023-05-09 16:44:36 + 0: done with setup ... + 0: training ... +31: time (ms) | model-and-optimizer-setup: 32830.04 | train/valid/test-data-iterators-setup: 4290.58 + 0: [after training is done] datetime: 2023-05-09 16:44:36 +31: ----------------------------------------------------------------------------------------------------------------- +31: validation loss at the end of training for val data | lm loss value: 2.641864E+00 | lm loss PPL: 1.403934E+01 | +31: ----------------------------------------------------------------------------------------------------------------- +END 3484963: Tue 09 May 2023 04:46:14 PM EEST diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..59894d3c5032051f698a28cce769c3fc754f65d5 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7882ddb9d0897a6c5314739ad7eca72ad757328de436cb4ef0b7e8a7120920a +size 131963479 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..afd25af6b85ba2307d9e4a48d44debd44a356fed --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e13979393ec4b4e2684889eafe13825de6837572d94e1d525608a07d4b751976 +size 131963479 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4b7fd046e66c7a691648975d83fd41a59b20e2c8 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccd4586ca2789fe86fcc87ab932cf09d7c1e9cbcbd837dd5c661152ae4e1856c +size 131963629 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..df1dcadd75b86b7b7ac5a4e189f86829fc5d54d8 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e5110f40d30b33e75af67de7fa0492d9177265c75fc4232bcf4cc9928a5e12c +size 131963629 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..489698c4f1bbedb519381f60f9f4450ca4cf373d --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:107816ba1ed41eb2b77cddca9fa1bebe0bf5f12aad0cfc1d62f8650bfa04238b +size 131963501 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..79fad3af5a1a72d222ddfac60b85a81a5c82046e --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b19b67635b1842a3971b217bc8dda3e273b146b2f68c2ac892c1872c7ce88ec +size 131963501 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..debb3768d22cf19a28859170953ac1054f1c4d88 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4d44f4b4c3a22c51b59b75f7d7a5728c09b0538dbe00a5f07360a69f4dda29b +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..60e505a546e27f795d08e97597cfbbd35b7ea70b --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1196a9ddc2ddd8eab8bd96841b4ba1bcb757b7367338d33e68c3f338d328ba9 +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c556491545c35399315b76e018c795aabdd422a2 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dbf3df0eb58ba0e0f5a853c15a78ef0a315dd4b155cafad2541d9eb93ae25e67 +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ca3cbb999985d45ff244b6dfeb139e5b1d06ce2a --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe5f85604bda73ac4214d520aa6cafd3c0dcb07ebb8ec37f6b655ec3e7feca3d +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..41f76ceb3e6e82b24a87984f493ca9b6cfdd439c --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e3fe1c1e595fece9ab642b160b0f5d534a31ebe9a2505a83b7a22dcb7efcee5 +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7fe6a00480186416b1c4e03e6e4cafd588ffcf8e --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddd025239a5e67efc2b2c55e370695af250396c559f9902954c94e6abd611633 +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..15263b1e3c2ded0f1bbdf76ad7417b83262d9c13 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69299e174f3488d0580bb381672ae453c3fd9440da427aac509a80a4963ffe40 +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9e290527768ec866282fb56fa7f40b5201134b19 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e051d4ccd65246f5f752b8ac56469d3160110ee3f6eb1eb32803b00c56318a92 +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..62726cca045724c7e7652d4b6d4453c955e30d2d --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42d745979d69e0e69d04d3e586b467936ff2b44b8712e0762213a94ddf484089 +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8cd4c6e1525b6cc5ab4e10e7011a25cce96fe95e --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c78afc1c0518e7eb0a871c8d06d65b742ab4db15292054fe13d3b98656a7569a +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c85cec796e681e53c9efbe4066b44b9b06d2d43 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19e00fd0922530983c27c2e14a009a0d508dbde59b75be3fc9bd63d34e486e57 +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..998eb2cb12257c4bd836260ab0095215c1d8d27a --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58dcea835b43388b5ccdca3bcd003173ea0fdb0f7289106ff389ba583c8e6ee4 +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3723a4ceac27488e5cf644dfeef032e4cb35ff6 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c2ba1c8105304c8af404aca2ed9e422a825958f93232b9915b083b734af4bf2 +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b4a33b5cffd58d64b34f2e7b6af5086c8f65958 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d327b1517a9e53799494f97b1ba9fcd694bfc598a64de156efb01d619d93dbba +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6e6367028dd3b092b46e2fad47a791859acacf7f --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b27cb36798d2d6a5b72e4d226af175a0f27dfd13cf43f4792f74e03da11d1340 +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cfdfe8192a865801d86d60d6dddde4c9fbf45b75 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9063ab3fdc4c59861919efabe414812bf7ca72e604d9cb2d9e4f6f5acbfc2b39 +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..028639e1e91e3d423983e389cb9ef14ca2c10fd6 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:beb015b9af3529fa1b2fe3c9f1a6998503db7f3db7f7cff88e115cd9a7673f63 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aee5bc9d3f10910a16b4161f7b2d094daa92c559 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:793250b2985c5eb271ee7b0db86530cc41ea2d35bac50871b8acb8a473acd842 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..82a99bacc72abf80cb84379ff0501c3491bdea21 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d67ff38daef307f35db98df753c44783828c8db7f71017605b75bcd1d8fadb7 +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..61c9ccd48a0b8ca58873d1ce8a1a2e36b705aa9a --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9590ba3c71cc1a85227486a4f979050a479fae4d9503de75d6d385c729d1a92c +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a64828ba50147b405ed8799a6d737352876aaa8d --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac388b87d402473e975ca5a1d62cd99a2bed3c31f4044cf6a70db7d48945dd00 +size 131963629 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..11efa13294ad1955810901bd64a28b6bc7b1b5bc --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0bf53f27c3e252304a6249f0f7fdcefd76ef8881a8f5b345644eac651dc81300 +size 131963629 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d51191d66e3524b2a4806c9fc6c9abe4c5d27d0a --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31e7040c8b1e3e76fafe7b935dc6c02b18a708f4addfafb39d7d593faee5cb60 +size 131963437 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..91bdde7fcd011e18bafc9201d4674da5757eac45 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8c555322a95f8487f9f8dd541942db15577810fc2c1f65d0b64a07406d10474 +size 131963437 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5717c65a1ac757d2bd19cf262865bc70223afae5 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdf7cc75875bf4e08bcfa31a501c621a408cdffb14ae29aa0a2f645eeff8929c +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..113e7bdf8f7e589201c8afeabfc358b90f7b3462 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3994842adf4aa9a3a5147fb08be100a1ade87def3cd37e0a2005474568fca9f6 +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa3ab249564792146154023fbe9af283c414f1ed --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:380426f1c7cd72b0c4af7e97971182c5443536c7aa3a329d35bd8f4a544d9ced +size 131963629 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b9380fe12acd055dd2a5781d12fe4f673e864c31 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:507c692ae79e420d5b6d0d45f110b042ed0ce44fe2a7afbdc250a034114b5a6e +size 131963629 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f9b742034bfd56bb20dab1b9b955251b96ea321 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fd4b77a00b6500b572760fc7ada87ac1ed324827cf153274f6e5d174c7df7b4 +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ab02d01e0f2b8d67b5e1b3d1008213c90ba6b10 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a1123b7cc78155aa1058b6c097f82559b061122f105f7b234b7de0bee27cd29 +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5068cad7fc19bc92afc17625f07762331d62dedb --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dad6c4638aad1f8f0dc244b84e3eb91bce721eb736a96a6b830713c9032ba9cd +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec568b0007fae400f88c382b15ace9f6cf7aff43 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33db9c2ee879a5543bf8f29437ee0c84d0019a57a3d08b0fe029942e0c1d1445 +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..45f37f211a5caa4eb8c5b84eece825a7c505611d --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6003aadde7651baa892654e8c251b48490da69338e3db8222c9be1f00f6a7fc +size 131963501 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f6cda9fb5121f88238a208e93e3558473ee40351 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee1d630d060c9b3729161e55d6c65c37e12861819a19e9a48f60ca6a5ccbb0d0 +size 131963501 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..82e6eaeae9fd3ecdb373b6ed81289bbff9f0fe9b --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cc2f3613a2e5b9b76a299b578ec47de2fbecbd3632b5ac0a7ff649a11b41cc0 +size 131963693 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..02414a210d3660e8228eb9bce226361f93ae29a1 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c666c8cc8924638b3ba68bdf1600d97b67c63534e7cb14b4ab40bdd00296fb8 +size 131963693 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fbad5d71536c0694748798acd46e04928a7ef004 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58e22c052523f0df2bf9fb6e831928135d29bc3ac023f647c483bc1875bbaa9f +size 131963437 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..399ebc93f7ff7360cadf48e2037158cfd8272a1c --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09f31815256efab49255fdb258589d489034f3ce90b4e411541f50fd13162a3e +size 131963437 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3d659999e8ae74ae0f84dd14cbe6deeef748a54 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8d21ff1b7e445c3299bb70d4e340e94681d1e8439cb152cf039b4c1efd789e2 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b31608a7ef4cd87fd89deafaeec20fe4a90de96 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a290a8167498e3fdc4eadb044f72723b459304d71965f324be515bbfd786f0b +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fdf1edece73e3f6fe2b2c46a2cf65a847029ba01 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:522a5e804d1e441c8111d5d1fb37c14d95e3026e2025710159d23677ad28ba4e +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ac9bae55a6c956d3ee8491ef397ea17151bfe5a --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f76fa209b31aac3d62a391c968e6356d3f63207cb7792cd15240c4abc0aee1eb +size 131963565 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..08d3737e8af3d853b55191838af3767c65331a15 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ab2890e67477aebe2a5e55cbd06ea7183687133cf2eccce633ecd4cf116fb65 +size 131963629 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e6fe3389a77c0169957ddc21c5d48d5c48624268 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9d6204701c42c62aedde6919b0741e98c7362e99d64b70d3985f49a590dfea3 +size 131963629 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aa7dd630b991ec28e1d19a32dc84022642ce277b --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87bdf9d34c8fce6394b6cf99869be151a8b08e6b47a37f7fb92b185390b6e70e +size 131963501 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..36d701285dd3c6ef402ebf91a0246a20752e631c --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78ab5f0038514a8971e86fbff49bb0414a9e1c4c322be615b02666d58dbc5556 +size 131963501 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5bea03a18247bdce62515208318dc28938ed50a9 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f5e4dacb4c171947794b51bff12a6487c4c186ed15223be161c37cd579eff39 +size 131963629 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2c4c60664f85997ed15119d5e20c4a0892a2ecc6 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd9361a9dfb61de4bbd4ce88707736cc7812f96f941c34ca483fcc8e919b57bc +size 131963629 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1e92a0c2e386226b03407041ffb2b0b82123bb24 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d260f10d1e93d825b25eaf4a141a8b972e2ee2f61068c6b430b0fbc689421c4 +size 131963501 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d7b7a3c7fe21e344ace3eb9e3c917e9535b84eed --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a95d68342fa0d25e12559521b1e29456a730b0db7fac72d3061ea821ea883398 +size 131963501 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..58d7cb5609bc8af3fb1944a6a7a963ba055a1f2b --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6ce2e4112156899c40e3cdc03fc08712269731ee1a8a6103561a8af5fea4c06 +size 131963693 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..726f5089551614c0440f2384bfc0d00d0ac58c64 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cff01b5ed61b859b92f67de838bd0224f658c15c9edada1d6457fc03db7b6a3 +size 131963693 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3881b225d012308a3e7f1469a6cb2cdbe61f7f8 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b919ff0fec33cfd52dd94d592210e134b43327caf4e72828a7ce3b6c6775864 +size 131963437 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c8da9dceb48d20bf303695c180d1ae1bbef4258 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9609ae883340f0d319b9cf67c834f418e2449e7019316f11edc486d9b356244e +size 131963437 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ceae1b5a3d732caacfa57edc70c9b3c24f2f375 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1f249386c81bdec55d449b3b690a89b906e5a48c61c1a9bf4b9b202d4f42a79 +size 131963437 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9a2633c2f6703bc2263c6d8a26e2ff7bc07bc544 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:308e1f513c96b79cb9ef1e8f43fd157bc543edec2050d4ca4db89e68f4c8ff2c +size 131963437 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..73f3511ee43282fd23a25370ccd963f98a3b0214 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:363532bd45eb8f9f0bafb2ee4bc94751e0a62162e588bf5fdd28f5f17c5ced1c +size 131963682 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eba24b0f5997944cac6252a18a3efd7bddd41941 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1fd9b3a707b38555adbf21dd2e065194a57da026c22bfcaadab9dd1aa6607aa2 +size 131963682 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cadca05008309099d2b1b3ecdb34f3d2eaef02dd --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3879d6f060017f80bbde5e82d418844c558d2fdf31506933c91bbc9fb8045e17 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e17cb7496c44a86c7137e55171864390476d1ef1 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51818c1eaf5d0ea0c41bf61d1403a7030054da334320a6920ce289c917215d6c +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b48dd48d72d3dbfcf75cebb8daf75a5b1834c1e --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2e32ce26c4f074fe51ea697334b6331bfaca86bb73b200342847fe98b26542b +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a28f987cd5a32cd341d9412d419b8af249445be0 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d4d41d736a2518751bdb14c8aa1c5f326a56990be11d32eb7b461dd4ea44651 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..959d925f770b78b53501f39ff06043c7057ac91b --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a4ec20b7256eedf1e592be96cac8242ba5de6b859bf4d552c3482cab876e287 +size 131963426 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..666537e162e958455be37e62ac37be7bb3e180cd --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b0c31842e1809bfd342bce75fb7eaa27ab5023bdcb615364f636f1c1842cdbc +size 131963426 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e45d7f5acc52c1fe05040b1285478b3a821f5b99 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c2c15abb92206811b03a61d76827e124d516734a24e06c4a8875eaf66888ea5 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..37d1df0d47d6fc47cbec3558ca41d573a68c2c5f --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f94b00eaef841b3a9a3f8ab38cbb8dfa0a78d9f34b11fe6bc5caa9563d6a5fcf +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..89c289ad24317407f247c966d257ec2c5e976822 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aeb5df651455af6bcadfdc9bd1e2f3a2d1362f49c52ee541d0174a92cd6c840b +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5095c4883af9f59267b494b8e36d8c325bb70c63 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:596a06178d5703dcdae2f970b3d9ec15d7bcdda6e22f7f18ba164e1a7cb689de +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f89cbe0523fd1d7cdd98245becbd23369ca2b54 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffca65a9822d0437e01100c9caceca217d8203520f34385dffc28e544942c2fb +size 131963426 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..741130ffa60dcc3512feabb6784be37dfb90bca1 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e696d113973db868066fbacbbe9613b494c3d4a6b71303588cd56526d8f9c8b +size 131963426 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d45b9683b9c66cbed91902d8a1a5bff8de16394 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cfdc225ad3fa480b1e1ca6dde24d5e03ced942c29484882c595992d778d5c18 +size 131963682 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ef1fb889952a22a4d10e4f69e3eaba143be8e3d3 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb162fd6bd63e67dfd94dfc78c45b04ea0b8480f60b970c129b4f53080dc30d4 +size 131963682 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f83a75dd46a277e1eead659d42352aff5fcfe3e7 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86d48b4cda20e675ad7a76bdcfb57ba538cec81ea4bc8cb7252c2977036dda21 +size 131963479 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8969c1d44da751dc9a7338c0b6e6cf78cbadeb1a --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23ccff9357491249155d9bdf8430edb9e1c9b3fedb460f6534eb6529067368e1 +size 131963479 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa242145c4f83494c2c9b4236fac003f4f586ae5 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35311d8c55eed7fc7b45fec28805bc7824daaaea38e3017aac173a0d83014eda +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5fff75e03fcdaab7ce81b82029039003925e5de2 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88a8a92c2e368ab2e442c77b7be9ccb9c88cb9deb7f335004820016517d14d0b +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..18d7a5c0e5092fdfa3be1f8b6031b64a0fad56de --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a77a854403f9a801541d4ee27a674b3fac0479af1b3b0b7f6b9b45732f61fc1b +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..92ca29eae76239bee8f110a0777c727471127e07 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9a72b5c4591e9197d07860d243578d520b3a2b3a9b58372f547e8b97427729c +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c751044b5d878cb0738dc45b876056d419f2dbe --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:238f0ae7e17a41fc18f9aed0c8af4ec89119b2dec04fcf745e8ec0821252b24b +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..586e7a934747e9de226369a34416a353812fdde7 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3f5f4f9f769c63c3cb67c11a9c1f5bacf0f10837761c78f6e9e30e9c862910b +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..af18ae23a2a75a1af88fa733c45943d704c1979f --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:924ad3bd89efbab0a60d844d7bfefbad9f1f5107f53107fc6a68fc44704d9345 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec3e628ae82b98f9d2e995afd9f569ce7433bbac --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:300845f024848aec1d5c579fa173faf91e9572f114863cfd8d8be1ee333a271e +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7f332b0f3e213c0ef04cbf782ecaa5ada7595f7f --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aceed87a479d8b9eecf7da8e5b75f61bb2e7ab93d2a3310392c21e41ebdc1f56 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8eace85b67f328d695efc23aec1afdda79168cfc --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:059e1d18c300dda11821bd11ef0747867feea74a74efe4cbbca9cca2935f113e +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..974dba6d05fd73705b9b5eda1aff1c5d18be2609 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e405e2c701b35a87923894a4c901cc2ed7f97ae95e51a01f0bba28c996cc3be8 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a3646fd279d37bb7923f4d9048f90f4583809a3d --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d7c484e680175b0371df1a756d56685c373989d014766636373806e5e1f8c2c +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ced5d1b6b4c94ea1787f86872403654cf66c785 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9ddfd65c34f9c2cd68cc02340dc44511157ad02f36667ca32bc248ee9312077 +size 131963682 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dfe6ae39c6e9b6d264b994287c573ba253635872 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:640a7e0c2c56dfdd39712c7ab17b82e9017769aa45b6b107aa5a1acd1b468eee +size 131963682 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b0d7f784bfd89a94e990f80c563bfbf82089583a --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b3d5ec1f6efec3c2a3ea14d3851389801151ca7198338184cab6de1fad49d5d +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c026be80c00881b7ad2993a1f9198535ea086b9 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:658321c211e6463b45dc7c0487d1790a14a578ec8d73161b78a6714872f87523 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d263e88c2b818558ee97bc2ff3ee397df33c0266 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:823ee8b76d90b47230f316c7984bab63a07bf09c7d39827726f0fab0cfaa3001 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c30665a89c3f38babd1291a07cdf825e47608065 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21f66527c4b9191008273e0a1ab956ad9c8c07b74294e64e32d2835af24297b3 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cdf4c7d86c00ddfacb8f6b659ebe76c5bca944d8 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d4a9d1ad7c1bc1175b475584586eb681428ed49170a41397c618d6f21dd1ef2 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fd205a2ea87fc83b72c71ad0a00fd60e3b9409a4 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf5b2aff33bd88e2b506c2d2633af24a63d25b09ab68ac1ef4f95e7b844ffb8b +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d3aeef6ebc32948a7ea1a025fe92ae488d86689c --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b1e5b81758e23b7b4f0b71a34bbaeb34aa66109345de077c9a930303e7a8971 +size 131963543 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c58b8f00379edbc8ae280e087e4e24c777b7b6d --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61ff826eec6debcfb5eb0bf29f2c758ecae2f0e27b100d425e43c4dfb283ab10 +size 131963543 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b5f71ec7aa1027656136c41dab77100d1267fb5 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e01de2c53f858cb9e3f49f4e690547c1997d1248059b8dcd97a7f266397bf7d +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6d79c85856cf4457620f6ed22c1e4d5728304978 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a45e27d09604b92fb4a3d3f5394bd1d012aed69ef9afd09bf19d811998d2fef0 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..efbe951de1c7e0793f989f3c441cff5059319271 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aba7d9499ba72cfa24c48a06b1e13436601693d3e4ae66b4738cf8c7b200de7d +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..914d8c59b7fa68bda3f668803647a8c852ef24c8 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:25edddf5da678e152d77cf20b608a34cfd9118451f27cd173457db199824171f +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f94bf5bb44091d0bfc93135d7e1e7a7d8530ebb --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:014f9082157ceaaf1a5752cd1b1fce3561df0340526096256c57e3207a1d09f6 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..492a67e510377b54aa05751e8c692721bf6e8fb0 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7613a94ecfac7bbef59507aaed525b7a503691aca7e1a6093e8c5caff7a19e28 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..639fdef3e30707f2ff29a066d6026740004f026a --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5e72b3a40b1c2ca6d1a726b725d169e70e7c2c2533c910664ec2a67901c47fa +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fc802422dd164d2dfba3c871200971b8866530c4 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8a76fa8411d822de027d6395e6438e8db7a0928811983961e051eedc2a2ace9 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..35b0ee703c39f99251e0b03d74a7d40c814d1aef --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8125ed27200c038f75c1032d1269ae61933329e40d643c00bbf67d69da804856 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f67ca48ebb317808266f988497c786b8cb5406e --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f618d30d54a73b395cee8c8460d4ae112ecd9b0a5bda534ce31308f40a910f0b +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..176ad64b2ef1ba172c4222d7bc9c8a23bc79f34c --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32553d3803372d05de6dcff10940ad770be5288fa2bab44785ad10f5ce44088e +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8cd59f77076a56b251fcd80c3875d533b4c36df3 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0487e274a0bf234c5be1142a3843026578d72b1d41f1ea282f48baa601bd271 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cfe7a103e7343f259f053d00d318c5b650616f9b --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:777b21744c9284ce3f4e0d7abf219e5b1f2a594ddeaccefc4f89fd409cc5931f +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bfc1feea772e6c8a2c6efc938660586d9dfe11b6 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db18added01ddc44cb6c17e0ebda4017e00afa94514442634c0379d8988cc16c +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..31e843f92964b2f5d95d12445b4102b9eed2e6b2 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38315a761b82ecfeff1c49ffeca504442fdc01f91fe7fb1deda41a33774ddac6 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3099af7a8b60122232d10c8d638c2a307224e60f --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:907553b8d338ca3c702d7c4a2b1ac8a6d9725a9d2ae7f7289b4c68f87e42601f +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..be7956136df5a80150a545ab0ee0586ade1695a1 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c9a335415714e8ce3e1b45fa30218de4b3d00e871ab4274dc5a0635e1c2092c +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..581e3bbf22bc99f5759ccf9f913c89afbd863c66 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8af148a454e3cf53fd0667d52696dd4f19db780f3c71b37e1bafb49037815f78 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5df1996d73b114cb6a2d6e112a9e50ca3ab27408 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d104fd53e9fdeb61976f377e1a237eda58e0733db45f613570ff4b43a200f0b +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2d227eeaf878e32881334f6e88bef531391fbd2 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83c0101016e8f214277d45ab3bc6ec2667e318fb2f658620a7aee7d898610616 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad840c8216a7cf1755707e11cda3a06164b7b313 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd9dc5248132c44b61d7794f7675850c33adc9035c8afc3ae4d53c100bc79f63 +size 131963479 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5dce907585146575c582fbb2cac950f58327f748 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd200be6fd09611d75b8dd0d927b3e96461a02003eed497403646b415889bdf6 +size 131963479 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b32486bf04065c4eddcffc113df2b98db12bf8b1 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed5478ee2f7c49e9c3c877793e516f49f89db5dcf9005a7ba8a5feaed374bc86 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0e99015a61d7c574c86fe46e74b23c8db5d97515 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ecc1c5607753b4d7171d6685259afd92fb1355ac5515ebcf90520a95876e1de +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..84e8a32bd35720427367a97d00d43f2c3a7762b6 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6a3f8925c654d82de35f27726dedf94c3584dfd7d4c9afc8e883d9fcef3c8f0 +size 131963746 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c9d100ba43eab445da259aefb92ef4de6174cd0 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6bcb8921dc570f42de1ddc4f59f654567128e643e97fdacf89c4edb4a8936aeb +size 131963746 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d9ef85126202ab06ea3e8df36eae5c44378e606 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52cfe1cd2f943b5957ab68d563767f2b6482568e7a426cd63776f3c1a32e54fb +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d603a2276f12aa97beeecd95aa00c346fa0c458b --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6bba70b0282fb82fe73e6bc824fe38e4e31f40e041ee9ac76a62678ff739cb2 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b1c1bc4ca611dc11e7e7d414150bd817882646b1 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:164072a2c2210e3140f00f32868cd0666577f9f7276f761978d7115748449a3e +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..92cd118699b97e92274f169c133e98a3454f5ead --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a3ed316736e84cf2cce7a7fda1364a1342414d6da03692a9e18e2571d02f582 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b4efbc7996cce15fa4dcdba203c20ab5f54aa324 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6a40da2823f0ef7e7a8b5b88ddf9775057ca61db469771dbd389ce836ed7d155 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0ae4ac64976b76a545d0c033af669a3f1a64df59 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7426dd9f78184da9a2f01a42364ebc5c7c8f94930c5af176d6da40792e70dcf5 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d2200ae340138f3df7166cef075b3b400f7963ef --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9233feadc425bb27a45a90710718fcd4b1325c76f7446faf346ce8a57b3fe218 +size 131963426 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f1d125ad8a2a4d75730c290058e64c724921ea2f --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c710d49c724cfc1b9e21ad5d8be1968f20da0aa8876e0f2489b28c24f9884c0c +size 131963426 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..da953f63a10567d0bc3f3b7ff912a98e63bc5725 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:beb407a6af8927c594487973c230cba535ec145d07e254a6897324b5ca1834f5 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..186ab0f983654d62c86b1bfcf4c816387a87fb60 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d649d399815a60f523ae91d172f6f2431650437b2593eda93319ed2b60925e3 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ae28f5c1dcf7c35a0a99afa3bfa3be4ef19bcc6 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d915b72f352b34ae83094bd37c1ce93906987221514ca2232e207b8e1c9a3e60 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bfdae96d67bb9245434442078dcb3684655ae194 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a73bcbb90f4c0914bd6e767991d1df4821c002ab5c2019fd32c31fec90a4a3d +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e630bdbbe3e6f42c931a7b2cd134bf1de7193647 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec24cd04ecb318c0854b104e551d1b8557856426353f6081c6d5910937ab6971 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b454fa7a4462e33dae26a1b950ad18e082888c0d --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66ade21ee2ded462e244499223f1d72c20939746e6da52514ffd01adb68532fb +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3e0ce83bac5137b133ebb6ef1dd0d04746e094bd --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e00e24ffe4366f30fcb36e92841765a25c049ce027f74d98a8aa59556cb26f7 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f395e09fe091e69fa193f089fc51a90deb25b0cb --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4362454e580d9d5a6820536a3fe603d800c23c3d7156e04c931d2a8405549049 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8bca0334ce14954b0460fd17446e29aa9eade3e5 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28a053c1f172f03748e524327018dba866e3e2deb2dd4e477ab11815e82d9411 +size 131963607 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b41f76533ddda6cd02f9d645815890d0db4a619 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5fa41055e76da7af5bed6f1346110356464767d36e8fda8de0814aca6e2e31c +size 131963607 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1265569c2275852f7ed94980d0dd2c074972cd78 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a60e3379f667082df01cc85ba7e66715c461f82a7e43e1e655c9ae5719310c8 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf8b768235ad43dbb0581727f330badc0b083873 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cd0bf2273d8769cbf7f4ba2219efff663b27802d49c3155c22c5c39e6c3b8cf +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b25f9781500772793dc58807b4eaea210dac97ae --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b9a8146525c56329ccb9878afb137361ec6ef546f42204188dc8ede26ac1323 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3642c51601a2dc90517fe3640bfecabda0df6911 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e2c6d2a1ff844e640373a2c026871430a5198ae225e858eb3d9db349f3ba56c +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ab8f04ade95d91f169b51e159b8d82b667fe7b67 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a1171d256aeb211b66b73b476b482230ba78a2daedaac43a1ce20e6838fd7ed +size 131963426 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..defaba34326cc31185f8398a71e2f82c291f458b --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63d4963b7ee53e1a258a5e699c385038c725bb124d277d1624dba8baf5c4d987 +size 131963426 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..50410f5f43f7f649a134c26f19b826b012f499c1 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:112dd6b7a5ca380b14928cd3e384fc631adafa00cac326497c1a127d6146938c +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6cef34e6e2b26517d60e4eab691f2b248c22545 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cc8ea16a888358ae23fa5c5a054e76f614b84e16d1b01fc23054a0cb803962a +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ceafebf56d4a8fe47db6b8fbf768b034339f37c6 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a3602c1b48a7cec53975e84068b8490ffa6fa6d3cae25748d7fc62c0619477b +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..963d4ac0ddc8fce4f68c792370a1b208ebc4520f --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bc7d4d5e53d7587050e9f8ce72cf14ff270b7a7a22654d634bac10d81121514 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5581012c7105a46c9df629be1a3f17fdfa51a39 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5786f30f8ed933d0077368b78adf8f6b7c774d547c6565e43f78d34a3c235da5 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..17528c2b33cfddb0783832394700141ff8c45fff --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ffce3fb1bf1af706c9406444313fe539db33cf7fcd763951a07a63c016ac612 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e6ee5b231ade8c13c4e3fbe8b32aaa4c5d726ef9 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cac623f60aa3b1ac491f9f61ca6863eeee495821d37a9383c091de54e9d10e60 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ca11f61e00ef35e9db349328f42344c50edb91ae --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c38c85c21134facfe63b0f8bd2b99cb93451c8a6cbb20b35f670823e4c182bb +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..faafe4ba41ad2b4f3152677887aa43b625af60b8 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94c5e061e93375b0782b5ae5bb0c2d4cd8c080305faff0c261ec1773d2215845 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3bf642798c5c574bac2ca6d5574cfea153250166 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b70d5d346c24a9cd30c37dc0b2e7674ca6113265d4af76b7352a20806155efcc +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..de30d3f16fb4424f9131522104ae213c89424bec --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2553ee302831ed9157702ddba50eb8d7ee443750a319afb22dc4ffce895043af +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..38f85a60bdfeeea685ad9c0a6ea4e17b2ceb6bfe --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dad82cc129f478c5ed70b3e8c49c1a4dcb19e5f0120b06d137c737f7a118ac9d +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ad07b063f4a4f97c1036a8f1b51e899ba9502e02 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a18a95dfdb0594ca958303259b8294be9363541dba5d5cf076fb47e489899fa1 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..248b5de4c964bef7627ce421e368f8eca99c813e --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aab7edab1e499e599119fa0304deb70e3ba3ab13b97f1271f59126dbe138927f +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..691f1dbba8d0b893b8d15988891896976c24609e --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89aaf99bce193328b26ccda1ee9a3e0d574a679ed13278953b35041b6bbca42f +size 131963543 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..598639c5732a85708640b3f7069cc11be97b4f96 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:102f32f61299663747e7d419ceabd9da7029b360dfc0c628159979b2647a4dda +size 131963543 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c4bc101b703fabecfcd65e4a6fad69f58fc9f603 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aaae65c1faf10e496cbc324a35c9aae3abe72b9e424137b1c56c247b54fe06b0 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..69d16d6d960dd7383133085bf9477caab25673c1 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6497787eddac1a00ca257459cc34621150f0fb6269bf69373acbacfd968cec02 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cb332201eeeafed2107fdddc8cf8dc6d3828ebc3 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86e9b9f565897c01cd4929907e313ae9d54ee27bed69a277060aaa1930ce9635 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b6b14e1705af19b6caec90142ee12002627d89e --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e156ed60dbb7fd3752a6c2682b2473cb974f0e05c7ff26298a291ae05727fc7 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8987a0dd68d37cc1f9b1b2e1f3199777b57c7ac1 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa5baf06eef44e6d10e8c0db7e0d0a7265e0768d2be6879697857187f3303ef5 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..63e300707142cc29b30ba5cca35ac52cef13ebaa --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4af988f1936ea87b28be1b4a0e2485fc2afc3978b9a637f41aa277e98cf80482 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..620df8cea162ce2429e8ddb49743d98fea200200 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8fbf9139dddc45b301aa11405c0feff90978951b33a83bdf8bb4d04e95c1209 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6b8eb998378c304fe9db2f333f79c25661e7aa1 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:943ee7c3a2fa29db1f965e1b57d52210e7f0b997ea1f22a72578a558a6011085 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5d313cb0537f237210e6ef89598c37f178e7668 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4df9fe8806115db446b5959ed44832d2726fd12f0224aaa9690226d9145aa1a +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8561763460f7ba8c0cb3f6261eabdd1b5a22201a --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfc416dd4eb1c4c107a1008e5da313910a319628bc0adf9b251189d8873e9f85 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e52fc9eb3b73ee3682edbfe8e37990ac49eabc38 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79a979e216c22dd197fba7d7c5de94a3a51e6597a86bda3ab25efdde00470bd6 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..108e7f5f498f7ad0f26d54380616871fc5c0c87c --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:640318e956130f2ebdea5639690794436cc9f44abb4f4a4aba4c03de355cc5f2 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6bf5ddb11cc779aee61b5e9ba1f58adf9548ea4c --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed8b0cdcc80220d62f73e70554936ca55a0fb6bccd6ff430631dd8796372c5ef +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7137a20585d126f6e0c931b4650552100ed7127e --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cea3f8ca40de84775b6d568f804cdb88b3aac3599976724b0f93b8241563be6 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..94b5d9d275f18aad7f1bfefee182529cee207cd9 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38e042ec2fda4f328b29858f4b4fe12fc08d25ae283e829419d84dd7990c6aff +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3356ade53878453be3e56e850404a8834901f171 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5e7e10e18fc6a6232d7f096e2c7d3120c91d84985da8ebf64c6c9c55317ba02 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..98da44b59db7239df58ea3d18c8e0c54eb458f4b --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35ecf885c24c878e7fa8d065e1aef44f7530c6e6301732359956aaaa793bf503 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7548d6cdfe8faef83b3f7786ca55e8f373e5797c --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54d6abb2e0a444b4a736a01ec63bb3fa3be7ee968ccbac05440266a7f49ca3b5 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8610b400f5f9fe2bceaed5873b2b2b8f3432c875 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1692aadce448edcf9b271c8a2a8df10966a9288cbd369f94f99d473fe5861c38 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3e1a1eff94e5a5636a1b8ab7e0d28b294e0cfd15 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c381c8de3208ec180599c6428a95edc769ccb6fea13ec1f1c7affa5136f7ca0d +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..31ec34f59fa6cfac8bf92db78b746d73c168fde4 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cde135f03f0d3529bbf2e6dcdaa5e8c143a381fcb6361fff69a8f67fa757520c +size 131963415 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3bd7c30c7c4f59f9e94bd2fff8e0b019f9148961 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2ebce48a664ff00080fc133ff2493c3b48d42484de6004a645fb1a165f4da12 +size 131963415 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9cee66b8f926135692eeeb7f6df625adc943d629 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab76d9a955fb74007b0757a63d830f9b5a5f83dd3a5791db1ea80c846e9cb075 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..070a714ac2690cb5f4ac5f00677674b3f9dd2edf --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dac24e1f85accf2ee733b8a4cfc92f124ddd599d8534108a73695bd081f5cc84 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4666c628509baefb8aaf3ff04f8ccf1329ac0376 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:119b60cead05c0e3c30a394eea3d23bb05f02f1a15fc2236ce33482aa71d4d29 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..808353bf2e65f42869d14e1270381032e3012042 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b6701a1dfa6a6b2886f93d88f5a5bcc5c15412865ff48bdec0f51e70ad5b32c +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aa1744eafa294454b9f999aea7a203f651745651 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e1417334b5d164cb4b4bf4d8af1542217fc0b08d9fce2fc14755f32f1298502 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..76b1b3ac17c3d402f58384f0fd51e3dfd8f17f6f --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1a8bbdf52c3bb8b0620dd0c274e6bb1e954e6406f8c0726e004c354f370094e +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1e64d4ab097e88a697928f723856fd0921751e31 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42724e6ce1835064a2306af1cbdbdfa3a385e124c731ec700b161245affb585d +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..23a9da37ea537a253103d8dfd14f9a9263d6e340 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3eb76bd9278ff18112da78d22156ed6dca29c510a25a63b73adb765014b47a1f +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa56713b5410c69b037964a1c55753fd6b5282e2 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21803151f35af70716d393259f7d9912faffa5cb25b508dbb4ebd91b08cb31ae +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2bbbdc73e2ceb634bc5b9ada495ec54a07e6610f --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0283200862804145c6cf420379dae7f361f91c6d9b328bcd815ab7cc575398c4 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c8cc71473f278c6ead7091c03f0640f7f49ec35 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff7abbbc8ad0d05c0c831dc3f8f8f9e5444cce0ef993a0bfe675d2410ee92c7e +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..132a0761b3b44e6278f67c5f3abcb6c8079d4839 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88337a790c2a1a7c1e72c14847c0c259d845f343cc2e06ddd9f24eac1ab3b388 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f6c0bb6838688bde037641cce1be8de7ba34f348 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c433c2bcc7385fc84eed0dc10084c14169f747aec784dae1ff972adca16c646f +size 131963682 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..359fe3c86d378a78c0feba77e9a0eceadbfb4b39 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f78d335ced354d2f69663ee201c976cd5fa2e64b60993d4cf90a56d9c924f9c +size 131963682 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9b2ab1c8ba5aa972c3379aae3e8f393e4cc7682e --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:923f52824e2b57e2b1b5cbdaed3fd88b03ce35f9d88a74317ba006bf62053ec9 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..00e7fe22235cfbb5553acdafe42096a86c03f284 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76ec7c31076674ce226c09a7de50f950141d537fc00a6b5801b50414d0a716fd +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..98a5d15236938acde4fcecef083c1653dec54776 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c935af6c6c216d2a19bd6dab73c93152ffbbcfab8c414655669398d017b7aa72 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..19f39ca2cca656ad08d76e40f98e97afe1672567 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09384b2f7f15ee7f9baae9df1eff429cb6b33ccb3fc78fd15a3f2822349c13d7 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..758b55078ff2ec1182ecdf1d840ebaf1f8e9d59e --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36b8ebcdd9868e561df8a59fdaa3df0719d622a1856314b72bdf95ed4a7cc5fa +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a545f377b0568542c633519c50bdd3dc2d7c37a --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:024d5343a2099cab16c69c8cb27432092a8c3344efc67ef2bec910240b535794 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9343bac297d922fca24c827aa062ba4cacb2bc3c --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e25d2f44fb053add45c2fa795f7ee8369782e63dd120c407e783405c472487e +size 131963543 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..403d501bbb59377c62314dd51fb4cbc5880d66de --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb233752893e85d45aefe375831cae2471030362f12dd97bd13c8a59e111fabf +size 131963543 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7ff679da97bea0e1bffe0bbc0357772946860a89 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f43e8fedf264fdd0374dce77c7f9b8813cf4a3156ff19aaad095fd35423cfc5 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..92faad3a7ac660d14e78c4a20f84d47f4992cfa8 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1f2dff9748a3ef4abd09f59f698740cd9cb4bc549c898f98665e6465632f507 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e796f1710a81c5f326aaa3b12e19fdb0ad184924 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6437bb7697537fe08ceb569398a8d67035584a4bbfca3994782bcc253b9fed5e +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..86eba63a384450f94000f6016830e7b7be0569ee --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94dd5c0e04a85f64c316a318a707321a9ce4c56127da4186982e1e390e9076d8 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b4d5847657b0c5e457f44661cd6846fff39e3c2 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bc652fe65c601c6a40bda51c801e65aafbe331f08452b76597b38e3e8606e65 +size 131963426 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d9b97febfd9343082172984d4954015097a19bb8 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cbe604a1f2d0e709a6fe1a1914fbc41ad9c46d3665314e1bb3fe715e4cfb535f +size 131963426 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1d201593c1f56fe873a2a6ffed4b1462992bf9ef --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ab223e880faafe4774a7d3741e7a3c95ccb0f77aa597f2143231c9bc7f8ada8 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dc52461945ee49280e34a20dd246db2ec6c924c5 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61f5b459ff9de2bc4c52e6d0946c8656ce1a31bbf3a03e12e56c5eb33f0c9fd1 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..733cc36176ff67166c2093500ed3ecac2904b110 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91ab4edc845846497b82b39efdd8a2d8bed33f9797bd792585a80e411ae073ce +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e9fe9c67d54e704a2b49ab2fc30aeecd503451d9 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec141b69c18b245229e4ab28498f77af168bfa48897c7b31e8a4cbdf99c67c57 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ab2906c32b034e6120e1f214047869bf373fbd11 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7dc1bb03e4fa08fcb1abf1320ce2108170fc522208ffa30153a682fd6ea64794 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..14642db9712e52f78388dc802602b3edd5754c5d --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d477bff2ea36c5db0530965b3ff02dc9916fded844371d0107901b1fe7bf91a1 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..64ad0aaac60b2e0c4ce9259288270c0ff9ff3724 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b3784c74a8c29e22065695aa5b2af0d511eae20c8cf401055e90ef52fe8b76a +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..52076da3a90dc7818e5b8dc05b936255923fd6da --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d580f7f19b7fb723dd441938202b5db15bd55bd987bc05dc4ae8ab1f43b2f653 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..140ddf17a7e069ac79323566d9e0509727a33209 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7fc6cea8319b7e20d72a77a869d6e39b87108bb86250aefd36eaed2bd2e342b +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9cd13da58a9a2c0f277cdb709090600d70e6f824 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e96926c1b112964964de6094e7fefa3381adaef8d2af21dfdc44dd99da750d01 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b739c3294e3460e28543af3a03b1e1475d92da2f --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2981e92d8898ba76ae6141a00468b44d9eaa9305667e823820024c5d1ac57774 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..491448ea2bdb0ccfcc59f3a03ea51a004b75aeab --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88d1ca065fe6f8bc59b018a27e98660c0ee951b824c23b983d489be5f1ea9a57 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2fcccf456521e9b4328de63346d5f69e303a154 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:866a73de1f53256449e9c13077942426b1a30470ced938b65e419823a1ab73a0 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b9b1e290e65eaa503efca0b8da94e520b847fc5 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8185f818230b2d691dabc918096debd450f0adad10a3edca7d650faaa516be4a +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..548cde6b8dff0f7c55ef20739948116c040fb611 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f633409a82201e77e799ea144899ee855a6355953d585726016a8aa46a63b9ac +size 131963479 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b8dbbafc2a6245c4a2e26e6d768bbe2fe35fe05d --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:848e1e2b7ab2bbb05f04bdee1c1fda2ac986bcb18f011aba940d7931886e00ec +size 131963479 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..89364f8612f808bb0f1e07aa022a54cd4d07916f --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d31233b04ecfc5f2c6d8bcebdaaf426718980035508a1a31f5b4888822e4326 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f69b86ff9cae13889ed7cbc53f62f761fc07813f --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:349e94811cf0f1facf5b5da3717a3f4c4585f935b0636826365f664de546160b +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c39e5083df17cabd069919acd527271427b97afc --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:832db780da509f933f748b406ee712d2e6e5105618a2f5d8da5464981f463d14 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..76bffa397e155ee68bb0d1ec599a8dc136fa1fac --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c492589f99cb304ab752542123ab40eae0b452ed8c53ba3c72b079546a9b9318 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f438b49c07b02865f4840c1a321fbfb916c01a56 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2f38b90046ffe812ccca239e30d559e78ef59d109871acc425e70fca0280e34 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..90fd680817e5f74de9b420d1826f973ccb59ab19 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e6c09cfa1638db234ee0c3b5dc34e9ae8574f589312b308a13196d9a0f69349 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7eea550a0ba53056ad133eff14f68da7aee54a0b --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e3a8d2591a3842a26d66a9087b4028a21e7efc60f51a12590cd24edce7261ac +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3942058c8ae35af510fa47fc4501761b35196b76 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3110a54b7c251f8d67b3f2c2cccc1b6630225d6379033839e05281f7efa4a25 +size 131963618 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0420cec858d77617a466777b9d9736760df48f8a --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62adeec7a105c1b2081cc41140e03953fd55c5033f128af4d6a3e71f46979814 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bf7f01e06ff391012ea01157f2575faf70f7e3a8 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:200084e25e93cefd8e966dde3244ab8966df518343f0cd7a0edc2b180f6980b4 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..44ed908dc66779a24a23284caffe9c53e0f727fd --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea741f7560dc1c4b24a34ef6012c849e91158a5caff824d2e3ba2a511bdd3971 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f708cc9b15cafa8630ea8cc7531838671d6cce33 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c523abadd52272b682c0f9668885b2cab5cded5c26663e23c4751150a00c6b4c +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..312f40e8aa01a3b3727c58f29578f0cdcd50c826 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee5a2e1d6a27388aa9160c1f54c0239859324c67dcba7130968a38b84ac2eb1f +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4961a2a2f4a2dc422dfec2480d7e1eaac053fdf4 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95723a1dbefd7736ae1f59f387809b03900471f0143cb13cdb57312d077926a1 +size 131963490 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d7ec2f375dc38ec6c7dfabed865f33537c5752c9 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a2a149d73971f6733cf517b864df29781ef3a38f8dc1dcb047e1ce3fb9d3d3c +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0e436a165f01dca931db1033bd3598e0c0c262c0 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:003da5b5e89bbe94cbba2be569c97406485acd6fd5cfa697b2e2fdcf9f932928 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..761b21080596bffb8d2847437e669ea8107a293d --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07fda71c60d128ac648532c3d31a6d4ebfe5489f9b1e2426d8771516833c12bc +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff1cfe2b258669a5b244673f902e183d834e3eaa --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:349bfa8823f0c97a820118f097e8ce43726d7f908e7d4e7f69b6e046d60c8117 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9095d9e6d0c6849932e6a90c4de244f11d97332a --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da1682c92ddf1896ba4a366118fd94a31d5ffecf85137ebd800386719555c702 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c1b8d887f4612dc8d856dca60a3e3e4360752b14 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4bb1abed190243eb7e226724d27ce77632b447378a247faf3b9fac39b02f117 +size 131963554 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aa03b63ea2ebd630dabc80448010e36bb4cb20c6 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b0ab4812c9030abf651b6fcf8b9a1588914d98181db9305699d9f495b4128ef +size 131963543 diff --git a/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a4757e83d6f1f8d97ec6c7da5342c32df4dc6e6 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:454172d83f4b2d87076709e5dbc8308b58e1c702803f4cfc055960dc2af2d070 +size 131963543 diff --git a/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..039f05bd638058b16c74dae5d83473681ac310ec --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a397c2886ebb89ed1df55303ec8a90427c4edde1e224d5dd8b87b24852c7b953 +size 139592963 diff --git a/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..98adec391ce72d6cd1fca48818cfca56dfbef0c1 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_01-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:358f9ffb0f8fd8b91c446ce000990609c76a3b0f9577de48dc373acc2d36917a +size 139592963 diff --git a/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd7239e72f8768787ac6f5b33152c0145cd91ec4 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75c03f9ef882af7f70c9b4bb0ecc90371c6492a7ea450bffb4adc9b60d0253d8 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..91a131275666a44862e0d32bc30ac72971401e88 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_03-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44738181cc582bb13a1f064f9d63fcfd7284172cec13825c4dc143a344b743d2 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ce32c73c41eb94325bc00321ba047a8c211cfcae --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71439608c6fc29069916ee7419577a94c7be89f59989a4f960de0dc39a75c569 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d36d54ce3d49eefb93c9b0c6b1450a2f25975c2b --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_04-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c14b280977e346bd44629ab4aa5e4cc867aa1277ca8810822a9e4a76e02cd675 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0540f667c2242a5550e2f7ed91b4f84abddd3558 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c84ce6a620138e065a22181f5b06e3daa77ebe09ed1e13e7f7f85a56ec2b97d8 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e7cc947fc39ae183b4ef52c4dc77ce9674dab454 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_05-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6bd7ff2858c4a48c632df24a50639c4cd3d8a6c31a7641ddd479c4c51280fd8 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..39861847c172fb636c00d5e67eab870d5a818052 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34231a02ad4de10e1c1ef92ccd248ce0a7debc10e8ff4876f2cc2497efee09f2 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..857d6783cad368e33a81a3454f51a37d01c2e002 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_06-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb709abbee4db8f630cd1e1d0c973d5fc4898e5c00c82b11b307012cf3b67ab7 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9e99bd143afc7b49a342819c086cee3a504fcbd7 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b3e75f713730919650dd9b11e6693b034e3cb5011dd3eeea927e8392db4baccd +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b9bbaeaddfd90fd228215fb6df2dbc0f01bf66b8 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_07-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4db9bbd77bdff9e84f38855ee4f6ce86eac95c47c8d1b650a907e90780416f7d +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..215ec6304c4ec6259f9be96bd89297ee97acf4b6 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35a52c9eb9574879864179fd14fe9eb8af063c7e334a6f38a30bef789a8d1bd6 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2b3c1dc93803b496c1d5a2a573a5a5e226bf5887 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_08-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07891ece198f2d4b2c9e40322b81004a6ad239d26179e4e9ddf9df3c426edb3e +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7ed0408f76a2bc0f2c969abff590cef4c188e4b2 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:601099d6999c09f14f5450ef0535bb598e296a12592e03128265a62ef582d1f5 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a4da555ffbf83155b990faf7557aec0351c30158 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_09-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b17865fc8c4b330343a1671c801256a009b2aaf53284a65d71724cc03f0c68bb +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2fa015044040765ad05eef6e1de4ffac001bafec --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6fedca67bb77abd57c4daa330390d115d0e93810af37e081588eb31e47a1f28 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ab93e0ec9c27b435c0f06881cc9e639661143f30 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_10-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ddc789714f1d80f491fa05625243a782ec3d1cef5e5093624602c4a8d4c1b42 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e35f67e7b1613dab7c66aed473fdeac0b9cc939a --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:929efcd7358fa793859c3abb165a77d0a714cd6b9b251abc0a43c3a87dbc2ad7 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b9cc51b3a8c760750e55100e4d721242d980dfe2 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_11-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e269df8734d2bc11910e1f9d3848b95b6342aa4e44ae555f86a3bb8d65c08a8 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3eba7293e89bdeef1fb486b2da9deddbd31c512e --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea9dc4a39736bddf253ed125f25e04a9aab8ff228e3745abbb7160001e3cdf4b +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a44aee6f4e8bd019c4e77720f34cbe1f9a425476 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_12-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96e261611ffdc37742249efaaca6cdb71f079c05770105842a07cc03fd29b65d +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f71424170e02a470f7adb2dfb63434213b608717 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:040bf7d551d4f8ef32db346c0ca2f6d53aa283026f25f4e16a2968e7fc2aea7f +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..752c5722cb80051bf3c211ff5ac154347b9d52d7 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_13-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37717f2e66b40d0139ef5bb577a4964453ea23368172a6f2d8e718521270a250 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c7783ba68b9b1cc110fd0c2cf08db6e1889c9625 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:506bb9c729208072725db7a132f6d7e27e6ebaf066e5748017010fcd0546578f +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b3752b04a74b8bbf577a17834d4edaa0c75c45a1 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_14-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98c4032fe050a4832be2884f9e69401ff1ab9c8a5067f6a8fa62a8afbdc5c326 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a0abc2a37f7066718b2193bd98f574ca964374f --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4a88a2dd25da1fe0c6fdd6ea13da509dabc1588dd513f85b8b1f487bab9dea6 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6ba8710d669c38686586255d4c4549998e7c21a --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_15-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f35f32f5c073af333570ec6ccdc5f9290c5cc1ca8709fcf4b3379af5df6f267a +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..db3396816250ae7cf97a5db412ce3ddedde33306 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d69d6ac789b58e2c834234a8ed5be3652431f54dc0ca1b716204a307dbd14ffc +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ffbfae98061d8965a86f570413b17f1bb616545d --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_16-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1205dc5d15e1c192902dc84ea940cef624eee4be06be57234e585ae4bcb5854a +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c3b95c50c36ae73db291579fb2ed3055eae0ef03 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae2faf716c72689ab8dfb253c18aec844c91180d982cb450a80b7aea6f128b08 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9314bac3bcc4d490863f27d8bb94e699f1ae3e52 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_17-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b13e85573a8f045f07d1256ec7b93d25aecacf2281bdc6fdc85a25fbfe277e5c +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b86a61c53041286c633e3472b65040949bcd019c --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:969d5fb21ae69b837a1fd4d7998a6faf8c93f1dfe103e27079817f87d3ef9b99 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a0f6e1b732093bf67b3e320d888ab34c5b5fa552 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_18-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b35dde9593d01bff8b980068f8eeb08a8090b4305512ae8ea4d78ac4403f3df0 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..db5ca4273c7442a011892c6411341bb030c4d844 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b282a4948661b2c3b4eb756556334e472275e49b903fcf24f2103bc56bf7e41 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..012c96a02bddb2f1985eda8ab11bbe007562c446 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_19-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8e0fa856bdd9000068530254964e49445549663e3b1d12ff0689429ee810335 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b940d1dd85a7c041dfd2ed9ce903c7aaff7f5617 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e58dc9c86b4820161020358c8870eec1f8eb68017c140f87c6406d43e6b0c2c8 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd8a20760e0db36de240e51e03a67022aca6716c --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_20-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fd6e7f2520673e2a20e62cc1562712d976ba00dd7d25ccd78a7ac44f5f47f0d +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f6eb4cabfa1c6d18ff394006f3e4aad7d75bd0c0 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7e5f834d3760c2f4e7ae92f5ab66072057c0ae09b38ecac1ec9d438d33cf8fd +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6766cf18f33031c946476c902654c24d9ac6c0c5 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_21-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad767d8179e761e8a1444f9354c85ae31abeee6d76d8316930677b45aca4323e +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..728250971444a085640e2863d9457bd182254aa2 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_22-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58ae28cb091fc8fe1201bda0d3a0a3222a73793f25378445527a39f5f07c1ac5 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..db9db4dc50c776d52542fc4740e64b1c4a07452c --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_22-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7178602c67741d4b0e351c3e91affd7ae4c0a40e253da29b35908352f49d10d +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3a15e66f56a1538b6ba01147014c0cadd69c7072 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42c71a6484912600f29f60bbfdb9b62682a607c5c08135a23453aff13a451131 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6af45038c19906aea30e66fceacb199330aa2809 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_23-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:801499db1bb10808d00f945072c6e8c360ca26dd21d3116db2e92f8ceb8b033d +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec9a78a2afd505c111c7f4ee8e2013e4ae834484 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_24-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1846a1768e2f69f663575790e1e6afba7ffe7fdd47e71f45f62534f324870bd6 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2e829fd981ab34c846f4723cfb05c758a8a2264 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_24-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cf3c0a8f922ee91898a4914d1d63c830e9a11fb2126bd184b86b6e6960f2249 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..202a6d4d993382d07efc2f652ec7f42a1e4b508b --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_25-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:725757e33904ee24db1217b8b26fbffd5d103225d1c23eadf6c57965914e190a +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..56c5f25812ff4e7564e69a3850e8bf47ddf447b0 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_25-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4af42b1f9e5db222fb0a2c8e88b53edc50b559877881b657eca5fc04e474511d +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3fa69fac7fffc0580f73b5e9aa26e244ca77b696 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_26-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5658f5ba70d13867ea9f46be7bab73d0254cd9a87d601c81aee3118b68b128e5 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9cf87827401d6266adf6559927ffdc880a698ded --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_26-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d39fb62a987b7b6681ba0ddb301ad1fd11e26bac317a6598ad29d5595e1a31f +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d8e1e1806d04b41dee1dc53e26c0f04ffab07903 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_27-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a5b25e9e8f51fbc9fc1ae7b0c6ca4e1d8872592bd53d9fffa0fc5492d2daea82 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..18ba482bd84b9587817021890de295d97657c815 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_27-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4370e61829e0d10a48e35972d251659202f6c82c471cafdf47ae5d67fdbbb02 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8be6895b3940e246459ccfe041e3c184160e6991 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_28-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ecdb9d411758e0f016d4ffa007842d3e6757f048a037483b77af7d5b4c04400 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4d533d4422abce7c57b8dfc58f73ebac1176f96e --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_28-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b48e02396fd000c4cf31e5aec0d8df5ae0f31a8af1d7e5a5f08b7cbc5578ad1 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..de731b71a58214912d380059d1cd88ee8f46c234 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_29-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6c69f5001f8527d454ba59b22a1b0b45314c2b28d06ea1fcb669cf013bd014b +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6eadd99dbe8afc5c5fe8415854e2d8a92c5b7e9 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_29-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9187266f6283d92a186bc86669ad5d96b932f7cacb486a8c97e2a0d4084b0f72 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a864696bc4a33d68394c83a469944804f1308393 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_30-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6efad80810e21b6e19e3ba56878d4c5ac9b41b9b8a670f74568c3a7179f3fb3 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..65899e8c24913593bd4aeeb68790be515ef6544f --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_30-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:598f731118fd7c2705dcc703ed2e856a23dc863b42ff8174a8ef222ee9ad508e +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f71d8e09a3ad565b4ede1510a3bc03786e0f1040 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_31-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cae8c7ae895ea04e742ad2cbec45671d67c39aaf4adc3e44b9ef1c66f6ba9bbf +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5447c686c9ba67f0630f6115c4abe560586aad74 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_31-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60515b6d40c6883173a49d87477921807c3a59ef8a45ec7b1a560492a272e037 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d885696899df4f50773314107b9f6e7a3cc5209d --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_32-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:772a0154d435fdc4a73eb99d1a204f9bb7a3598d2dbcc2ce733ad17af701ad57 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..28dc8114a88674bd0c5f868bca7bcf1bde287b44 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_32-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5e6308e861153793a0881014e60912e03f5eea7dd720b4108f7185e53ea60d8 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..16ed3232f0af6a2a77a6ed0358b72f0541eab6be --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_33-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15c44ae13ff48f334b045f1ca904ac0eeb0c85fdd276b22252d557b89c3eb168 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aec68819e199088e04691386acbffe8483a650e0 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_33-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9d2a61a9cb8030a9e5029cbabd3505867d282452ccfd4f502753321dd46db2f +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d7017356d48bc00697f034187a1c28eb46d24cab --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_34-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e610321799a1d8bf3090593c551b9741aa6dddfe8a00f58d65a7cad2870db55 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d50436e4af7e1476d65bacc4cbeb1338536fba60 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_34-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae280cfbf825dda92c0a4614cc4b3a1db6a2fb82ebcf291020eb5e58003b5f89 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e02bd52308892dc5382edf27aafbea58980e4f73 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_35-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b54cb01dddcb39caa5dec1ad95dc3f754139c15847211fcc88de1cec3459568 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a839b03bc2dc72ebf5cf02780a2e22da1596d36 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_35-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e965a3f6dc637931f4a377bc289451591620f8a911f4c95e899043a3acd1d5d +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..92b71e937b4c46b0a0f5d4503db2b066bd29a8e0 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_36-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8c61cf48f4af2d3ddd8ba77aedd1e4c50bf970a2c831e3d907b6a7b682d990a +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..37eb5c4f8b6f50b408b6754425b1f0f4f655f634 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_36-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a78392df77752aad15e3c42581faa858ae5781a61dbb1cca16fc5bc13cf8182 +size 78696195 diff --git a/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..75982639754a620a3562b14cdc51bad3f680a94d --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_38-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d89d5a0cbf1415bf1514e6de07b09c4be9991bc38bca1120f4478258f3f3c6c0 +size 11459 diff --git a/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt b/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5f220d98105a98fe7b1f4ecec3996094772c510 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/layer_38-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8111e6887503025e0b555dd7af2d82c5f7833b37a1a1aada1beed2cc5fb4400e +size 11459 diff --git a/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt b/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..578ce505761e4c0991f2514d7072a308a50bedd6 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77d12eae3fcc410bd8c253605f026308e22ba1ad77ce7d5330d6d5a0aa593f9d +size 49971 diff --git a/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt b/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f0a3ee062192e1740f25b9a3088b75048ba7b76 --- /dev/null +++ b/4b284b6bc4opt2/global_step122070/mp_rank_01_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d0e8a2a733a53fc898aa7758f3249249fe799f479fbe544d87c447a175cc103 +size 49971 diff --git a/4b284b6bc4opt2/latest b/4b284b6bc4opt2/latest new file mode 100644 index 0000000000000000000000000000000000000000..a5e10d18c6420ad34cc42e89780321191737db3e --- /dev/null +++ b/4b284b6bc4opt2/latest @@ -0,0 +1 @@ +global_step122070 diff --git a/4b284b6bc4opt2/sbatch_4b284b6bc4opt2.sh b/4b284b6bc4opt2/sbatch_4b284b6bc4opt2.sh new file mode 100644 index 0000000000000000000000000000000000000000..a87622ebae11a2ae40f77a68056d09327a2a9e43 --- /dev/null +++ b/4b284b6bc4opt2/sbatch_4b284b6bc4opt2.sh @@ -0,0 +1,165 @@ +#!/bin/bash +#SBATCH --nodes=32 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=40 ############## +#SBATCH --mem=256G ############## +#SBATCH -p standard-g ############## +#SBATCH -t 48:00:00 ############## +#SBATCH --gpus-per-node=mi250:8 ############## +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread ############## +#SBATCH --account=project_462000119 ############## +#SBATCH -o %j.out ############## +#SBATCH -e %j.err ############## + +VARIANT=4b284b6bc4opt2 + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" + +TRAIN_DATA_PATH=train6b.txt +# "train: 1.0 0:1 gpt2tok_c4_en_6B_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 gpt2tok_c4validation_text_document" + +PP_SIZE=1 +TP_SIZE=2 + +MICRO_BATCH_SIZE=2 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_2980M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=10000 + +# Tokens: 84_000_000_000 +# -> Samples: 41_015_625.0 +# TRAIN_SAMPLES=41_015_625 +# TRAIN_SAMPLES=62_550_493 +# TRAIN_SAMPLES=48_828_125 +TRAIN_SAMPLES=62_500_000 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.95 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 625_000 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1000 \ + --eval-iters 100 \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/4b284b6bc4opt2/sbatch_4b284b6bopt2val.sh b/4b284b6bc4opt2/sbatch_4b284b6bopt2val.sh new file mode 100644 index 0000000000000000000000000000000000000000..1045cf749996dc9aad413c35ef54ad2d0316f659 --- /dev/null +++ b/4b284b6bc4opt2/sbatch_4b284b6bopt2val.sh @@ -0,0 +1,168 @@ +#!/bin/bash +#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901 +#SBATCH --nodes=32 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=40 +#SBATCH --mem=256G +#SBATCH -p standard-g +#SBATCH -t 48:00:00 +#SBATCH --gpus-per-node=mi250:8 +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread +#SBATCH --account=project_462000119 +#SBATCH -o logs/%j.out +#SBATCH -e logs/%j.err + +VARIANT=4b284b6bc4opt2val +VARIANT_CKPT=lm1-4b2-84b-c4-repetitions/4b284b6bc4opt2 + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=$VARIANT_CKPT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" +#DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document" +TRAIN_DATA_PATH=train1b5.txt +# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_12B_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document" + +PP_SIZE=1 +TP_SIZE=2 + +MICRO_BATCH_SIZE=2 +GRADIENT_ACCUMULATION_STEPS=2 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_2980M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=1000 + +# Tokens: 25611230000 +# -> Samples: 12505484 +TRAIN_SAMPLES=1 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.999 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 0 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + --override-lr-scheduler \ + --reset-progress \ + --no-load-optim \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1 \ + --eval-iters 100 \ + --eval-only true \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun_32.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2/events.out.tfevents.1683364451.nid006970.89702.0 b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2/events.out.tfevents.1683364451.nid006970.89702.0 new file mode 100644 index 0000000000000000000000000000000000000000..4f6e21e2f7c96672b3afbaa497168def1f9c791b --- /dev/null +++ b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2/events.out.tfevents.1683364451.nid006970.89702.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3a45dbf506a4175450c680a4cd1ab252ef61d20c29389d1ea98ca56443f9f38 +size 158975207 diff --git a/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2/events.out.tfevents.1683535163.nid006037.36222.0 b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2/events.out.tfevents.1683535163.nid006037.36222.0 new file mode 100644 index 0000000000000000000000000000000000000000..c7882cbfda12afdbb6afbe70f146354bca4ee3bc --- /dev/null +++ b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2/events.out.tfevents.1683535163.nid006037.36222.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f329f63d3010ebe247692068c293d17997180cf862834ffa6618e5dd9be62885 +size 51627163 diff --git a/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2/events.out.tfevents.1683592046.nid006518.57091.0 b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2/events.out.tfevents.1683592046.nid006518.57091.0 new file mode 100644 index 0000000000000000000000000000000000000000..363c65f8312807d5c3958fab6c1f7cd9a7975bd5 --- /dev/null +++ b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2/events.out.tfevents.1683592046.nid006518.57091.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6e07688918f96d7c760b7edf7b6c8bba423167e4d5c8d5d6bde87a76809048f +size 40 diff --git a/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2/events.out.tfevents.1683592589.nid005976.77505.0 b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2/events.out.tfevents.1683592589.nid005976.77505.0 new file mode 100644 index 0000000000000000000000000000000000000000..9f5d799be355ae63d7ecdfc4814ab0c633d90d19 --- /dev/null +++ b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2/events.out.tfevents.1683592589.nid005976.77505.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:076126cd1f8d1d02deab8bebdea3daa3426650025796034b9bd80884071a06bd +size 40 diff --git a/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2/events.out.tfevents.1683593118.nid006970.71348.0 b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2/events.out.tfevents.1683593118.nid006970.71348.0 new file mode 100644 index 0000000000000000000000000000000000000000..f1de00ff88f950b84fd07b0e2e13746e68403274 --- /dev/null +++ b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2/events.out.tfevents.1683593118.nid006970.71348.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:343f466fe20be1379a60e0a9937e6abee3ec9c346aff1fb77259eff77f796b78 +size 40 diff --git a/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2/events.out.tfevents.1683613091.nid007164.84189.0 b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2/events.out.tfevents.1683613091.nid007164.84189.0 new file mode 100644 index 0000000000000000000000000000000000000000..364784d5de5cf7a5120b7601cd61c1a21791f0c5 --- /dev/null +++ b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2/events.out.tfevents.1683613091.nid007164.84189.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b139b07d50839c22e46e0e572beac0188de24523e071894c2d300f37e25f6233 +size 21712252 diff --git a/4b284b6bc4opt2/tensorboard_4b284b6bc4opt22/events.out.tfevents.1683568181.nid006970.42470.0 b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt22/events.out.tfevents.1683568181.nid006970.42470.0 new file mode 100644 index 0000000000000000000000000000000000000000..9ebe1013f2156843a622eed66a11f5adb940fda7 --- /dev/null +++ b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt22/events.out.tfevents.1683568181.nid006970.42470.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60cf78da09a6624705a3b9efed829c8a586cd349ac4fabda52a74dd5031e6350 +size 12787137 diff --git a/4b284b6bc4opt2/tensorboard_4b284b6bc4opt22/events.out.tfevents.1683592046.nid006037.124158.0 b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt22/events.out.tfevents.1683592046.nid006037.124158.0 new file mode 100644 index 0000000000000000000000000000000000000000..2f288a065820aebd863d1a595f67f3e097e8019f --- /dev/null +++ b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt22/events.out.tfevents.1683592046.nid006037.124158.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccd207438151c03ecb167d63ee34eec59ca28586c75911f6828bd9a868990def +size 40 diff --git a/4b284b6bc4opt2/tensorboard_4b284b6bc4opt22/events.out.tfevents.1683592589.nid006518.65021.0 b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt22/events.out.tfevents.1683592589.nid006518.65021.0 new file mode 100644 index 0000000000000000000000000000000000000000..7f06430f28db85b57e51703ba648c5d04b1fc019 --- /dev/null +++ b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt22/events.out.tfevents.1683592589.nid006518.65021.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70e797820c7a6caf1a5ffa47f7e3682b539f08adeb2cfe75c7458abbf203d09c +size 40 diff --git a/4b284b6bc4opt2/tensorboard_4b284b6bc4opt22/events.out.tfevents.1683593118.nid005976.81898.0 b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt22/events.out.tfevents.1683593118.nid005976.81898.0 new file mode 100644 index 0000000000000000000000000000000000000000..e05b3a1ddaf1971319592360f288c67a2b8ad9d8 --- /dev/null +++ b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt22/events.out.tfevents.1683593118.nid005976.81898.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cefa09d77ec1128f3cbfad4629ee216c33e0759c329e00696687a9bc31395628 +size 40 diff --git a/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2val/events.out.tfevents.1683566930.nid007109.68824.0 b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2val/events.out.tfevents.1683566930.nid007109.68824.0 new file mode 100644 index 0000000000000000000000000000000000000000..a7def4f1a3d36ea34db9499490531bc371a5a51d --- /dev/null +++ b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2val/events.out.tfevents.1683566930.nid007109.68824.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae25dbb675386f4ac59b2a4d61fe58236d0f23c8c4e041fa594aea1ca528826d +size 40 diff --git a/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2val/events.out.tfevents.1683567370.nid007109.76550.0 b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2val/events.out.tfevents.1683567370.nid007109.76550.0 new file mode 100644 index 0000000000000000000000000000000000000000..999c1eaff6eb3bc2faf683685a47bd21788cdfae --- /dev/null +++ b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2val/events.out.tfevents.1683567370.nid007109.76550.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6df15af5b68abfba4f315bc61cc4d24d6df694661073c0b8fa69594d66017815 +size 40 diff --git a/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2val/events.out.tfevents.1683567544.nid006565.68946.0 b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2val/events.out.tfevents.1683567544.nid006565.68946.0 new file mode 100644 index 0000000000000000000000000000000000000000..fb30132bf1e9bd51249231a7980da94de0f2b9c2 --- /dev/null +++ b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2val/events.out.tfevents.1683567544.nid006565.68946.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb529d1c2a270061f63c1f76223f8d13af7204ad3140119108f26ffba4fdbdd5 +size 40 diff --git a/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2val/events.out.tfevents.1683567717.nid007109.80534.0 b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2val/events.out.tfevents.1683567717.nid007109.80534.0 new file mode 100644 index 0000000000000000000000000000000000000000..eb30e532a55d5235dd7cb2acd2848018b4514779 --- /dev/null +++ b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2val/events.out.tfevents.1683567717.nid007109.80534.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3840289d73474f219facabf3d0e64dab80d0f22c8fdafb1c99b25cd2ce4da37b +size 40 diff --git a/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2val/events.out.tfevents.1683639805.nid005245.35925.0 b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2val/events.out.tfevents.1683639805.nid005245.35925.0 new file mode 100644 index 0000000000000000000000000000000000000000..7443da5b58f6de578c494bb2361e562195675051 --- /dev/null +++ b/4b284b6bc4opt2/tensorboard_4b284b6bc4opt2val/events.out.tfevents.1683639805.nid005245.35925.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca28c9e2cf8a38b558642b0bb1d766660ad71b4c7c25cdd89edb862f70014bb8 +size 980 diff --git a/4b284b6bc4subopt/3476842.err b/4b284b6bc4subopt/3476842.err new file mode 100644 index 0000000000000000000000000000000000000000..feee0ccf0666c8ea19431b9fe949832ac6483a8f --- /dev/null +++ b/4b284b6bc4subopt/3476842.err @@ -0,0 +1,3651 @@ + 0: 2023-05-08 11:22:59.962561: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-08 11:22:59.962573: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-08 11:22:59.962604: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-08 11:22:59.962615: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-08 11:22:59.962628: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-08 11:22:59.962628: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-08 11:22:59.962633: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-08 11:22:59.962640: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 0: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-08 11:22:59.963166: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-08 11:22:59.963174: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-08 11:22:59.963201: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-08 11:22:59.963211: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-08 11:22:59.963210: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-08 11:22:59.963224: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-08 11:22:59.963231: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +19: 2023-05-08 11:22:59.963258: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +19: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-08 11:22:59.964152: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-08 11:22:59.964155: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-08 11:22:59.964153: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: 2023-05-08 11:22:59.964169: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-08 11:22:59.964181: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-08 11:22:59.964190: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-08 11:22:59.964174: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-08 11:22:59.964183: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-08 11:22:59.964273: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-08 11:22:59.964280: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-08 11:22:59.964292: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: 2023-05-08 11:22:59.964136: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-08 11:22:59.964151: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-08 11:22:59.964128: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: 2023-05-08 11:22:59.964186: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-08 11:22:59.964190: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +10: 2023-05-08 11:22:59.964199: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +10: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-08 11:22:59.964307: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-08 11:22:59.964317: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-08 11:22:59.964299: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-08 11:22:59.964303: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-08 11:22:59.964313: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: 2023-05-08 11:22:59.964316: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-08 11:22:59.964335: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-08 11:22:59.964328: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-08 11:22:59.964330: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +20: 2023-05-08 11:22:59.964332: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: 2023-05-08 11:22:59.964290: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-08 11:22:59.964303: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-08 11:22:59.964296: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-08 11:22:59.964174: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-08 11:22:59.964185: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-08 11:22:59.964327: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-08 11:22:59.964327: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-08 11:22:59.964370: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-08 11:22:59.964382: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-08 11:22:59.964384: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +20: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-08 11:22:59.964310: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-08 11:22:59.964315: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-08 11:22:59.964193: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-08 11:22:59.964200: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 1: 2023-05-08 11:22:59.964220: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-08 11:22:59.964192: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-08 11:22:59.964199: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-08 11:22:59.964332: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-08 11:22:59.964312: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: 2023-05-08 11:22:59.964347: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-08 11:22:59.964393: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-08 11:22:59.964395: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-08 11:22:59.964321: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-08 11:22:59.964319: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: 2023-05-08 11:22:59.964310: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 1: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-08 11:22:59.964211: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-08 11:22:59.964213: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +16: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +22: 2023-05-08 11:22:59.964405: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +22: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +11: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 7: 2023-05-08 11:22:59.964261: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 7: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-08 11:22:59.965014: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-08 11:22:59.965039: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-08 11:22:59.965055: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-08 11:22:59.965082: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-08 11:22:59.965088: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-08 11:22:59.965066: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-08 11:22:59.965103: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 2: 2023-05-08 11:22:59.965111: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 2: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-08 11:22:59.965257: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-08 11:22:59.965265: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-08 11:22:59.965283: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-08 11:22:59.965292: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-08 11:22:59.965301: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-08 11:22:59.965309: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-08 11:22:59.965312: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +30: 2023-05-08 11:22:59.965326: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +30: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-08 11:22:59.965444: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-08 11:22:59.965466: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-08 11:22:59.965476: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-08 11:22:59.965483: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-08 11:22:59.965493: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-08 11:22:59.965512: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-08 11:22:59.965527: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +17: 2023-05-08 11:22:59.965546: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +17: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-08 11:22:59.966401: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-08 11:22:59.966410: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-08 11:22:59.966404: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-08 11:22:59.966430: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-08 11:22:59.966441: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-08 11:22:59.966463: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-08 11:22:59.966517: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +25: 2023-05-08 11:22:59.966539: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +25: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-08 11:22:59.967527: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-08 11:22:59.967572: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-08 11:22:59.967577: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-08 11:22:59.967620: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-08 11:22:59.967623: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-08 11:22:59.967625: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-08 11:22:59.967616: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +21: 2023-05-08 11:22:59.967614: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +21: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-08 11:22:59.968057: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-08 11:22:59.968071: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-08 11:22:59.968097: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-08 11:22:59.968064: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-08 11:22:59.968100: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-08 11:22:59.968127: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-08 11:22:59.968137: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +14: 2023-05-08 11:22:59.968155: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +14: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-08 11:22:59.968643: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-08 11:22:59.968665: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-08 11:22:59.968647: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-08 11:22:59.968686: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-08 11:22:59.968698: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-08 11:22:59.968694: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-08 11:22:59.968733: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +24: 2023-05-08 11:22:59.968734: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +24: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-08 11:22:59.969569: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-08 11:22:59.969616: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-08 11:22:59.969655: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-08 11:22:59.969612: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-08 11:22:59.969664: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-08 11:22:59.969686: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-08 11:22:59.969697: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 8: 2023-05-08 11:22:59.969712: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 8: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-08 11:22:59.970439: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-08 11:22:59.970442: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-08 11:22:59.970485: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-08 11:22:59.970502: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-08 11:22:59.970521: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-08 11:22:59.970534: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-08 11:22:59.970546: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +23: 2023-05-08 11:22:59.970555: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +23: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-08 11:22:59.971139: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-08 11:22:59.971170: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-08 11:22:59.971198: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-08 11:22:59.971218: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-08 11:22:59.971235: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-08 11:22:59.971248: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-08 11:22:59.971250: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 9: 2023-05-08 11:22:59.971272: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 9: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-08 11:22:59.971948: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-08 11:22:59.971996: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-08 11:22:59.972013: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-08 11:22:59.972015: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-08 11:22:59.971997: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-08 11:22:59.972000: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-08 11:22:59.972004: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +15: 2023-05-08 11:22:59.972047: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +15: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-08 11:22:59.972770: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-08 11:22:59.972783: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-08 11:22:59.972797: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-08 11:22:59.972796: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-08 11:22:59.972806: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-08 11:22:59.972918: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-08 11:22:59.972930: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-08 11:22:59.972940: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: 2023-05-08 11:22:59.972824: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-08 11:22:59.972833: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +18: 2023-05-08 11:22:59.972842: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +18: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-08 11:22:59.972949: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-08 11:22:59.973001: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-08 11:22:59.973025: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-08 11:22:59.973034: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +27: 2023-05-08 11:22:59.973054: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +27: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-08 11:22:59.974173: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-08 11:22:59.974255: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-08 11:22:59.974268: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-08 11:22:59.974294: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-08 11:22:59.974310: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-08 11:22:59.974295: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-08 11:22:59.974323: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 4: 2023-05-08 11:22:59.974333: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 4: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-08 11:22:59.974710: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-08 11:22:59.974730: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-08 11:22:59.974735: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-08 11:22:59.974741: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-08 11:22:59.974771: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-08 11:22:59.974779: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-08 11:22:59.974789: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: 2023-05-08 11:22:59.974734: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-08 11:22:59.974759: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-08 11:22:59.974736: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +29: 2023-05-08 11:22:59.974794: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +29: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-08 11:22:59.974787: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-08 11:22:59.974810: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-08 11:22:59.974819: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-08 11:22:59.974833: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +28: 2023-05-08 11:22:59.974850: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +28: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-08 11:22:59.974995: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-08 11:22:59.975016: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-08 11:22:59.975024: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-08 11:22:59.975049: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-08 11:22:59.975072: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-08 11:22:59.975054: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-08 11:22:59.975091: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +26: 2023-05-08 11:22:59.975116: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +26: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-08 11:22:59.975403: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-08 11:22:59.975458: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-08 11:22:59.975485: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-08 11:22:59.975498: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-08 11:22:59.975526: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-08 11:22:59.975529: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-08 11:22:59.975564: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 3: 2023-05-08 11:22:59.975530: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 3: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-08 11:22:59.975765: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-08 11:22:59.975761: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-08 11:22:59.975785: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-08 11:22:59.975797: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-08 11:22:59.975826: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-08 11:22:59.975834: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-08 11:22:59.975820: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +12: 2023-05-08 11:22:59.975883: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +12: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-08 11:22:59.976158: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-08 11:22:59.976184: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-08 11:22:59.976211: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-08 11:22:59.976213: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-08 11:22:59.976217: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-08 11:22:59.976243: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-08 11:22:59.976264: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +31: 2023-05-08 11:22:59.976264: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +31: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-08 11:22:59.976541: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-08 11:22:59.976569: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-08 11:22:59.976586: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-08 11:22:59.976600: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-08 11:22:59.976571: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-08 11:22:59.976619: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-08 11:22:59.976638: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 6: 2023-05-08 11:22:59.976648: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 6: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-08 11:22:59.977010: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-08 11:22:59.977044: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-08 11:22:59.977054: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-08 11:22:59.977049: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-08 11:22:59.977072: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-08 11:22:59.977121: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-08 11:22:59.977137: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-08 11:22:59.977128: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: 2023-05-08 11:22:59.977083: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-08 11:22:59.977150: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-08 11:22:59.977164: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-08 11:22:59.977113: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. +13: 2023-05-08 11:22:59.977116: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA +13: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-08 11:22:59.977166: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-08 11:22:59.977172: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 5: 2023-05-08 11:22:59.977173: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 FMA + 5: To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags. + 0: 2023-05-08 11:23:15.565244: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:23:15.565278: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:23:15.565298: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:23:15.565296: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:23:15.565308: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:15.567231: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:15.567251: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:15.567260: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:15.567280: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:15.567285: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:15.567294: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:15.567297: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:15.567297: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:15.567838: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-08 11:23:15.567859: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-08 11:23:15.567870: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-08 11:23:15.567891: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-08 11:23:15.567897: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-08 11:23:15.567901: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-08 11:23:15.567908: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 9: 2023-05-08 11:23:15.567912: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-08 11:23:15.572798: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:15.572816: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:15.572829: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:15.572844: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:15.572838: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:15.572851: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:15.572855: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:15.572864: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:15.573252: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-08 11:23:15.573269: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-08 11:23:15.573275: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-08 11:23:15.573281: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-08 11:23:15.573302: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-08 11:23:15.573307: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-08 11:23:15.573318: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +24: 2023-05-08 11:23:15.573326: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-08 11:23:15.591071: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:15.591079: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:15.591108: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:15.591111: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:15.591131: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:15.591133: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:15.591143: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:15.591133: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:15.592055: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-08 11:23:15.592076: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-08 11:23:15.592089: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-08 11:23:15.592116: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-08 11:23:15.592122: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-08 11:23:15.592126: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-08 11:23:15.592127: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +13: 2023-05-08 11:23:15.592136: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-08 11:23:15.596087: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:15.596107: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:15.596136: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:15.596145: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:15.596127: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:15.596146: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:15.596151: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:15.596158: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:15.596676: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-08 11:23:15.596696: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-08 11:23:15.596703: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-08 11:23:15.596716: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-08 11:23:15.596730: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-08 11:23:15.596731: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-08 11:23:15.596736: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +11: 2023-05-08 11:23:15.596742: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-08 11:23:15.597267: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:15.597286: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:15.597304: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:15.597314: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:15.597301: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:15.597323: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:15.597312: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:15.597327: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:15.597800: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-08 11:23:15.597817: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-08 11:23:15.597823: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-08 11:23:15.597831: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-08 11:23:15.597830: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-08 11:23:15.597843: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-08 11:23:15.597849: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 1: 2023-05-08 11:23:15.597854: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-08 11:23:15.597729: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:23:15.597854: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:15.597745: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:23:15.597874: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:15.597751: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:15.597908: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:23:15.597903: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:15.597765: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:15.597922: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:23:15.597910: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:15.597769: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:15.597951: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:23:15.597919: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:15.597773: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:15.597935: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:23:15.597891: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:15.597778: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:15.597955: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:23:15.597923: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:15.597782: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:15.597962: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:23:15.597916: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:15.598280: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-08 11:23:15.597963: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:23:15.598415: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-08 11:23:15.598299: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-08 11:23:15.597972: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:23:15.598435: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-08 11:23:15.598313: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-08 11:23:15.598433: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-08 11:23:15.598446: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-08 11:23:15.598318: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-08 11:23:15.598450: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-08 11:23:15.598456: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-08 11:23:15.598461: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-08 11:23:15.598462: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-08 11:23:15.598480: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-08 11:23:15.598479: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-08 11:23:15.598483: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-08 11:23:15.598467: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-08 11:23:15.598472: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-08 11:23:15.598494: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-08 11:23:15.598326: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-08 11:23:15.598334: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +12: 2023-05-08 11:23:15.598473: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-08 11:23:15.598335: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +16: 2023-05-08 11:23:15.598343: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +23: 2023-05-08 11:23:15.598498: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-08 11:23:15.599654: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:15.599674: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:15.599695: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:15.599699: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:15.599711: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:15.599685: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:15.599710: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:15.599717: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:15.600139: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-08 11:23:15.600159: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-08 11:23:15.600189: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-08 11:23:15.600207: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-08 11:23:15.600219: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-08 11:23:15.600226: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-08 11:23:15.600232: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +22: 2023-05-08 11:23:15.600240: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-08 11:23:15.565333: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:23:15.565355: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:23:15.565351: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:23:15.566216: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-08 11:23:15.566237: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-08 11:23:15.566249: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-08 11:23:15.566271: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-08 11:23:15.566276: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-08 11:23:15.566286: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-08 11:23:15.566291: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 0: 2023-05-08 11:23:15.566295: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-08 11:23:15.608139: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:15.608169: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:15.608160: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:15.608189: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:15.608162: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:15.608204: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:23:15.608290: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:15.608182: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:15.608219: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:23:15.608311: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:15.608192: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:15.608219: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:23:15.608324: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:15.608182: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:15.608229: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:23:15.608339: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:15.608195: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:15.608236: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:23:15.608346: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:15.608189: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:15.608228: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:23:15.608348: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:15.608830: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-08 11:23:15.608848: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-08 11:23:15.608883: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-08 11:23:15.608358: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:15.608902: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-08 11:23:15.608363: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:15.608860: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-08 11:23:15.608872: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-08 11:23:15.608914: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-08 11:23:15.608884: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-08 11:23:15.608886: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-08 11:23:15.608889: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +21: 2023-05-08 11:23:15.608890: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-08 11:23:15.608926: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-08 11:23:15.608933: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-08 11:23:15.608937: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-08 11:23:15.608939: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +17: 2023-05-08 11:23:15.608956: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-08 11:23:15.608935: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-08 11:23:15.608951: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-08 11:23:15.608962: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-08 11:23:15.608982: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-08 11:23:15.608995: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-08 11:23:15.608999: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-08 11:23:15.609005: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 3: 2023-05-08 11:23:15.609023: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-08 11:23:15.624279: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:15.624303: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:15.624322: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:15.624333: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:15.624353: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:15.624371: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:15.624377: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:15.624359: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:15.625183: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-08 11:23:15.625202: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-08 11:23:15.625216: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-08 11:23:15.625219: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-08 11:23:15.625230: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-08 11:23:15.625238: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-08 11:23:15.625242: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +10: 2023-05-08 11:23:15.625250: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-08 11:23:15.625632: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:15.625707: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:15.625718: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:15.625737: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:15.625732: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:15.625673: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:15.625759: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:15.625747: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:15.625777: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:15.625776: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:15.625810: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:15.625824: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:15.625825: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:15.625852: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:15.625859: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:23:15.625821: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:15.625871: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:23:15.625854: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:23:15.625864: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:23:15.625930: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:23:15.625882: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:23:15.625896: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:23:15.625910: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:23:15.625917: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:23:15.625932: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:23:15.625966: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:23:15.625995: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:23:15.625990: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:23:15.626015: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:15.625949: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:23:15.626019: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:15.626003: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:23:15.626031: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:23:15.626039: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:15.625978: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:15.626035: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:15.626003: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:15.626056: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:15.626021: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:23:15.626061: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:15.626069: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:15.626039: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:15.626096: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:15.625996: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:15.626104: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:15.626040: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:15.626114: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:15.626052: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:15.626117: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:15.626065: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:23:15.626092: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:23:15.626124: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:23:15.626155: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:23:15.626163: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:15.626094: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:23:15.626174: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:15.626121: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:23:15.626199: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:15.626149: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:23:15.626169: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:15.626156: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:15.626179: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:15.626167: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:15.626184: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:15.626231: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:15.626260: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:15.626274: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:15.626283: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:15.626313: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:23:15.626201: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:15.626322: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:15.626341: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:15.626342: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:23:15.626233: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:23:15.626257: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:23:15.626288: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:23:15.626300: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:23:15.626319: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:23:15.626303: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:23:15.626330: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:15.626226: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:23:15.626235: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:23:15.626266: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:15.626256: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:23:15.626306: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:15.626304: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:23:15.626328: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:15.626263: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:15.626325: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:23:15.626331: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:15.626354: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:23:15.626293: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:15.626296: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:15.626360: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:23:15.626357: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:15.626319: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:15.626376: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:23:15.626348: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:15.626336: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:15.626374: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:15.626360: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:15.626234: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:15.626354: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:15.626360: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:15.626375: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:15.626267: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:15.626295: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:15.626336: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:15.626342: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:15.626358: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:15.626364: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:15.626371: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:15.631494: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-08 11:23:15.631525: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-08 11:23:15.631569: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-08 11:23:15.631586: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-08 11:23:15.631596: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-08 11:23:15.631543: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-08 11:23:15.631629: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-08 11:23:15.631658: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-08 11:23:15.631555: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-08 11:23:15.631594: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-08 11:23:15.631582: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-08 11:23:15.631589: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-08 11:23:15.631627: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-08 11:23:15.631644: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-08 11:23:15.631655: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-08 11:23:15.631690: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-08 11:23:15.631720: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-08 11:23:15.631663: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-08 11:23:15.631651: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-08 11:23:15.631704: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-08 11:23:15.631685: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-08 11:23:15.631720: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-08 11:23:15.631625: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-08 11:23:15.631632: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-08 11:23:15.631739: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-08 11:23:15.631557: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-08 11:23:15.631611: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-08 11:23:15.631627: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-08 11:23:15.631729: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-08 11:23:15.631735: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-08 11:23:15.631648: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-08 11:23:15.631736: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-08 11:23:15.631773: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-08 11:23:15.631569: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-08 11:23:15.631676: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-08 11:23:15.631687: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-08 11:23:15.631745: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-08 11:23:15.631756: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-08 11:23:15.631777: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-08 11:23:15.631759: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-08 11:23:15.631655: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-08 11:23:15.631799: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-08 11:23:15.631826: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-08 11:23:15.631592: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-08 11:23:15.631599: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-08 11:23:15.631607: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-08 11:23:15.631704: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-08 11:23:15.631712: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-08 11:23:15.631606: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-08 11:23:15.631684: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-08 11:23:15.631693: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-08 11:23:15.631696: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-08 11:23:15.631772: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-08 11:23:15.631804: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-08 11:23:15.631737: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-08 11:23:15.631749: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-08 11:23:15.631791: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-08 11:23:15.631785: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-08 11:23:15.631796: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +18: 2023-05-08 11:23:15.631828: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-08 11:23:15.631845: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-08 11:23:15.631762: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-08 11:23:15.631879: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-08 11:23:15.631629: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-08 11:23:15.631645: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-08 11:23:15.631635: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-08 11:23:15.631702: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-08 11:23:15.631827: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-08 11:23:15.631775: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-08 11:23:15.631777: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-08 11:23:15.631803: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-08 11:23:15.631747: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-08 11:23:15.631760: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-08 11:23:15.631860: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-08 11:23:15.631878: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-08 11:23:15.631880: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-08 11:23:15.631883: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-08 11:23:15.631773: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-08 11:23:15.631790: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-08 11:23:15.631795: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 4: 2023-05-08 11:23:15.631806: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-08 11:23:15.631661: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-08 11:23:15.631642: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +27: 2023-05-08 11:23:15.631801: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-08 11:23:15.631856: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-08 11:23:15.631867: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-08 11:23:15.631871: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-08 11:23:15.631966: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 2: 2023-05-08 11:23:15.631985: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 5: 2023-05-08 11:23:15.632011: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-08 11:23:15.631819: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-08 11:23:15.631776: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-08 11:23:15.631803: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-08 11:23:15.631807: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-08 11:23:15.631822: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-08 11:23:15.631902: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-08 11:23:15.631668: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-08 11:23:15.631881: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-08 11:23:15.631795: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +20: 2023-05-08 11:23:15.632084: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-08 11:23:15.631815: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-08 11:23:15.631804: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-08 11:23:15.631833: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +26: 2023-05-08 11:23:15.631866: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +25: 2023-05-08 11:23:15.631840: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +15: 2023-05-08 11:23:15.632060: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 8: 2023-05-08 11:23:15.632041: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-08 11:23:15.631805: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +29: 2023-05-08 11:23:15.631835: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-08 11:23:15.631861: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +14: 2023-05-08 11:23:15.632001: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +28: 2023-05-08 11:23:15.632057: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-08 11:23:15.626295: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:23:15.626327: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:23:15.626341: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:23:15.626363: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:23:15.626355: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:15.627042: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:15.626270: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:23:15.626383: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:23:15.629459: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:23:15.626385: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:15.626312: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:15.627068: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:15.626338: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:15.627090: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:15.626363: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:23:15.629491: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:15.627104: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:15.626417: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:23:15.626399: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:23:15.629514: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:15.627117: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:15.626428: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:23:15.629525: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:15.627134: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:15.626439: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:23:15.629542: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:15.627119: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:15.626444: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:23:15.629558: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:15.627149: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:15.632721: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-08 11:23:15.632744: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-08 11:23:15.629586: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:23:15.629589: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:23:15.632777: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-08 11:23:15.632788: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-08 11:23:15.632790: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-08 11:23:15.632800: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-08 11:23:15.632814: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-08 11:23:15.632757: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-08 11:23:15.632824: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-08 11:23:15.632851: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-08 11:23:15.632870: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-08 11:23:15.632883: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-08 11:23:15.632891: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-08 11:23:15.632897: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-08 11:23:15.632914: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 6: 2023-05-08 11:23:15.632911: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-08 11:23:15.632781: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-08 11:23:15.632800: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-08 11:23:15.632815: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-08 11:23:15.632831: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-08 11:23:15.632996: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +31: 2023-05-08 11:23:15.633004: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-08 11:23:15.632845: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +30: 2023-05-08 11:23:15.632933: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-08 11:23:15.632863: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-08 11:23:15.632893: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-08 11:23:15.632917: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-08 11:23:15.632929: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-08 11:23:15.632941: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-08 11:23:15.632957: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-08 11:23:15.632963: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. + 7: 2023-05-08 11:23:15.633109: I tensorflow/compiler/xla/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine. +19: 2023-05-08 11:23:45.270347: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:45.270375: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:45.270394: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:45.270404: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:45.270424: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:45.270431: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:45.270417: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:45.270626: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:45.274328: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:45.274334: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:45.274334: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:45.274335: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:45.274335: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:45.274337: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:45.274337: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:45.274337: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +19: 2023-05-08 11:23:45.274347: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-08 11:23:45.274346: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-08 11:23:45.274351: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-08 11:23:45.274353: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-08 11:23:45.274352: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-08 11:23:45.274357: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-08 11:23:45.274357: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +19: 2023-05-08 11:23:45.274356: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-08 11:23:45.278672: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:45.278685: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:45.278719: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:45.278723: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:45.278732: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:45.278742: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:45.278751: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:45.278913: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:45.286184: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:45.286189: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:45.286189: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:45.286194: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:45.286193: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:45.286192: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:45.286213: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-08 11:23:45.286205: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:45.286204: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +27: 2023-05-08 11:23:45.286224: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-08 11:23:45.286222: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-08 11:23:45.286227: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-08 11:23:45.286232: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-08 11:23:45.286231: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-08 11:23:45.286237: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +27: 2023-05-08 11:23:45.286238: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-08 11:23:45.287031: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:23:45.287069: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:23:45.287106: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:23:45.287125: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:23:45.287152: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:23:45.287160: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:45.287336: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:23:45.287338: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:23:45.287344: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:45.287367: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:45.287382: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:45.287404: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:45.287416: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:45.287613: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:45.287617: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:45.287625: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:45.288247: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:45.288274: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:45.288297: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:45.288309: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:45.288301: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:45.288304: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:45.288464: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:45.288487: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:45.288613: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:45.288615: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:45.288616: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:45.288617: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:45.288620: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:45.288670: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:45.288622: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:45.288622: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:45.288622: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:45.288636: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-08 11:23:45.288634: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-08 11:23:45.288635: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-08 11:23:45.288637: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-08 11:23:45.288781: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 2: 2023-05-08 11:23:45.288637: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-08 11:23:45.288640: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-08 11:23:45.288652: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 2: 2023-05-08 11:23:45.288651: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-08 11:23:45.288702: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:45.288726: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:45.288734: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:45.288740: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:45.288746: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:45.288762: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:45.288764: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:45.288770: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:45.288970: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:45.288787: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:45.288808: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:45.288839: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:45.288841: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:45.288850: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:45.289071: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:45.289229: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:45.289274: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:45.289295: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:45.289309: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:45.289316: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:45.289325: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:45.289517: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:45.289521: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:45.289725: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:45.289725: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:45.289727: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:45.289725: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:45.289728: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:45.289728: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:45.289730: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:45.289729: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:45.289742: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-08 11:23:45.289752: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +21: 2023-05-08 11:23:45.289746: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-08 11:23:45.289747: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-08 11:23:45.289748: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-08 11:23:45.289749: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-08 11:23:45.289750: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-08 11:23:45.289752: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +21: 2023-05-08 11:23:45.289753: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-08 11:23:45.289909: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:23:45.289785: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:23:45.289815: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:45.289911: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:23:45.289829: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:45.289915: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:23:45.289842: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:45.289915: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:23:45.289848: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:45.289916: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:23:45.290027: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:45.289918: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:45.289934: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-08 11:23:45.289937: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-08 11:23:45.289940: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-08 11:23:45.289941: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-08 11:23:45.290051: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:45.289947: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-08 11:23:45.289946: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-08 11:23:45.289981: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:45.289998: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +20: 2023-05-08 11:23:45.290007: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +20: 2023-05-08 11:23:45.290022: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-08 11:23:45.290187: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:45.290187: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:45.290196: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:45.290206: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-08 11:23:45.290207: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-08 11:23:45.290211: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-08 11:23:45.290200: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:45.290201: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:45.290155: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:45.290207: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:45.290206: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:45.290209: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +25: 2023-05-08 11:23:45.290231: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-08 11:23:45.290232: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-08 11:23:45.290233: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-08 11:23:45.290234: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +25: 2023-05-08 11:23:45.290233: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-08 11:23:45.290186: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:45.290196: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:45.290205: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:45.290244: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:45.290252: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:45.290263: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:45.290279: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:45.290460: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:45.290494: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:45.290508: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:45.290531: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:45.290555: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:45.290562: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:45.290583: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:45.290769: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:45.290632: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:45.290644: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:45.290676: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:45.290937: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:45.290684: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:45.290693: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:45.290873: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:45.291028: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:45.290973: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:45.290869: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:45.290984: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:45.290870: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:45.291005: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:45.291014: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:45.291035: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:45.291204: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:45.291225: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:23:45.291370: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:45.291431: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:45.291462: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:45.291504: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:45.291512: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:45.291532: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:45.291536: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:45.291741: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:45.291745: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:45.291902: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:23:45.291684: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:45.291904: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:45.291865: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:45.291905: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:45.291908: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:45.291882: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:23:45.291711: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:45.291911: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:45.291916: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:23:45.291736: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:45.292037: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:45.291911: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:45.291915: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:23:45.291743: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:45.292043: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:45.291911: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:45.291928: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-08 11:23:45.291930: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:23:45.291749: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:45.292045: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:45.291929: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-08 11:23:45.291929: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-08 11:23:45.291930: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-08 11:23:45.291944: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:23:45.291758: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:45.292043: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:45.291934: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-08 11:23:45.291934: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +30: 2023-05-08 11:23:45.291935: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-08 11:23:45.291948: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:23:45.291769: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:45.292047: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:45.292005: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +30: 2023-05-08 11:23:45.292022: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:23:45.291977: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:45.292046: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:45.292054: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-08 11:23:45.292056: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-08 11:23:45.292048: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:45.292050: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +22: 2023-05-08 11:23:45.292062: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-08 11:23:45.292063: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-08 11:23:45.292064: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-08 11:23:45.292065: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-08 11:23:45.292068: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +22: 2023-05-08 11:23:45.292068: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-08 11:23:45.292148: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:45.292177: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:45.292179: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:45.292180: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:45.292182: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:45.292181: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:45.292184: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:45.292189: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:45.292197: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-08 11:23:45.292196: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-08 11:23:45.292199: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-08 11:23:45.292199: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-08 11:23:45.292200: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-08 11:23:45.292201: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-08 11:23:45.292205: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +11: 2023-05-08 11:23:45.292247: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +11: 2023-05-08 11:23:45.292259: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-08 11:23:45.292123: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:45.292360: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:45.292390: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:23:45.292150: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:45.292422: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:23:45.292173: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:45.292451: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:23:45.292184: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:45.292457: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:23:45.292201: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:45.292476: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:23:45.292202: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:45.292480: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:23:45.292227: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:45.292480: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:23:45.292445: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:45.292593: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:45.292719: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:45.292641: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:45.292634: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:45.292666: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:45.292674: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:45.292684: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:45.292691: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:45.292767: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:45.292879: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:45.292805: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:45.292823: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:45.292845: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:45.292857: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:45.292867: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:45.292877: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:45.293050: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:45.293114: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:45.293354: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:45.293361: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:45.293363: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:45.293376: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:23:45.293504: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:45.293377: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-08 11:23:45.293378: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:23:45.293507: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:45.293371: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:23:45.293506: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:45.293373: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:23:45.293508: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:45.293373: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:23:45.293508: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:45.293377: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:45.293405: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:23:45.293512: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:45.293407: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-08 11:23:45.293409: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 1: 2023-05-08 11:23:45.293410: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:23:45.293510: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:45.293434: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 1: 2023-05-08 11:23:45.293447: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:23:45.293516: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +12: 2023-05-08 11:23:45.293520: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:23:45.293526: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:23:45.293527: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:23:45.293527: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:23:45.293527: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:23:45.293529: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:23:45.293530: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +12: 2023-05-08 11:23:45.293531: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-08 11:23:45.293830: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:23:45.293846: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:45.293942: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:23:45.293858: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:23:45.293899: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:23:45.293868: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:23:45.293873: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:45.293947: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:23:45.293876: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:23:45.293900: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:45.293953: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:23:45.293885: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:23:45.293900: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:45.293949: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:23:45.293893: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:23:45.293901: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:45.293948: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:23:45.293907: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:45.293953: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:45.293972: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-08 11:23:45.293907: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:45.293962: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:45.293976: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-08 11:23:45.293908: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +13: 2023-05-08 11:23:45.293979: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-08 11:23:45.293979: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-08 11:23:45.293982: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-08 11:23:45.293913: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:23:45.293918: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-08 11:23:45.293979: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-08 11:23:45.293987: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-08 11:23:45.293919: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-08 11:23:45.293920: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-08 11:23:45.293921: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-08 11:23:45.294077: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +14: 2023-05-08 11:23:45.293924: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-08 11:23:45.293926: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-08 11:23:45.293928: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +13: 2023-05-08 11:23:45.294092: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +14: 2023-05-08 11:23:45.293929: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-08 11:23:45.294311: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:45.294316: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:23:45.294336: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:45.294319: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:23:45.294364: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:45.294318: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:23:45.294358: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:45.294322: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:45.294328: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-08 11:23:45.294376: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:45.294323: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:23:45.294384: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:45.294322: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:45.294333: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-08 11:23:45.294390: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:45.294340: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-08 11:23:45.294332: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:23:45.294392: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:45.294339: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-08 11:23:45.294341: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-08 11:23:45.294340: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-08 11:23:45.294401: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +23: 2023-05-08 11:23:45.294343: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +23: 2023-05-08 11:23:45.294355: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-08 11:23:45.294538: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:45.294540: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:45.294538: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:45.294540: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:45.294541: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:45.294542: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:45.294543: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:45.294544: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +17: 2023-05-08 11:23:45.294559: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-08 11:23:45.294560: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-08 11:23:45.294561: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-08 11:23:45.294563: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-08 11:23:45.294562: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-08 11:23:45.294563: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-08 11:23:45.294564: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +17: 2023-05-08 11:23:45.294565: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-08 11:23:45.294861: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:23:45.294861: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:23:45.294863: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:23:45.294860: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:23:45.294869: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:23:45.294870: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:23:45.294869: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:23:45.294872: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 7: 2023-05-08 11:23:45.294883: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-08 11:23:45.294883: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-08 11:23:45.294885: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-08 11:23:45.294883: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-08 11:23:45.294887: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-08 11:23:45.294888: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-08 11:23:45.294889: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 7: 2023-05-08 11:23:45.294889: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-08 11:23:45.295660: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:23:45.295660: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:23:45.295659: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:23:45.295661: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:23:45.295660: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:23:45.295659: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:23:45.295664: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:23:45.295667: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +18: 2023-05-08 11:23:45.295678: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-08 11:23:45.295678: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-08 11:23:45.295681: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-08 11:23:45.295682: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-08 11:23:45.295681: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-08 11:23:45.295684: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-08 11:23:45.295684: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +18: 2023-05-08 11:23:45.295682: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-08 11:23:45.307616: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:45.307652: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:45.307673: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:45.307686: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:45.307696: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:45.307701: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:45.307715: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:45.307869: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:45.307799: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:45.307893: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:45.307901: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:45.307916: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:45.307924: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:45.307928: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:45.307942: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:45.307945: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:23:45.308796: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:45.308736: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:23:45.308829: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:45.308722: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:45.308736: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-08 11:23:45.308738: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:45.308752: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-08 11:23:45.308845: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:45.308734: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:45.308744: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:23:45.308865: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:45.308738: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:45.308744: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:23:45.308884: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:45.308734: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:45.308745: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:23:45.308898: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:45.308739: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:45.308744: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:23:45.309089: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:45.308734: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:45.308760: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-08 11:23:45.308751: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:23:45.309166: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:45.308739: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:45.308753: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +29: 2023-05-08 11:23:45.308769: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-08 11:23:45.309197: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:45.308741: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:45.308760: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-08 11:23:45.308768: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-08 11:23:45.308771: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-08 11:23:45.308771: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-08 11:23:45.309198: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:45.308761: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-08 11:23:45.308762: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-08 11:23:45.308763: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-08 11:23:45.308773: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +29: 2023-05-08 11:23:45.308775: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-08 11:23:45.309206: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 5: 2023-05-08 11:23:45.308763: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-08 11:23:45.308765: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 5: 2023-05-08 11:23:45.308765: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-08 11:23:45.309208: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:23:45.309220: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:23:45.309231: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:23:45.309333: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:45.310089: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:45.310114: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:45.310122: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:45.310150: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:45.310154: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:23:45.310198: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:45.310159: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:23:45.310201: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:45.310162: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:23:45.310199: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:45.310193: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:23:45.310199: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:23:45.310204: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:23:45.310199: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:23:45.310203: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:23:45.310207: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +28: 2023-05-08 11:23:45.310218: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-08 11:23:45.310219: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-08 11:23:45.310222: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-08 11:23:45.310222: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-08 11:23:45.310224: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-08 11:23:45.310225: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-08 11:23:45.310224: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +28: 2023-05-08 11:23:45.310225: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-08 11:23:45.310472: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:45.310509: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:45.310528: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:45.310547: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:45.310562: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:45.310565: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:45.310588: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:45.310594: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:45.310693: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:45.310725: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:45.310736: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:45.310862: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:45.310758: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:45.310864: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:45.310764: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:45.310863: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:45.310768: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:45.310867: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:45.310900: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:45.310868: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:45.310876: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-08 11:23:45.310915: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:45.310870: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:45.310870: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:45.310870: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +16: 2023-05-08 11:23:45.310883: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-08 11:23:45.310885: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-08 11:23:45.310886: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-08 11:23:45.310886: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-08 11:23:45.310889: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-08 11:23:45.310889: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +16: 2023-05-08 11:23:45.310890: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-08 11:23:45.311453: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:45.311432: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:23:45.311473: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:45.311431: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:23:45.311479: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:45.311432: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:45.311446: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-08 11:23:45.311493: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:45.311434: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:23:45.311501: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:45.311437: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:23:45.311501: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:45.311439: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:23:45.311508: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:45.311440: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:45.311458: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-08 11:23:45.311627: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:45.311458: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-08 11:23:45.311460: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-08 11:23:45.311461: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-08 11:23:45.311462: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +15: 2023-05-08 11:23:45.311463: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-08 11:23:45.311594: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:45.311608: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-08 11:23:45.311597: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:45.311520: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +15: 2023-05-08 11:23:45.311537: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-08 11:23:45.311619: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:45.311600: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:23:45.311638: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:45.311601: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:23:45.311646: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:45.311599: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:23:45.311657: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:45.311604: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:23:45.311659: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:45.311603: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:23:45.311783: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:45.311608: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:45.311626: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-08 11:23:45.311792: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:45.311627: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-08 11:23:45.311628: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-08 11:23:45.311627: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-08 11:23:45.311628: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-08 11:23:45.311629: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 9: 2023-05-08 11:23:45.311675: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 9: 2023-05-08 11:23:45.311689: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-08 11:23:45.312029: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:45.312061: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:45.312054: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:45.312076: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:45.312090: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:45.312085: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:45.312093: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:45.312104: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:23:45.312290: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:23:45.312293: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:23:45.312295: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:23:45.312299: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:23:45.312300: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:23:45.312301: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:23:45.312303: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:23:45.312325: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-08 11:23:45.312324: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-08 11:23:45.312327: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-08 11:23:45.312327: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-08 11:23:45.312327: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-08 11:23:45.312328: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-08 11:23:45.312327: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 8: 2023-05-08 11:23:45.312377: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 8: 2023-05-08 11:23:45.312397: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-08 11:23:45.312907: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:45.313079: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:45.313079: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:45.313078: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:45.313084: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:45.313082: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:45.313083: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:45.313089: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:45.313083: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 6: 2023-05-08 11:23:45.313097: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-08 11:23:45.313099: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-08 11:23:45.313100: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-08 11:23:45.313100: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-08 11:23:45.313101: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-08 11:23:45.313103: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-08 11:23:45.313104: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 6: 2023-05-08 11:23:45.313102: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-08 11:23:45.291031: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:23:45.288781: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:23:45.291370: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:45.291031: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:23:45.288784: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:23:45.291376: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:45.291035: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:23:45.288784: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:23:45.291376: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:45.291034: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:23:45.288784: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:23:45.291377: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:45.291039: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:23:45.288786: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:23:45.291375: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:45.291041: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:23:45.288788: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:23:45.291379: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:45.291040: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +10: 2023-05-08 11:23:45.291049: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-08 11:23:45.288787: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +31: 2023-05-08 11:23:45.288799: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-08 11:23:45.291383: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +26: 2023-05-08 11:23:45.291393: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-08 11:23:45.291048: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-08 11:23:45.291051: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-08 11:23:45.291049: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-08 11:23:45.288799: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-08 11:23:45.288800: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-08 11:23:45.288801: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-08 11:23:45.291394: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-08 11:23:45.291395: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-08 11:23:45.291393: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-08 11:23:45.291053: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-08 11:23:45.291052: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-08 11:23:45.291060: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-08 11:23:45.288803: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-08 11:23:45.288803: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-08 11:23:45.288806: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-08 11:23:45.291396: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-08 11:23:45.291395: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-08 11:23:45.291397: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +10: 2023-05-08 11:23:45.291060: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +31: 2023-05-08 11:23:45.288806: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +26: 2023-05-08 11:23:45.291406: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-08 11:23:45.293115: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:45.292722: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:45.293118: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:45.292724: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:45.293116: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:45.292731: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:45.293122: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:45.292730: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:45.293126: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:45.292730: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:45.293125: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:45.292730: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:45.293128: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 +24: 2023-05-08 11:23:45.293133: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-08 11:23:45.292735: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 4: 2023-05-08 11:23:45.292738: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-08 11:23:45.293133: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-08 11:23:45.293133: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-08 11:23:45.293138: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-08 11:23:45.292738: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-08 11:23:45.292742: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-08 11:23:45.292746: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-08 11:23:45.293136: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-08 11:23:45.293142: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-08 11:23:45.293144: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-08 11:23:45.292749: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-08 11:23:45.292749: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-08 11:23:45.292752: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. +24: 2023-05-08 11:23:45.293145: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 4: 2023-05-08 11:23:45.292751: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-08 11:23:45.309151: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:23:45.309440: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:23:45.309452: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:23:45.309459: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-08 11:23:45.309467: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-08 11:23:45.309483: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:23:45.309502: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-08 11:23:45.309585: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:23:45.309586: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:23:45.309586: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:23:45.309589: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:23:45.309588: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 0: 2023-05-08 11:23:45.309601: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-08 11:23:45.309603: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-08 11:23:45.309604: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-08 11:23:45.309605: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: 2023-05-08 11:23:45.309605: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-08 11:23:45.312907: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:23:45.312910: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:23:45.312910: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:23:45.312911: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:23:45.312911: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:23:45.312912: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:23:45.312918: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /pfs/lustrep2/projappl/project_462000125/samantao-public/apps-rocm-5.2.3/aws-ofi-rccl:/opt/rocm/lib64:/opt/rocm/lib:/opt/rocm/rocprofiler/lib:/opt/rocm/rocprofiler/tool:/opt/rocm/roctracer/lib:/opt/rocm/roctracer/tool:/opt/rocm/hip/lib:/opt/cray/pe/python/3.9.13.1/lib:/opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + 3: 2023-05-08 11:23:45.312925: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-08 11:23:45.312926: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-08 11:23:45.312925: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-08 11:23:45.312931: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-08 11:23:45.312932: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-08 11:23:45.312933: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-08 11:23:45.312932: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 3: 2023-05-08 11:23:45.312934: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly. + 0: Successfully preprocessed all matching files. + 0: Detected CUDA files, patching ldflags + 0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... + 0: Building extension module scaled_upper_triang_masked_softmax_cuda... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module scaled_upper_triang_masked_softmax_cuda... + 0: Successfully preprocessed all matching files. + 0: Detected CUDA files, patching ldflags + 0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... + 0: Building extension module scaled_masked_softmax_cuda... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module scaled_masked_softmax_cuda... + 0: Successfully preprocessed all matching files. + 0: Detected CUDA files, patching ldflags + 0: Emitting ninja build file /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/build/build.ninja... + 0: Building extension module fused_mix_prec_layer_norm_cuda... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module fused_mix_prec_layer_norm_cuda... + 9: Successfully preprocessed all matching files. + 9: Successfully preprocessed all matching files. + 9: Successfully preprocessed all matching files. + 2: Successfully preprocessed all matching files. + 0: Successfully preprocessed all matching files. + 0: Successfully preprocessed all matching files. +14: Successfully preprocessed all matching files. +14: Successfully preprocessed all matching files. +16: Successfully preprocessed all matching files. +16: Successfully preprocessed all matching files. +16: Successfully preprocessed all matching files. +24: Successfully preprocessed all matching files. +24: Successfully preprocessed all matching files. +12: Successfully preprocessed all matching files. +12: Successfully preprocessed all matching files. + 2: Successfully preprocessed all matching files. +17: Successfully preprocessed all matching files. + 4: Successfully preprocessed all matching files. + 4: Successfully preprocessed all matching files. +29: Successfully preprocessed all matching files. +29: Successfully preprocessed all matching files. +23: Successfully preprocessed all matching files. +23: Successfully preprocessed all matching files. +25: Successfully preprocessed all matching files. +25: Successfully preprocessed all matching files. +25: Successfully preprocessed all matching files. +25: Successfully preprocessed all matching files. +14: Successfully preprocessed all matching files. + 1: Successfully preprocessed all matching files. + 7: Successfully preprocessed all matching files. + 7: Successfully preprocessed all matching files. +22: Successfully preprocessed all matching files. +22: Successfully preprocessed all matching files. +22: Successfully preprocessed all matching files. +10: Successfully preprocessed all matching files. + 8: Successfully preprocessed all matching files. + 8: Successfully preprocessed all matching files. +20: Successfully preprocessed all matching files. +20: Successfully preprocessed all matching files. + 7: Successfully preprocessed all matching files. +20: Successfully preprocessed all matching files. +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +25: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +25: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +28: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +28: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( +31: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +31: warnings.warn( + 7: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 7: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +23: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +23: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( +13: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +13: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 8: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 8: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( + 3: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 3: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( + 5: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 5: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +22: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +22: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +26: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +26: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +24: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +24: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +18: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +18: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +30: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +30: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +17: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +17: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +15: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +15: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +16: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +16: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +20: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +20: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( +29: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +29: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( + 4: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 4: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +27: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +27: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 6: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 6: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( +14: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +14: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( + 9: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 9: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( + 1: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 1: warnings.warn( +19: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +19: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( +12: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +12: warnings.warn( +10: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +10: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 0: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( +11: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +11: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( + 2: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead + 2: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( +21: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/distributed/distributed_c10d.py:429: UserWarning: torch.distributed.distributed_c10d._get_global_rank is deprecated please use torch.distributed.distributed_c10d.get_global_rank instead +21: warnings.warn( + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: + 2: + 2: + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: + 3: + 3: +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: + 6: + 6: +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: +11: +11: +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: +29: +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: +13: +13: +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: +19: +19: + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: +27: +27: +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: +15: +15: +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: +21: +21: +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: +18: +18: +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: +28: +28: +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: +30: +30: +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: +31: +31: +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: +26: +26: +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... + 0: Building extension module utils... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Loading extension module utils... + 0: Loading extension module utils... + 0: Loading extension module utils... + 0: Loading extension module utils... +10: Loading extension module utils... +10: Loading extension module utils... +10: Loading extension module utils... +10: Loading extension module utils... + 2: Loading extension module utils... + 2: Loading extension module utils... + 2: Loading extension module utils... + 2: Loading extension module utils... + 4: Loading extension module utils... + 1: Loading extension module utils... + 4: Loading extension module utils... + 1: Loading extension module utils... + 4: Loading extension module utils... + 1: Loading extension module utils... + 8: Loading extension module utils... + 4: Loading extension module utils... + 1: Loading extension module utils... + 8: Loading extension module utils... + 8: Loading extension module utils... + 8: Loading extension module utils... + 9: Loading extension module utils... + 9: Loading extension module utils... + 9: Loading extension module utils... + 7: Loading extension module utils... + 3: Loading extension module utils... +14: Loading extension module utils... + 9: Loading extension module utils... + 7: Loading extension module utils... + 3: Loading extension module utils... +14: Loading extension module utils... + 7: Loading extension module utils... +14: Loading extension module utils... +17: Loading extension module utils... + 3: Loading extension module utils... + 7: Loading extension module utils... +11: Loading extension module utils... +14: Loading extension module utils... + 3: Loading extension module utils... +17: Loading extension module utils... +17: Loading extension module utils... +11: Loading extension module utils... +24: Loading extension module utils... +17: Loading extension module utils... +11: Loading extension module utils... +22: Loading extension module utils... +29: Loading extension module utils... + 6: Loading extension module utils... +11: Loading extension module utils... +25: Loading extension module utils... +20: Loading extension module utils... +24: Loading extension module utils... +22: Loading extension module utils... + 6: Loading extension module utils... +16: Loading extension module utils... +29: Loading extension module utils... +20: Loading extension module utils... +24: Loading extension module utils... +25: Loading extension module utils... + 6: Loading extension module utils... +22: Loading extension module utils... +16: Loading extension module utils... +20: Loading extension module utils... +24: Loading extension module utils... +29: Loading extension module utils... +25: Loading extension module utils... +23: Loading extension module utils... + 6: Loading extension module utils... +16: Loading extension module utils... +22: Loading extension module utils... +20: Loading extension module utils... +23: Loading extension module utils... +25: Loading extension module utils... +29: Loading extension module utils... +16: Loading extension module utils... +23: Loading extension module utils... +13: Loading extension module utils... +19: Loading extension module utils... +23: Loading extension module utils... +27: Loading extension module utils... +15: Loading extension module utils... +21: Loading extension module utils... +13: Loading extension module utils... +19: Loading extension module utils... +27: Loading extension module utils... +13: Loading extension module utils... +15: Loading extension module utils... +21: Loading extension module utils... +18: Loading extension module utils... +27: Loading extension module utils... +19: Loading extension module utils... +13: Loading extension module utils... +21: Loading extension module utils... +15: Loading extension module utils... +18: Loading extension module utils... +19: Loading extension module utils... +15: Loading extension module utils... +28: Loading extension module utils... +27: Loading extension module utils... +21: Loading extension module utils... +18: Loading extension module utils... +30: Loading extension module utils... +28: Loading extension module utils... +26: Loading extension module utils... +18: Loading extension module utils... +28: Loading extension module utils... +30: Loading extension module utils... +26: Loading extension module utils... +30: Loading extension module utils... +28: Loading extension module utils... +31: Loading extension module utils... +26: Loading extension module utils... +30: Loading extension module utils... +26: Loading extension module utils... +31: Loading extension module utils... +31: Loading extension module utils... +31: Loading extension module utils... +12: Loading extension module utils... +12: Loading extension module utils... +12: Loading extension module utils... +12: Loading extension module utils... + 5: Loading extension module utils... + 5: Loading extension module utils... + 5: Loading extension module utils... + 5: Loading extension module utils... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +15: +15: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +16: +16: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: + 6: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 6: + 6: Loading extension module utils... + 6: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 2: + 2: Loading extension module utils...Loading extension module utils... + 2: + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... + 4: + 4: Loading extension module utils... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +30: +30: Loading extension module utils... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +29: +29: Loading extension module utils... +29: Loading extension module utils... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 9: + 7: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 7: + 7: Loading extension module utils...Loading extension module utils... + 7: +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Loading extension module utils...Loading extension module utils... + 9: + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +25: +25: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... + 8: + 8: Loading extension module utils... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +11: +11: Loading extension module utils... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: No modifications detected for re-loaded extension module utils, skipping build step... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Loading extension module utils... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 3: + 3: Loading extension module utils...Loading extension module utils... + 3: +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +13: +13: Loading extension module utils... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +28: +28: Loading extension module utils... +28: Loading extension module utils... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +21: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +21: +21: Loading extension module utils...Loading extension module utils... +21: +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +23: +23: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +18: +18: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +31: +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... +31: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: + 1: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 1: + 1: Loading extension module utils...Loading extension module utils... + 1: + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +10: +10: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +26: +26: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +10: Loading extension module utils... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +19: +19: Loading extension module utils...Loading extension module utils... +19: + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 5: + 5: Loading extension module utils...Loading extension module utils... + 5: + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils...No modifications detected for re-loaded extension module utils, skipping build step... +27: +27: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: No modifications detected for re-loaded extension module utils, skipping build step... +20: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step... +20: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step... +20: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step... +20: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Emitting ninja build file /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu/utils/build.ninja... + 0: Building extension module utils... + 0: Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: Loading extension module utils... + 6: Loading extension module utils... +15: Loading extension module utils... + 2: Loading extension module utils... + 1: Loading extension module utils... +21: Loading extension module utils... +13: Loading extension module utils... + 7: Loading extension module utils... + 6: Loading extension module utils... +15: Loading extension module utils... + 2: Loading extension module utils... + 1: Loading extension module utils... +21: Loading extension module utils... +13: Loading extension module utils... + 4: Loading extension module utils... + 7: Loading extension module utils... + 6: Loading extension module utils... +11: Loading extension module utils... +10: Loading extension module utils... +10: Loading extension module utils... +15: Loading extension module utils... + 2: Loading extension module utils... + 1: Loading extension module utils... +21: Loading extension module utils... +13: Loading extension module utils... +11: Loading extension module utils... +11: Loading extension module utils... +11: Loading extension module utils... + 4: Loading extension module utils... +10: Loading extension module utils... +10: Loading extension module utils... + 7: Loading extension module utils... + 6: Loading extension module utils... + 3: Loading extension module utils... +15: Loading extension module utils... + 2: Loading extension module utils... + 1: Loading extension module utils... +12: Loading extension module utils... +21: Loading extension module utils... +13: Loading extension module utils... + 8: Loading extension module utils... +14: Loading extension module utils... + 5: Loading extension module utils... + 9: Loading extension module utils... +16: Loading extension module utils... + 4: Loading extension module utils... + 7: Loading extension module utils... + 3: Loading extension module utils... +22: Loading extension module utils... +17: Loading extension module utils... +12: Loading extension module utils... +24: Loading extension module utils... + 8: Loading extension module utils... + 0: Loading extension module utils... +14: Loading extension module utils... + 5: Loading extension module utils... +16: Loading extension module utils... + 9: Loading extension module utils... +23: Loading extension module utils... + 4: Loading extension module utils... +20: Loading extension module utils... +25: Loading extension module utils... +24: Loading extension module utils... + 3: Loading extension module utils... + 0: Loading extension module utils... +22: Loading extension module utils... +29: Loading extension module utils... +17: Loading extension module utils... +30: Loading extension module utils... + 9: Loading extension module utils... +12: Loading extension module utils... + 8: Loading extension module utils... + 8: Loading extension module utils... +14: Loading extension module utils... +14: Loading extension module utils... + 5: Loading extension module utils... +16: Loading extension module utils... +24: Loading extension module utils... + 0: Loading extension module utils... +23: Loading extension module utils... +20: Loading extension module utils... +25: Loading extension module utils... + 3: Loading extension module utils... +22: Loading extension module utils... +29: Loading extension module utils... +17: Loading extension module utils... +30: Loading extension module utils... +26: Loading extension module utils... + 9: Loading extension module utils... +12: Loading extension module utils... + 5: Loading extension module utils... +16: Loading extension module utils... +23: Loading extension module utils... +20: Loading extension module utils... +24: Loading extension module utils... +25: Loading extension module utils... +22: Loading extension module utils... +29: Loading extension module utils... +26: Loading extension module utils... +17: Loading extension module utils... +18: Loading extension module utils... +30: Loading extension module utils... +19: Loading extension module utils... +25: Loading extension module utils... +23: Loading extension module utils... +27: Loading extension module utils... +20: Loading extension module utils... +29: Loading extension module utils... +18: Loading extension module utils... +30: Loading extension module utils... +19: Loading extension module utils... +31: Loading extension module utils... +26: Loading extension module utils... +27: Loading extension module utils... +28: Loading extension module utils... +18: Loading extension module utils... +19: Loading extension module utils... +31: Loading extension module utils... +26: Loading extension module utils... +18: Loading extension module utils... +27: Loading extension module utils... +28: Loading extension module utils... +31: Loading extension module utils... +19: Loading extension module utils... +27: Loading extension module utils... +28: Loading extension module utils... +31: Loading extension module utils... +28: Loading extension module utils... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +11: +11: +11: Loading extension module utils...Loading extension module utils...Loading extension module utils... +11: +11: + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... +11: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +11: No modifications detected for re-loaded extension module utils, skipping build step... +11: Loading extension module utils... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... + 8: No modifications detected for re-loaded extension module utils, skipping build step... + 8: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... +10: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +10: No modifications detected for re-loaded extension module utils, skipping build step... +10: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... + 0: No modifications detected for re-loaded extension module utils, skipping build step... + 0: Loading extension module utils... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +15: +15: Loading extension module utils...Loading extension module utils... +15: +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: Loading extension module utils... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +19: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step... +20: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step... +19: Loading extension module utils... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... +19: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +19: +19: +19: Loading extension module utils...Loading extension module utils...Loading extension module utils... +19: +19: +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... +12: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +12: +12: +12: Loading extension module utils...Loading extension module utils...Loading extension module utils... +12: +12: +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... + 6: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 6: + 6: Loading extension module utils...Loading extension module utils... + 6: + 2: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 2: No modifications detected for re-loaded extension module utils, skipping build step... + 2: Loading extension module utils... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... +20: No modifications detected for re-loaded extension module utils, skipping build step... +20: Loading extension module utils... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 4: No modifications detected for re-loaded extension module utils, skipping build step... + 4: Loading extension module utils... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: No modifications detected for re-loaded extension module utils, skipping build step... +20: Loading extension module utils... +20: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +20: No modifications detected for re-loaded extension module utils, skipping build step... +20: Loading extension module utils... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +21: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +21: No modifications detected for re-loaded extension module utils, skipping build step... +21: Loading extension module utils... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 1: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +15: No modifications detected for re-loaded extension module utils, skipping build step... +15: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... + 1: No modifications detected for re-loaded extension module utils, skipping build step... + 1: Loading extension module utils... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +13: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...No modifications detected for re-loaded extension module utils, skipping build step... +13: +13: Loading extension module utils... +13: No modifications detected for re-loaded extension module utils, skipping build step... +13: Loading extension module utils... +30: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +30: +30: +30: Loading extension module utils... +30: Loading extension module utils...Loading extension module utils... +30: + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +25: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: No modifications detected for re-loaded extension module utils, skipping build step... +25: No modifications detected for re-loaded extension module utils, skipping build step... +25: Loading extension module utils... +14: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: Loading extension module utils... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +14: No modifications detected for re-loaded extension module utils, skipping build step... +14: Loading extension module utils... + 7: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 7: No modifications detected for re-loaded extension module utils, skipping build step... + 7: Loading extension module utils... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +30: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +30: No modifications detected for re-loaded extension module utils, skipping build step... +30: Loading extension module utils... + 6: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 6: No modifications detected for re-loaded extension module utils, skipping build step... + 6: Loading extension module utils... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... + 9: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 9: No modifications detected for re-loaded extension module utils, skipping build step... + 9: Loading extension module utils... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +12: No modifications detected for re-loaded extension module utils, skipping build step... +12: Loading extension module utils... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: No modifications detected for re-loaded extension module utils, skipping build step... + 3: Loading extension module utils... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +16: No modifications detected for re-loaded extension module utils, skipping build step... +16: Loading extension module utils... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... + 3: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root...Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 3: + 3: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... + 3: + 3: Loading extension module utils...Loading extension module utils... + 3: +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +24: No modifications detected for re-loaded extension module utils, skipping build step... +24: Loading extension module utils... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: No modifications detected for re-loaded extension module utils, skipping build step...Loading extension module utils... +23: +23: Loading extension module utils... +23: No modifications detected for re-loaded extension module utils, skipping build step... +23: Loading extension module utils... + 5: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... + 5: No modifications detected for re-loaded extension module utils, skipping build step... + 5: Loading extension module utils... +28: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +28: No modifications detected for re-loaded extension module utils, skipping build step... +28: Loading extension module utils... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +17: No modifications detected for re-loaded extension module utils, skipping build step... +17: Loading extension module utils... +27: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +27: +27: Loading extension module utils...Loading extension module utils... +27: +22: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +22: No modifications detected for re-loaded extension module utils, skipping build step... +22: Loading extension module utils... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +29: +29: Loading extension module utils...Loading extension module utils... +29: +26: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +26: No modifications detected for re-loaded extension module utils, skipping build step... +26: Loading extension module utils... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +18: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +18: No modifications detected for re-loaded extension module utils, skipping build step... +18: Loading extension module utils... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +27: No modifications detected for re-loaded extension module utils, skipping build step... +27: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... +31: No modifications detected for re-loaded extension module utils, skipping build step...No modifications detected for re-loaded extension module utils, skipping build step... +31: +31: Loading extension module utils...Loading extension module utils... +31: +29: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +29: No modifications detected for re-loaded extension module utils, skipping build step... +29: Loading extension module utils... +31: Using /pfs/lustrep4/users/muennighoff/.cache/torch_extensions/py39_cpu as PyTorch extensions root... +31: No modifications detected for re-loaded extension module utils, skipping build step... +31: Loading extension module utils... + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/utils.py:349: UserWarning: Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings + 0: warnings.warn("Parameter count with the embeddings will be inaccurate with PP > 1, as the first and last stage hold several copies of the embeddings") diff --git a/4b284b6bc4subopt/3476842.out b/4b284b6bc4subopt/3476842.out new file mode 100644 index 0000000000000000000000000000000000000000..e08d2e14ee67bb35f74de524d785f085249fd824 --- /dev/null +++ b/4b284b6bc4subopt/3476842.out @@ -0,0 +1,36410 @@ +Model parameters: d_model 2560 ffw_size 10240 kv_size 128 n_heads 20 n_layers 30 +Megatron-DeepSpeed/pretrain_gpt.py --tensor-model-parallel-size 2 --pipeline-model-parallel-size 1 --num-layers 30 --hidden-size 2560 --num-attention-heads 20 --kv-channels 128 --ffn-hidden-size 10240 --seq-length 2048 --max-position-embeddings 2048 --micro-batch-size 2 --global-batch-size 1024 --train-samples 1 --vocab-file gpt2/vocab.json --merge-file gpt2/merges.txt --clip-grad 1.0 --kill-switch-path kill-switch-4b284b6bc4suboptval --bf16 --optimizer adam --adam-beta1 0.9 --adam-beta2 0.999 --adam-eps 1e-8 --lr 2e-4 --min-lr 2e-5 --lr-decay-style cosine --lr-decay-samples 1 --lr-warmup-samples 0 --clip-grad 1.0 --weight-decay 1e-1 --override-lr-scheduler --reset-progress --no-load-optim --log-interval 10 --save-interval 1000 --eval-interval 1 --eval-iters 100 --eval-only true --tensorboard-dir tensorboard_4b284b6bc4suboptval --tensorboard-queue-size 5 --log-timers-to-tensorboard --log-batch-size-to-tensorboard --log-validation-ppl-to-tensorboard --save lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt --load lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt --train-weighted-split-paths-path train1b5.txt --valid-weighted-split-paths-path val.txt --data-impl mmap --deepspeed --deepspeed_config ds_configs/3476842.json --zero-stage 0 +START 3476842: Mon 08 May 2023 11:21:54 AM EEST + 0: + 0: + 0: ======================= ROCm System Management Interface ======================= + 0: ================================= Concise Info ================================= + 0: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 0: 0 43.0c 98.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 2 42.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 4 49.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: 6 42.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 0: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 0: ================================================================================ + 0: ============================= End of ROCm SMI Log ============================== + 7: + 7: + 7: ======================= ROCm System Management Interface ======================= + 7: ================================= Concise Info ================================= + 7: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 7: 0 45.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 2 41.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 4 42.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 5 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: 6 40.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 7: 7 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 7: ================================================================================ + 7: ============================= End of ROCm SMI Log ============================== + 2: + 2: + 2: ======================= ROCm System Management Interface ======================= + 2: ================================= Concise Info ================================= + 2: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 2: 0 48.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 2 41.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 4 41.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: 6 44.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 2: 7 39.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 2: ================================================================================ + 2: ============================= End of ROCm SMI Log ============================== +14: +14: +14: ======================= ROCm System Management Interface ======================= +14: ================================= Concise Info ================================= +14: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +14: 0 47.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 2 42.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 4 37.0c 82.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: 6 44.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +14: 7 39.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +14: ================================================================================ +14: ============================= End of ROCm SMI Log ============================== +23: +23: +23: ======================= ROCm System Management Interface ======================= +23: ================================= Concise Info ================================= +23: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +23: 0 43.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 2 45.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 4 46.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: 6 44.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +23: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +23: ================================================================================ +23: ============================= End of ROCm SMI Log ============================== +24: +24: +24: ======================= ROCm System Management Interface ======================= +24: ================================= Concise Info ================================= +24: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +24: 0 47.0c 97.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 2 44.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 4 43.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: 6 43.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +24: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +24: ================================================================================ +24: ============================= End of ROCm SMI Log ============================== +12: +12: +12: ======================= ROCm System Management Interface ======================= +12: ================================= Concise Info ================================= +12: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +12: 0 44.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 2 45.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 4 46.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: 6 43.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +12: 7 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +12: ================================================================================ +12: ============================= End of ROCm SMI Log ============================== + 6: + 6: + 6: ======================= ROCm System Management Interface ======================= + 6: ================================= Concise Info ================================= + 6: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 6: 0 45.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 2 33.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 3 39.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 4 44.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: 6 39.0c 99.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 6: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 6: ================================================================================ + 6: ============================= End of ROCm SMI Log ============================== + 8: + 8: + 8: ======================= ROCm System Management Interface ======================= + 8: ================================= Concise Info ================================= + 8: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 8: 0 45.0c 98.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 2 46.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 3 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 4 50.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: 6 38.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 8: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 8: ================================================================================ + 8: ============================= End of ROCm SMI Log ============================== +20: +20: +20: ======================= ROCm System Management Interface ======================= +20: ================================= Concise Info ================================= +20: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +20: 0 40.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 2 39.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 4 43.0c 79.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 5 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: 6 40.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +20: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +20: ================================================================================ +20: ============================= End of ROCm SMI Log ============================== +30: +30: +30: ======================= ROCm System Management Interface ======================= +30: ================================= Concise Info ================================= +30: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +30: 0 50.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 2 45.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 4 46.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: 6 45.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +30: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +30: ================================================================================ +30: ============================= End of ROCm SMI Log ============================== +10: +10: +10: ======================= ROCm System Management Interface ======================= +10: ================================= Concise Info ================================= +10: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +10: 0 44.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 2 40.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 4 44.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: 6 37.0c 81.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +10: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +10: ================================================================================ +10: ============================= End of ROCm SMI Log ============================== +17: +17: +17: ======================= ROCm System Management Interface ======================= +17: ================================= Concise Info ================================= +17: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +17: 0 42.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 2 41.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 4 46.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: 6 49.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +17: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +17: ================================================================================ +17: ============================= End of ROCm SMI Log ============================== +22: +22: +22: ======================= ROCm System Management Interface ======================= +22: ================================= Concise Info ================================= +22: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +22: 0 46.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 2 41.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 4 42.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 5 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: 6 44.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +22: 7 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +22: ================================================================================ +22: ============================= End of ROCm SMI Log ============================== + 4: + 4: + 4: ======================= ROCm System Management Interface ======================= + 4: ================================= Concise Info ================================= + 4: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 4: 0 47.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 2 51.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 4 39.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: 6 38.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 4: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 4: ================================================================================ + 4: ============================= End of ROCm SMI Log ============================== +28: +28: +28: ======================= ROCm System Management Interface ======================= +28: ================================= Concise Info ================================= +28: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +28: 0 43.0c 97.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 2 50.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 3 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 4 49.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: 6 43.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +28: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +28: ================================================================================ +28: ============================= End of ROCm SMI Log ============================== +16: +16: +16: ======================= ROCm System Management Interface ======================= +16: ================================= Concise Info ================================= +16: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +16: 0 50.0c 85.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 1 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 2 38.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 4 46.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: 6 44.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +16: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +16: ================================================================================ +16: ============================= End of ROCm SMI Log ============================== +21: +21: +21: ======================= ROCm System Management Interface ======================= +21: ================================= Concise Info ================================= +21: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +21: 0 37.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 2 46.0c 98.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 4 47.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 5 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: 6 41.0c 98.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +21: 7 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +21: ================================================================================ +21: ============================= End of ROCm SMI Log ============================== + 5: + 5: + 5: ======================= ROCm System Management Interface ======================= + 5: ================================= Concise Info ================================= + 5: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 5: 0 45.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 2 41.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 4 42.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: 6 53.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 5: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 5: ================================================================================ + 5: ============================= End of ROCm SMI Log ============================== +26: +26: +26: ======================= ROCm System Management Interface ======================= +26: ================================= Concise Info ================================= +26: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +26: 0 43.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 1 50.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 2 42.0c 96.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 3 41.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 4 47.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: 6 42.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +26: 7 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +26: ================================================================================ +26: ============================= End of ROCm SMI Log ============================== +27: +27: +27: ======================= ROCm System Management Interface ======================= +27: ================================= Concise Info ================================= +27: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +27: 0 43.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 1 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 2 44.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 3 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 4 44.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: 6 46.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +27: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +27: ================================================================================ +27: ============================= End of ROCm SMI Log ============================== +18: +18: +18: ======================= ROCm System Management Interface ======================= +18: ================================= Concise Info ================================= +18: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +18: 0 47.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 1 54.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 2 45.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 4 41.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: 6 43.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +18: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +18: ================================================================================ +18: ============================= End of ROCm SMI Log ============================== +29: +29: +29: ======================= ROCm System Management Interface ======================= +29: ================================= Concise Info ================================= +29: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +29: 0 43.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 1 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 2 41.0c 88.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 3 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 4 45.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: 6 35.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +29: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +29: ================================================================================ +29: ============================= End of ROCm SMI Log ============================== +25: +25: +25: ======================= ROCm System Management Interface ======================= +25: ================================= Concise Info ================================= +25: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +25: 0 45.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 1 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 2 42.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 4 41.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: 6 45.0c 84.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +25: 7 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +25: ================================================================================ +25: ============================= End of ROCm SMI Log ============================== +31: +31: +31: ======================= ROCm System Management Interface ======================= +31: ================================= Concise Info ================================= +31: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +31: 0 46.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 1 39.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 2 40.0c 83.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 4 39.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 5 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: 6 42.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +31: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +31: ================================================================================ +31: ============================= End of ROCm SMI Log ============================== +19: +19: +19: ======================= ROCm System Management Interface ======================= +19: ================================= Concise Info ================================= +19: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +19: 0 41.0c 97.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 1 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 2 44.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 4 42.0c 94.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 5 48.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: 6 46.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +19: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +19: ================================================================================ +19: ============================= End of ROCm SMI Log ============================== +13: +13: +13: ======================= ROCm System Management Interface ======================= +13: ================================= Concise Info ================================= +13: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +13: 0 48.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 2 45.0c 91.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 3 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 4 44.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: 6 39.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +13: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +13: ================================================================================ +13: ============================= End of ROCm SMI Log ============================== +15: +15: +15: ======================= ROCm System Management Interface ======================= +15: ================================= Concise Info ================================= +15: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +15: 0 44.0c 98.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 1 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 2 43.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 4 41.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 5 47.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: 6 44.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +15: 7 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +15: ================================================================================ +15: ============================= End of ROCm SMI Log ============================== + 3: + 3: + 3: ======================= ROCm System Management Interface ======================= + 3: ================================= Concise Info ================================= + 3: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 3: 0 40.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 2 39.0c 92.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 3 40.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 4 44.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 5 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: 6 49.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 3: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 3: ================================================================================ + 3: ============================= End of ROCm SMI Log ============================== + 9: + 9: + 9: ======================= ROCm System Management Interface ======================= + 9: ================================= Concise Info ================================= + 9: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 9: 0 43.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 2 44.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 3 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 4 47.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 5 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: 6 42.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 9: 7 42.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 9: ================================================================================ + 9: ============================= End of ROCm SMI Log ============================== + 1: + 1: + 1: ======================= ROCm System Management Interface ======================= + 1: ================================= Concise Info ================================= + 1: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% + 1: 0 44.0c 95.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 1 49.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 2 43.0c 86.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 3 44.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 4 43.0c 87.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: 6 40.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% + 1: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% + 1: ================================================================================ + 1: ============================= End of ROCm SMI Log ============================== +11: +11: +11: ======================= ROCm System Management Interface ======================= +11: ================================= Concise Info ================================= +11: GPU Temp AvgPwr SCLK MCLK Fan Perf PwrCap VRAM% GPU% +11: 0 44.0c 90.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 1 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 2 39.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 3 46.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 4 49.0c 93.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 5 45.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: 6 45.0c 89.0W 800Mhz 1600Mhz 0% auto 560.0W 0% 0% +11: 7 43.0c N/A 800Mhz 1600Mhz 0% auto 0.0W 0% 0% +11: ================================================================================ +11: ============================= End of ROCm SMI Log ============================== + 8: Launching on nid006495 (8/32), master nid006487 port 9999, GPUs 8, CUDA: True +22: Launching on nid006509 (22/32), master nid006487 port 9999, GPUs 8, CUDA: True + 0: Launching on nid006487 (0/32), master nid006487 port 9999, GPUs 8, CUDA: True +27: Launching on nid006514 (27/32), master nid006487 port 9999, GPUs 8, CUDA: True +12: Launching on nid006499 (12/32), master nid006487 port 9999, GPUs 8, CUDA: True + 2: Launching on nid006489 (2/32), master nid006487 port 9999, GPUs 8, CUDA: True +21: Launching on nid006508 (21/32), master nid006487 port 9999, GPUs 8, CUDA: True +23: Launching on nid006510 (23/32), master nid006487 port 9999, GPUs 8, CUDA: True +28: Launching on nid006515 (28/32), master nid006487 port 9999, GPUs 8, CUDA: True + 7: Launching on nid006494 (7/32), master nid006487 port 9999, GPUs 8, CUDA: True +17: Launching on nid006504 (17/32), master nid006487 port 9999, GPUs 8, CUDA: True +18: Launching on nid006505 (18/32), master nid006487 port 9999, GPUs 8, CUDA: True +29: Launching on nid006516 (29/32), master nid006487 port 9999, GPUs 8, CUDA: True +24: Launching on nid006511 (24/32), master nid006487 port 9999, GPUs 8, CUDA: True +26: Launching on nid006513 (26/32), master nid006487 port 9999, GPUs 8, CUDA: True + 5: Launching on nid006492 (5/32), master nid006487 port 9999, GPUs 8, CUDA: True +14: Launching on nid006501 (14/32), master nid006487 port 9999, GPUs 8, CUDA: True +30: Launching on nid006517 (30/32), master nid006487 port 9999, GPUs 8, CUDA: True + 6: Launching on nid006493 (6/32), master nid006487 port 9999, GPUs 8, CUDA: True +25: Launching on nid006512 (25/32), master nid006487 port 9999, GPUs 8, CUDA: True + 4: Launching on nid006491 (4/32), master nid006487 port 9999, GPUs 8, CUDA: True +31: Launching on nid006518 (31/32), master nid006487 port 9999, GPUs 8, CUDA: True +20: Launching on nid006507 (20/32), master nid006487 port 9999, GPUs 8, CUDA: True +16: Launching on nid006503 (16/32), master nid006487 port 9999, GPUs 8, CUDA: True +10: Launching on nid006497 (10/32), master nid006487 port 9999, GPUs 8, CUDA: True +15: Launching on nid006502 (15/32), master nid006487 port 9999, GPUs 8, CUDA: True +19: Launching on nid006506 (19/32), master nid006487 port 9999, GPUs 8, CUDA: True +11: Launching on nid006498 (11/32), master nid006487 port 9999, GPUs 8, CUDA: True + 9: Launching on nid006496 (9/32), master nid006487 port 9999, GPUs 8, CUDA: True + 3: Launching on nid006490 (3/32), master nid006487 port 9999, GPUs 8, CUDA: True + 1: Launching on nid006488 (1/32), master nid006487 port 9999, GPUs 8, CUDA: True +13: Launching on nid006500 (13/32), master nid006487 port 9999, GPUs 8, CUDA: True + 0: using world size: 256, data-parallel-size: 128, tensor-model-parallel size: 2, pipeline-model-parallel size: 1 + 0: accumulate and all-reduce gradients in fp32 for bfloat16 data type. + 0: using torch.bfloat16 for parameters ... + 0: ------------------------ arguments ------------------------ + 0: abort_on_unmet_fused_kernel_constraints ......... False + 0: accumulate_allreduce_grads_in_fp32 .............. True + 0: adam_beta1 ...................................... 0.9 + 0: adam_beta2 ...................................... 0.999 + 0: adam_eps ........................................ 1e-08 + 0: adlr_autoresume ................................. False + 0: adlr_autoresume_interval ........................ 1000 + 0: apply_query_key_layer_scaling ................... True + 0: apply_residual_connection_post_layernorm ........ False + 0: attention_dropout ............................... 0.1 + 0: attention_softmax_in_fp32 ....................... False + 0: bert_binary_head ................................ True + 0: bert_load ....................................... None + 0: bf16 ............................................ True + 0: bias_dropout_fusion ............................. True + 0: bias_gelu_fusion ................................ True + 0: biencoder_projection_dim ........................ 0 + 0: biencoder_shared_query_context_model ............ False + 0: block_data_path ................................. None + 0: checkpoint_activations .......................... False + 0: checkpoint_in_cpu ............................... False + 0: checkpoint_num_layers ........................... 1 + 0: clip_grad ....................................... 1.0 + 0: codecarbon_dir .................................. None + 0: consumed_train_samples .......................... 0 + 0: consumed_train_tokens ........................... 0 + 0: consumed_valid_samples .......................... 0 + 0: contigious_checkpointing ........................ False + 0: cpu_optimizer ................................... False + 0: cpu_torch_adam .................................. False + 0: curriculum_learning ............................. False + 0: data_impl ....................................... mmap + 0: data_parallel_size .............................. 128 + 0: data_path ....................................... None + 0: dataloader_type ................................. single + 0: DDP_impl ........................................ local + 0: decoder_seq_length .............................. None + 0: deepscale ....................................... False + 0: deepscale_config ................................ None + 0: deepspeed ....................................... True + 0: deepspeed_activation_checkpointing .............. False + 0: deepspeed_config ................................ ds_configs/3476842.json + 0: deepspeed_mpi ................................... False + 0: distribute_checkpointed_activations ............. False + 0: distributed_backend ............................. nccl + 0: embed_layernorm ................................. False + 0: embedding_path .................................. None + 0: encoder_seq_length .............................. 2048 + 0: eod_mask_loss ................................... False + 0: eval_interval ................................... 1 + 0: eval_iters ...................................... 100 + 0: eval_only ....................................... True + 0: evidence_data_path .............................. None + 0: exit_duration_in_mins ........................... None + 0: exit_interval ................................... None + 0: ffn_hidden_size ................................. 10240 + 0: finetune ........................................ False + 0: fp16 ............................................ False + 0: fp16_lm_cross_entropy ........................... False + 0: fp32_residual_connection ........................ False + 0: gigaflos_no_embeds .............................. 0 + 0: global_batch_size ............................... 1024 + 0: glu_activation .................................. None + 0: hidden_dropout .................................. 0.1 + 0: hidden_size ..................................... 2560 + 0: hysteresis ...................................... 2 + 0: ict_head_size ................................... None + 0: ict_load ........................................ None + 0: img_dim ......................................... 224 + 0: indexer_batch_size .............................. 128 + 0: indexer_log_interval ............................ 1000 + 0: inference ....................................... False + 0: init_method_std ................................. 0.02 + 0: init_method_xavier_uniform ...................... False + 0: initial_loss_scale .............................. 4294967296 + 0: kill_switch_path ................................ kill-switch-4b284b6bc4suboptval + 0: kv_channels ..................................... 128 + 0: layer_norm_fusion ............................... True + 0: layernorm_epsilon ............................... 1e-05 + 0: lazy_mpu_init ................................... None + 0: load ............................................ lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt + 0: local_rank ...................................... None + 0: log_batch_size_to_tensorboard ................... True + 0: log_interval .................................... 10 + 0: log_learning_rate_to_tensorboard ................ True + 0: log_level ....................................... None + 0: log_level_replica ............................... None + 0: log_loss_scale_to_tensorboard ................... True + 0: log_num_zeros_in_grad ........................... False + 0: log_params_norm ................................. False + 0: log_path ........................................ None + 0: log_timers_to_tensorboard ....................... True + 0: log_validation_ppl_to_tensorboard ............... True + 0: loss_on_targets_only ............................ False + 0: loss_scale ...................................... None + 0: loss_scale_window ............................... 1000 + 0: lr .............................................. 0.0002 + 0: lr_decay_iters .................................. None + 0: lr_decay_samples ................................ 1 + 0: lr_decay_style .................................. cosine + 0: lr_decay_tokens ................................. None + 0: lr_warmup_fraction .............................. None + 0: lr_warmup_iters ................................. 0 + 0: lr_warmup_samples ............................... 0 + 0: make_vocab_size_divisible_by .................... 128 + 0: mask_prob ....................................... 0.15 + 0: masked_softmax_fusion ........................... True + 0: max_position_embeddings ......................... 2048 + 0: mean_noise_span_length .......................... None + 0: memory_centric_tiled_linear ..................... False + 0: merge_file ...................................... gpt2/merges.txt + 0: micro_batch_size ................................ 2 + 0: min_loss_scale .................................. 1.0 + 0: min_lr .......................................... 2e-05 + 0: mmap_warmup ..................................... False + 0: no_load_optim ................................... True + 0: no_load_rng ..................................... None + 0: no_save_optim ................................... None + 0: no_save_rng ..................................... None + 0: noise_density ................................... None + 0: num_attention_heads ............................. 20 + 0: num_channels .................................... 3 + 0: num_classes ..................................... 1000 + 0: num_layers ...................................... 30 + 0: num_layers_per_virtual_pipeline_stage ........... None + 0: num_workers ..................................... 2 + 0: onnx_safe ....................................... None + 0: openai_gelu ..................................... False + 0: optimizer ....................................... adam + 0: optimizer_fusion ................................ True + 0: override_lr_scheduler ........................... True + 0: pad_vocab_size_to ............................... None + 0: params_dtype .................................... torch.bfloat16 + 0: partition_activations ........................... False + 0: patch_dim ....................................... 16 + 0: pipeline_model_parallel_size .................... 1 + 0: position_embedding_type ......................... PositionEmbeddingType.absolute + 0: pp_partition_method ............................. None + 0: profile_backward ................................ False + 0: query_in_block_prob ............................. 0.1 + 0: rampup_batch_size ............................... None + 0: rank ............................................ 0 + 0: remote_device ................................... none + 0: reset_attention_mask ............................ False + 0: reset_position_ids .............................. False + 0: reset_progress .................................. True + 0: retriever_report_topk_accuracies ................ [] + 0: retriever_score_scaling ......................... False + 0: retriever_seq_length ............................ 256 + 0: reweight_loss_based_on_position_frequency ....... False + 0: sample_rate ..................................... 1.0 + 0: save ............................................ lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt + 0: save_interval ................................... 1000 + 0: scatter_gather_tensors_in_pipeline .............. True + 0: scattered_embeddings ............................ False + 0: seed ............................................ 1234 + 0: seq_length ...................................... 2048 + 0: sgd_momentum .................................... 0.9 + 0: short_seq_prob .................................. 0.1 + 0: skip_train_iteration_range ...................... None + 0: split ........................................... None + 0: split_transformers .............................. False + 0: sync_tp_duplicated_parameters ................... False + 0: synchronize_each_layer .......................... False + 0: tensor_model_parallel_size ...................... 2 + 0: tensorboard_dir ................................. tensorboard_4b284b6bc4suboptval + 0: tensorboard_log_interval ........................ 1 + 0: tensorboard_queue_size .......................... 5 + 0: test_weighted_split_paths ....................... None + 0: test_weighted_split_paths_path .................. None + 0: tile_factor ..................................... 1 + 0: titles_data_path ................................ None + 0: tokenizer_name_or_path .......................... None + 0: tokenizer_type .................................. GPT2BPETokenizer + 0: train_iters ..................................... None + 0: train_samples ................................... 1 + 0: train_tokens .................................... None + 0: train_weighted_split_names ...................... ['train'] + 0: train_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document']] + 0: train_weighted_split_paths_path ................. None + 0: train_weighted_split_splits ..................... [['0:1']] + 0: train_weighted_split_weights .................... [['1.0']] + 0: universal_checkpoint ............................ False + 0: use_bnb_optimizer ............................... False + 0: use_checkpoint_lr_scheduler ..................... False + 0: use_contiguous_buffers_in_ddp ................... True + 0: use_cpu_initialization .......................... None + 0: use_one_sent_docs ............................... False + 0: use_pin_memory .................................. False + 0: valid_num_workers ............................... 2 + 0: valid_weighted_split_names ...................... ['validation'] + 0: valid_weighted_split_paths ...................... [['/scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document']] + 0: valid_weighted_split_paths_path ................. None + 0: valid_weighted_split_splits ..................... [['0:1']] + 0: valid_weighted_split_weights .................... [['1.0']] + 0: virtual_pipeline_model_parallel_size ............ None + 0: vocab_extra_ids ................................. 0 + 0: vocab_file ...................................... gpt2/vocab.json + 0: weight_decay .................................... 0.1 + 0: world_size ...................................... 256 + 0: zero_allgather_bucket_size ...................... 0.0 + 0: zero_contigious_gradients ....................... False + 0: zero_reduce_bucket_size ......................... 0.0 + 0: zero_reduce_scatter ............................. False + 0: zero_stage ...................................... 0 + 0: -------------------- end of arguments --------------------- + 0: setting number of micro-batches to constant 4 + 0: > building GPT2BPETokenizer tokenizer ... + 0: > padded vocab (size: 50257) with 175 dummy tokens (new size: 50432) + 0: DeepSpeed general environment info: + 0: torch install path ............... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch'] + 0: torch version .................... 1.13.0+rocm5.2 + 0: torch cuda version ............... None + 0: torch hip version ................ 5.2.21151-afdc89f8 + 0: nvcc version ..................... None + 0: deepspeed install path ........... ['/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/deepspeed'] + 0: deepspeed info ................... 0.7.5, unknown, unknown + 0: deepspeed wheel compiled w. ...... torch 1.13, hip 5.1 + 0: **** Git info for Megatron: git_hash=unknown git_branch=unknown **** + 0: > initializing torch distributed ... + 0: [2023-05-08 11:25:02,968] [INFO] [comm.py:633:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +31: > setting tensorboard ... + 0: > initializing tensor model parallel with size 2 + 0: > initializing pipeline model parallel with size 1 + 0: > setting random seeds to 1234 ... + 0: > initializing model parallel cuda seeds on global rank 0, model parallel rank 0, and data parallel rank 0 with model parallel seed: 3952 and data parallel seed: 1234 + 0: > compiling dataset index builder ... + 0: make: Entering directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' + 0: make: Nothing to be done for 'default'. + 0: make: Leaving directory '/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/data' + 0: >>> done with dataset index builder. Compilation time: 0.102 seconds + 0: > compiling and loading fused kernels ... + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.cpp [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.hip [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] + 0: Total number of unsupported CUDA function calls: 0 + 0: + 0: + 0: Total number of replaced kernel launches: 87 + 0: [1/1] c++ scaled_upper_triang_masked_softmax_hip.o scaled_upper_triang_masked_softmax_hip.cuda.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/opt/rocm/lib -lamdhip64 -o scaled_upper_triang_masked_softmax_cuda.so + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.cpp [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_cuda.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.hip [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] + 0: Total number of unsupported CUDA function calls: 0 + 0: + 0: + 0: Total number of replaced kernel launches: 63 + 0: [1/1] c++ scaled_masked_softmax_hip.o scaled_masked_softmax_hip.cuda.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/opt/rocm/lib -lamdhip64 -o scaled_masked_softmax_cuda.so + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda.cpp [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_cuda_kernel.cu -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/layer_norm_hip_kernel.hip [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/type_shim.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/compat.h [skipped, no changes] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_upper_triang_masked_softmax_hip.h [skipped, already hipified] + 0: /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax.h -> /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/Megatron-DeepSpeed/megatron/fused_kernels/scaled_masked_softmax_hip.h [skipped, already hipified] + 0: Total number of unsupported CUDA function calls: 0 + 0: + 0: + 0: Total number of replaced kernel launches: 67 + 0: [1/1] c++ layer_norm_hip_kernel.cuda.o layer_norm_cuda.o -shared -L/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/lib/python3.9/site-packages/torch/lib -lc10 -lc10_hip -ltorch_cpu -ltorch_hip -ltorch -ltorch_python -L/opt/rocm/lib -lamdhip64 -o fused_mix_prec_layer_norm_cuda.so + 0: >>> done with compiling and loading fused kernels. Compilation time: 28.220 seconds + 0: time to initialize megatron (seconds): 3.672 + 0: [after megatron is initialized] datetime: 2023-05-08 11:25:39 + 0: building GPT model ... + 0: [2023-05-08 11:25:39,844] [INFO] [utils.py:827:see_memory_usage] Before Building Model + 0: [2023-05-08 11:25:39,845] [INFO] [utils.py:828:see_memory_usage] MA 0.0 GB Max_MA 0.0 GB CA 0.0 GB Max_CA 0 GB + 0: [2023-05-08 11:25:39,846] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 38.35 GB, percent = 7.6% + 0: SEED_LAYERS=False BASE_SEED=1234 SEED_FN=None + 0: Using topology: {ProcessCoord(pipe=0, data=0, model=0): 0, ProcessCoord(pipe=0, data=0, model=1): 1, ProcessCoord(pipe=0, data=1, model=0): 2, ProcessCoord(pipe=0, data=1, model=1): 3, ProcessCoord(pipe=0, data=2, model=0): 4, ProcessCoord(pipe=0, data=2, model=1): 5, ProcessCoord(pipe=0, data=3, model=0): 6, ProcessCoord(pipe=0, data=3, model=1): 7, ProcessCoord(pipe=0, data=4, model=0): 8, ProcessCoord(pipe=0, data=4, model=1): 9, ProcessCoord(pipe=0, data=5, model=0): 10, ProcessCoord(pipe=0, data=5, model=1): 11, ProcessCoord(pipe=0, data=6, model=0): 12, ProcessCoord(pipe=0, data=6, model=1): 13, ProcessCoord(pipe=0, data=7, model=0): 14, ProcessCoord(pipe=0, data=7, model=1): 15, ProcessCoord(pipe=0, data=8, model=0): 16, ProcessCoord(pipe=0, data=8, model=1): 17, ProcessCoord(pipe=0, data=9, model=0): 18, ProcessCoord(pipe=0, data=9, model=1): 19, ProcessCoord(pipe=0, data=10, model=0): 20, ProcessCoord(pipe=0, data=10, model=1): 21, ProcessCoord(pipe=0, data=11, model=0): 22, ProcessCoord(pipe=0, data + 0: =11, model=1): 23, ProcessCoord(pipe=0, data=12, model=0): 24, ProcessCoord(pipe=0, data=12, model=1): 25, ProcessCoord(pipe=0, data=13, model=0): 26, ProcessCoord(pipe=0, data=13, model=1): 27, ProcessCoord(pipe=0, data=14, model=0): 28, ProcessCoord(pipe=0, data=14, model=1): 29, ProcessCoord(pipe=0, data=15, model=0): 30, ProcessCoord(pipe=0, data=15, model=1): 31, ProcessCoord(pipe=0, data=16, model=0): 32, ProcessCoord(pipe=0, data=16, model=1): 33, ProcessCoord(pipe=0, data=17, model=0): 34, ProcessCoord(pipe=0, data=17, model=1): 35, ProcessCoord(pipe=0, data=18, model=0): 36, ProcessCoord(pipe=0, data=18, model=1): 37, ProcessCoord(pipe=0, data=19, model=0): 38, ProcessCoord(pipe=0, data=19, model=1): 39, ProcessCoord(pipe=0, data=20, model=0): 40, ProcessCoord(pipe=0, data=20, model=1): 41, ProcessCoord(pipe=0, data=21, model=0): 42, ProcessCoord(pipe=0, data=21, model=1): 43, ProcessCoord(pipe=0, data=22, model=0): 44, ProcessCoord(pipe=0, data=22, model=1): 45, ProcessCoord(pipe=0, data=23, model=0 + 0: ): 46, ProcessCoord(pipe=0, data=23, model=1): 47, ProcessCoord(pipe=0, data=24, model=0): 48, ProcessCoord(pipe=0, data=24, model=1): 49, ProcessCoord(pipe=0, data=25, model=0): 50, ProcessCoord(pipe=0, data=25, model=1): 51, ProcessCoord(pipe=0, data=26, model=0): 52, ProcessCoord(pipe=0, data=26, model=1): 53, ProcessCoord(pipe=0, data=27, model=0): 54, ProcessCoord(pipe=0, data=27, model=1): 55, ProcessCoord(pipe=0, data=28, model=0): 56, ProcessCoord(pipe=0, data=28, model=1): 57, ProcessCoord(pipe=0, data=29, model=0): 58, ProcessCoord(pipe=0, data=29, model=1): 59, ProcessCoord(pipe=0, data=30, model=0): 60, ProcessCoord(pipe=0, data=30, model=1): 61, ProcessCoord(pipe=0, data=31, model=0): 62, ProcessCoord(pipe=0, data=31, model=1): 63, ProcessCoord(pipe=0, data=32, model=0): 64, ProcessCoord(pipe=0, data=32, model=1): 65, ProcessCoord(pipe=0, data=33, model=0): 66, ProcessCoord(pipe=0, data=33, model=1): 67, ProcessCoord(pipe=0, data=34, model=0): 68, ProcessCoord(pipe=0, data=34, model=1): 69, Proce + 0: ssCoord(pipe=0, data=35, model=0): 70, ProcessCoord(pipe=0, data=35, model=1): 71, ProcessCoord(pipe=0, data=36, model=0): 72, ProcessCoord(pipe=0, data=36, model=1): 73, ProcessCoord(pipe=0, data=37, model=0): 74, ProcessCoord(pipe=0, data=37, model=1): 75, ProcessCoord(pipe=0, data=38, model=0): 76, ProcessCoord(pipe=0, data=38, model=1): 77, ProcessCoord(pipe=0, data=39, model=0): 78, ProcessCoord(pipe=0, data=39, model=1): 79, ProcessCoord(pipe=0, data=40, model=0): 80, ProcessCoord(pipe=0, data=40, model=1): 81, ProcessCoord(pipe=0, data=41, model=0): 82, ProcessCoord(pipe=0, data=41, model=1): 83, ProcessCoord(pipe=0, data=42, model=0): 84, ProcessCoord(pipe=0, data=42, model=1): 85, ProcessCoord(pipe=0, data=43, model=0): 86, ProcessCoord(pipe=0, data=43, model=1): 87, ProcessCoord(pipe=0, data=44, model=0): 88, ProcessCoord(pipe=0, data=44, model=1): 89, ProcessCoord(pipe=0, data=45, model=0): 90, ProcessCoord(pipe=0, data=45, model=1): 91, ProcessCoord(pipe=0, data=46, model=0): 92, ProcessCoord(pipe + 0: =0, data=46, model=1): 93, ProcessCoord(pipe=0, data=47, model=0): 94, ProcessCoord(pipe=0, data=47, model=1): 95, ProcessCoord(pipe=0, data=48, model=0): 96, ProcessCoord(pipe=0, data=48, model=1): 97, ProcessCoord(pipe=0, data=49, model=0): 98, ProcessCoord(pipe=0, data=49, model=1): 99, ProcessCoord(pipe=0, data=50, model=0): 100, ProcessCoord(pipe=0, data=50, model=1): 101, ProcessCoord(pipe=0, data=51, model=0): 102, ProcessCoord(pipe=0, data=51, model=1): 103, ProcessCoord(pipe=0, data=52, model=0): 104, ProcessCoord(pipe=0, data=52, model=1): 105, ProcessCoord(pipe=0, data=53, model=0): 106, ProcessCoord(pipe=0, data=53, model=1): 107, ProcessCoord(pipe=0, data=54, model=0): 108, ProcessCoord(pipe=0, data=54, model=1): 109, ProcessCoord(pipe=0, data=55, model=0): 110, ProcessCoord(pipe=0, data=55, model=1): 111, ProcessCoord(pipe=0, data=56, model=0): 112, ProcessCoord(pipe=0, data=56, model=1): 113, ProcessCoord(pipe=0, data=57, model=0): 114, ProcessCoord(pipe=0, data=57, model=1): 115, ProcessCoord( + 0: pipe=0, data=58, model=0): 116, ProcessCoord(pipe=0, data=58, model=1): 117, ProcessCoord(pipe=0, data=59, model=0): 118, ProcessCoord(pipe=0, data=59, model=1): 119, ProcessCoord(pipe=0, data=60, model=0): 120, ProcessCoord(pipe=0, data=60, model=1): 121, ProcessCoord(pipe=0, data=61, model=0): 122, ProcessCoord(pipe=0, data=61, model=1): 123, ProcessCoord(pipe=0, data=62, model=0): 124, ProcessCoord(pipe=0, data=62, model=1): 125, ProcessCoord(pipe=0, data=63, model=0): 126, ProcessCoord(pipe=0, data=63, model=1): 127, ProcessCoord(pipe=0, data=64, model=0): 128, ProcessCoord(pipe=0, data=64, model=1): 129, ProcessCoord(pipe=0, data=65, model=0): 130, ProcessCoord(pipe=0, data=65, model=1): 131, ProcessCoord(pipe=0, data=66, model=0): 132, ProcessCoord(pipe=0, data=66, model=1): 133, ProcessCoord(pipe=0, data=67, model=0): 134, ProcessCoord(pipe=0, data=67, model=1): 135, ProcessCoord(pipe=0, data=68, model=0): 136, ProcessCoord(pipe=0, data=68, model=1): 137, ProcessCoord(pipe=0, data=69, model=0): 138, Pr + 0: ocessCoord(pipe=0, data=69, model=1): 139, ProcessCoord(pipe=0, data=70, model=0): 140, ProcessCoord(pipe=0, data=70, model=1): 141, ProcessCoord(pipe=0, data=71, model=0): 142, ProcessCoord(pipe=0, data=71, model=1): 143, ProcessCoord(pipe=0, data=72, model=0): 144, ProcessCoord(pipe=0, data=72, model=1): 145, ProcessCoord(pipe=0, data=73, model=0): 146, ProcessCoord(pipe=0, data=73, model=1): 147, ProcessCoord(pipe=0, data=74, model=0): 148, ProcessCoord(pipe=0, data=74, model=1): 149, ProcessCoord(pipe=0, data=75, model=0): 150, ProcessCoord(pipe=0, data=75, model=1): 151, ProcessCoord(pipe=0, data=76, model=0): 152, ProcessCoord(pipe=0, data=76, model=1): 153, ProcessCoord(pipe=0, data=77, model=0): 154, ProcessCoord(pipe=0, data=77, model=1): 155, ProcessCoord(pipe=0, data=78, model=0): 156, ProcessCoord(pipe=0, data=78, model=1): 157, ProcessCoord(pipe=0, data=79, model=0): 158, ProcessCoord(pipe=0, data=79, model=1): 159, ProcessCoord(pipe=0, data=80, model=0): 160, ProcessCoord(pipe=0, data=80, model= + 0: 1): 161, ProcessCoord(pipe=0, data=81, model=0): 162, ProcessCoord(pipe=0, data=81, model=1): 163, ProcessCoord(pipe=0, data=82, model=0): 164, ProcessCoord(pipe=0, data=82, model=1): 165, ProcessCoord(pipe=0, data=83, model=0): 166, ProcessCoord(pipe=0, data=83, model=1): 167, ProcessCoord(pipe=0, data=84, model=0): 168, ProcessCoord(pipe=0, data=84, model=1): 169, ProcessCoord(pipe=0, data=85, model=0): 170, ProcessCoord(pipe=0, data=85, model=1): 171, ProcessCoord(pipe=0, data=86, model=0): 172, ProcessCoord(pipe=0, data=86, model=1): 173, ProcessCoord(pipe=0, data=87, model=0): 174, ProcessCoord(pipe=0, data=87, model=1): 175, ProcessCoord(pipe=0, data=88, model=0): 176, ProcessCoord(pipe=0, data=88, model=1): 177, ProcessCoord(pipe=0, data=89, model=0): 178, ProcessCoord(pipe=0, data=89, model=1): 179, ProcessCoord(pipe=0, data=90, model=0): 180, ProcessCoord(pipe=0, data=90, model=1): 181, ProcessCoord(pipe=0, data=91, model=0): 182, ProcessCoord(pipe=0, data=91, model=1): 183, ProcessCoord(pipe=0, data + 0: =92, model=0): 184, ProcessCoord(pipe=0, data=92, model=1): 185, ProcessCoord(pipe=0, data=93, model=0): 186, ProcessCoord(pipe=0, data=93, model=1): 187, ProcessCoord(pipe=0, data=94, model=0): 188, ProcessCoord(pipe=0, data=94, model=1): 189, ProcessCoord(pipe=0, data=95, model=0): 190, ProcessCoord(pipe=0, data=95, model=1): 191, ProcessCoord(pipe=0, data=96, model=0): 192, ProcessCoord(pipe=0, data=96, model=1): 193, ProcessCoord(pipe=0, data=97, model=0): 194, ProcessCoord(pipe=0, data=97, model=1): 195, ProcessCoord(pipe=0, data=98, model=0): 196, ProcessCoord(pipe=0, data=98, model=1): 197, ProcessCoord(pipe=0, data=99, model=0): 198, ProcessCoord(pipe=0, data=99, model=1): 199, ProcessCoord(pipe=0, data=100, model=0): 200, ProcessCoord(pipe=0, data=100, model=1): 201, ProcessCoord(pipe=0, data=101, model=0): 202, ProcessCoord(pipe=0, data=101, model=1): 203, ProcessCoord(pipe=0, data=102, model=0): 204, ProcessCoord(pipe=0, data=102, model=1): 205, ProcessCoord(pipe=0, data=103, model=0): 206, Process + 0: Coord(pipe=0, data=103, model=1): 207, ProcessCoord(pipe=0, data=104, model=0): 208, ProcessCoord(pipe=0, data=104, model=1): 209, ProcessCoord(pipe=0, data=105, model=0): 210, ProcessCoord(pipe=0, data=105, model=1): 211, ProcessCoord(pipe=0, data=106, model=0): 212, ProcessCoord(pipe=0, data=106, model=1): 213, ProcessCoord(pipe=0, data=107, model=0): 214, ProcessCoord(pipe=0, data=107, model=1): 215, ProcessCoord(pipe=0, data=108, model=0): 216, ProcessCoord(pipe=0, data=108, model=1): 217, ProcessCoord(pipe=0, data=109, model=0): 218, ProcessCoord(pipe=0, data=109, model=1): 219, ProcessCoord(pipe=0, data=110, model=0): 220, ProcessCoord(pipe=0, data=110, model=1): 221, ProcessCoord(pipe=0, data=111, model=0): 222, ProcessCoord(pipe=0, data=111, model=1): 223, ProcessCoord(pipe=0, data=112, model=0): 224, ProcessCoord(pipe=0, data=112, model=1): 225, ProcessCoord(pipe=0, data=113, model=0): 226, ProcessCoord(pipe=0, data=113, model=1): 227, ProcessCoord(pipe=0, data=114, model=0): 228, ProcessCoord(pipe=0 + 0: , data=114, model=1): 229, ProcessCoord(pipe=0, data=115, model=0): 230, ProcessCoord(pipe=0, data=115, model=1): 231, ProcessCoord(pipe=0, data=116, model=0): 232, ProcessCoord(pipe=0, data=116, model=1): 233, ProcessCoord(pipe=0, data=117, model=0): 234, ProcessCoord(pipe=0, data=117, model=1): 235, ProcessCoord(pipe=0, data=118, model=0): 236, ProcessCoord(pipe=0, data=118, model=1): 237, ProcessCoord(pipe=0, data=119, model=0): 238, ProcessCoord(pipe=0, data=119, model=1): 239, ProcessCoord(pipe=0, data=120, model=0): 240, ProcessCoord(pipe=0, data=120, model=1): 241, ProcessCoord(pipe=0, data=121, model=0): 242, ProcessCoord(pipe=0, data=121, model=1): 243, ProcessCoord(pipe=0, data=122, model=0): 244, ProcessCoord(pipe=0, data=122, model=1): 245, ProcessCoord(pipe=0, data=123, model=0): 246, ProcessCoord(pipe=0, data=123, model=1): 247, ProcessCoord(pipe=0, data=124, model=0): 248, ProcessCoord(pipe=0, data=124, model=1): 249, ProcessCoord(pipe=0, data=125, model=0): 250, ProcessCoord(pipe=0, data=125, + 0: model=1): 251, ProcessCoord(pipe=0, data=126, model=0): 252, ProcessCoord(pipe=0, data=126, model=1): 253, ProcessCoord(pipe=0, data=127, model=0): 254, ProcessCoord(pipe=0, data=127, model=1): 255} + 0: [2023-05-08 11:25:45,393] [INFO] [module.py:366:_partition_layers] Partitioning pipeline stages with method type:transformer + 0: stage=0 layers=37 + 0: 0: _to_float16 + 0: 1: EmbeddingPipe + 0: 2: + 0: 3: ParallelTransformerLayerPipe + 0: 4: ParallelTransformerLayerPipe + 0: 5: ParallelTransformerLayerPipe + 0: 6: ParallelTransformerLayerPipe + 0: 7: ParallelTransformerLayerPipe + 0: 8: ParallelTransformerLayerPipe + 0: 9: ParallelTransformerLayerPipe + 0: 10: ParallelTransformerLayerPipe + 0: 11: ParallelTransformerLayerPipe + 0: 12: ParallelTransformerLayerPipe + 0: 13: ParallelTransformerLayerPipe + 0: 14: ParallelTransformerLayerPipe + 0: 15: ParallelTransformerLayerPipe + 0: 16: ParallelTransformerLayerPipe + 0: 17: ParallelTransformerLayerPipe + 0: 18: ParallelTransformerLayerPipe + 0: 19: ParallelTransformerLayerPipe + 0: 20: ParallelTransformerLayerPipe + 0: 21: ParallelTransformerLayerPipe + 0: 22: ParallelTransformerLayerPipe + 0: 23: ParallelTransformerLayerPipe + 0: 24: ParallelTransformerLayerPipe + 0: 25: ParallelTransformerLayerPipe + 0: 26: ParallelTransformerLayerPipe + 0: 27: ParallelTransformerLayerPipe + 0: 28: ParallelTransformerLayerPipe + 0: 29: ParallelTransformerLayerPipe + 0: 30: ParallelTransformerLayerPipe + 0: 31: ParallelTransformerLayerPipe + 0: 32: ParallelTransformerLayerPipe + 0: 33: undo + 0: 34: MixedFusedLayerNorm + 0: 35: EmbeddingPipe + 0: 36: float16_to_fp32 + 0: loss: CrossEntropy + 0: [2023-05-08 11:25:45,890] [INFO] [utils.py:827:see_memory_usage] After Building Model + 0: [2023-05-08 11:25:45,891] [INFO] [utils.py:828:see_memory_usage] MA 2.4 GB Max_MA 2.4 GB CA 2.45 GB Max_CA 2 GB + 0: [2023-05-08 11:25:45,891] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 38.54 GB, percent = 7.7% + 0: setting training iterations to 0 + 0: > learning rate decay style: cosine + 0: DeepSpeed is enabled. + 0: [2023-05-08 11:25:45,894] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.5, git-hash=unknown, git-branch=unknown + 0: [2023-05-08 11:25:51,127] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False + 0: [2023-05-08 11:25:51,128] [INFO] [logging.py:68:log_dist] [Rank 0] Removing param_group that has no 'params' in the client Optimizer + 0: [2023-05-08 11:25:51,128] [INFO] [logging.py:68:log_dist] [Rank 0] Using client Optimizer as basic optimizer + 0: [2023-05-08 11:25:51,144] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = FusedAdam + 0: [2023-05-08 11:25:51,144] [INFO] [logging.py:68:log_dist] [Rank 0] Creating BF16 optimizer + 0: [2023-05-08 11:25:51,268] [INFO] [utils.py:827:see_memory_usage] begin bf16_optimizer + 0: [2023-05-08 11:25:51,269] [INFO] [utils.py:828:see_memory_usage] MA 2.39 GB Max_MA 2.41 GB CA 2.46 GB Max_CA 2 GB + 0: [2023-05-08 11:25:51,269] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.13 GB, percent = 7.8% + 0: ninja: no work to do. + 0: Time to load utils op: 0.3941631317138672 secondsTime to load utils op: 0.3933887481689453 secondsTime to load utils op: 0.28540563583374023 seconds + 0: + 0: + 0: Time to load utils op: 0.3935983180999756 seconds + 4: Time to load utils op: 0.3998265266418457 seconds + 4: Time to load utils op: 0.40021753311157227 seconds + 4: Time to load utils op: 0.40024685859680176 seconds + 4: Time to load utils op: 0.40079236030578613 seconds +24: Time to load utils op: 0.3892202377319336 seconds +24: Time to load utils op: 0.38708949089050293 seconds +24: Time to load utils op: 0.38974857330322266 seconds +24: Time to load utils op: 0.39034008979797363 seconds +20: Time to load utils op: 0.3920412063598633 secondsTime to load utils op: 0.3920423984527588 seconds +20: +20: Time to load utils op: 0.39114904403686523 secondsTime to load utils op: 0.39136719703674316 seconds +20: +21: Time to load utils op: 0.38397812843322754 seconds +25: Time to load utils op: 0.3877694606781006 secondsTime to load utils op: 0.38894104957580566 seconds +25: +25: Time to load utils op: 0.3886103630065918 seconds + 1: Time to load utils op: 0.40169525146484375 secondsTime to load utils op: 0.40133142471313477 seconds + 1: + 1: Time to load utils op: 0.4013674259185791 seconds +25: Time to load utils op: 0.38744163513183594 seconds + 1: Time to load utils op: 0.40140199661254883 seconds +21: Time to load utils op: 0.3840608596801758 secondsTime to load utils op: 0.38407087326049805 seconds +21: +21: Time to load utils op: 0.38407135009765625 seconds +27: Time to load utils op: 0.38424253463745117 secondsTime to load utils op: 0.38425111770629883 seconds +27: +27: Time to load utils op: 0.3842637538909912 seconds +12: Time to load utils op: 0.3755004405975342 seconds +12: Time to load utils op: 0.3755483627319336 secondsTime to load utils op: 0.3751249313354492 seconds +12: +16: Time to load utils op: 0.39392971992492676 seconds +16: Time to load utils op: 0.3940441608428955 seconds +28: Time to load utils op: 0.38271450996398926 secondsTime to load utils op: 0.38272833824157715 secondsTime to load utils op: 0.3827095031738281 seconds +28: +28: +28: Time to load utils op: 0.38274240493774414 seconds +27: Time to load utils op: 0.3842778205871582 seconds +12: Time to load utils op: 0.3776695728302002 seconds +16: Time to load utils op: 0.3942737579345703 seconds +16: Time to load utils op: 0.39461445808410645 seconds +10: Time to load utils op: 0.39679884910583496 secondsTime to load utils op: 0.39679622650146484 secondsTime to load utils op: 0.3972005844116211 seconds +10: +10: +10: Time to load utils op: 0.39673399925231934 seconds +14: Time to load utils op: 0.39513134956359863 secondsTime to load utils op: 0.3964505195617676 seconds +14: +17: Time to load utils op: 0.39344024658203125 seconds + 3: Time to load utils op: 0.3976712226867676 seconds + 3: Time to load utils op: 0.397686243057251 seconds +14: Time to load utils op: 0.39693570137023926 seconds + 3: Time to load utils op: 0.3977046012878418 secondsTime to load utils op: 0.39770936965942383 seconds + 3: + 9: Time to load utils op: 0.3932793140411377 secondsTime to load utils op: 0.39258790016174316 secondsTime to load utils op: 0.39299917221069336 seconds + 9: + 9: +17: Time to load utils op: 0.3942577838897705 seconds +17: Time to load utils op: 0.39370107650756836 secondsTime to load utils op: 0.3937094211578369 seconds +17: +14: Time to load utils op: 0.395827054977417 seconds + 9: Time to load utils op: 0.38518261909484863 seconds + 5: Time to load utils op: 0.37658047676086426 secondsTime to load utils op: 0.37129878997802734 seconds + 5: + 5: Time to load utils op: 0.3726785182952881 secondsTime to load utils op: 0.37237977981567383 seconds + 5: +22: Time to load utils op: 0.39205026626586914 secondsTime to load utils op: 0.3921835422515869 seconds +22: +22: Time to load utils op: 0.3927152156829834 seconds +13: Time to load utils op: 0.3879241943359375 seconds +13: Time to load utils op: 0.38794589042663574 secondsTime to load utils op: 0.38794946670532227 seconds +13: +23: Time to load utils op: 0.3919706344604492 secondsTime to load utils op: 0.39157605171203613 seconds +23: +23: Time to load utils op: 0.3915417194366455 seconds +22: Time to load utils op: 0.39223718643188477 seconds +13: Time to load utils op: 0.38797974586486816 seconds + 2: Time to load utils op: 0.40316128730773926 secondsTime to load utils op: 0.40315937995910645 seconds + 2: + 2: Time to load utils op: 0.40314221382141113 seconds + 2: Time to load utils op: 0.4032454490661621 seconds +23: Time to load utils op: 0.39186668395996094 seconds +30: Time to load utils op: 0.38367462158203125 seconds +30: Time to load utils op: 0.3836798667907715 seconds +15: Time to load utils op: 0.3857240676879883 seconds +19: Time to load utils op: 0.3860597610473633 seconds +19: Time to load utils op: 0.3860781192779541 secondsTime to load utils op: 0.3860764503479004 seconds +19: +29: Time to load utils op: 0.3890104293823242 secondsTime to load utils op: 0.388791561126709 seconds +29: +29: Time to load utils op: 0.38908863067626953 seconds +30: Time to load utils op: 0.38370680809020996 secondsTime to load utils op: 0.38370275497436523 seconds +30: +15: Time to load utils op: 0.3857603073120117 seconds +18: Time to load utils op: 0.3857405185699463 secondsTime to load utils op: 0.38574814796447754 seconds +18: +18: Time to load utils op: 0.38575100898742676 seconds +18: Time to load utils op: 0.3857595920562744 seconds +11: Time to load utils op: 0.39459776878356934 seconds +11: Time to load utils op: 0.394620418548584 seconds +15: Time to load utils op: 0.3857741355895996 seconds +15: Time to load utils op: 0.38577961921691895 seconds +29: Time to load utils op: 0.38856983184814453 seconds +19: Time to load utils op: 0.38610053062438965 seconds +26: Time to load utils op: 0.38335442543029785 seconds +26: Time to load utils op: 0.3833801746368408 seconds +26: Time to load utils op: 0.38338589668273926 seconds +31: Time to load utils op: 0.38330936431884766 seconds +31: Time to load utils op: 0.38332128524780273 secondsTime to load utils op: 0.38332581520080566 seconds +31: +11: Time to load utils op: 0.39464378356933594 secondsTime to load utils op: 0.39464330673217773 seconds +11: + 6: Time to load utils op: 0.39748382568359375 secondsTime to load utils op: 0.3974888324737549 seconds + 6: + 6: Time to load utils op: 0.3975026607513428 secondsTime to load utils op: 0.3974936008453369 seconds + 6: +31: Time to load utils op: 0.3833348751068115 seconds + 8: Time to load utils op: 0.3985862731933594 secondsTime to load utils op: 0.40137362480163574 seconds + 8: + 8: Time to load utils op: 0.4002082347869873 secondsTime to load utils op: 0.40006446838378906 seconds + 8: +26: Time to load utils op: 0.3834071159362793 seconds + 7: Time to load utils op: 0.40164828300476074 secondsTime to load utils op: 0.40189623832702637 seconds + 7: + 7: Time to load utils op: 0.4003603458404541 seconds + 7: Time to load utils op: 0.4006199836730957 seconds + 0: [2023-05-08 11:25:51,683] [INFO] [utils.py:827:see_memory_usage] before initializing group 0 + 0: [2023-05-08 11:25:51,684] [INFO] [utils.py:828:see_memory_usage] MA 2.39 GB Max_MA 2.39 GB CA 2.46 GB Max_CA 2 GB + 0: [2023-05-08 11:25:51,684] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.14 GB, percent = 7.8% +15: Time to load utils op: 0.0008118152618408203 seconds +15: Time to load utils op: 0.0008771419525146484 secondsTime to load utils op: 0.00086212158203125 seconds +15: +15: Time to load utils op: 0.0009338855743408203 seconds + 0: Time to load utils op: 0.0006077289581298828 seconds + 0: Time to load utils op: 0.0006136894226074219 seconds + 0: Time to load utils op: 0.0004944801330566406 seconds +16: Time to load utils op: 0.001007080078125 secondsTime to load utils op: 0.0010275840759277344 seconds +16: +16: Time to load utils op: 0.0008952617645263672 seconds +16: Time to load utils op: 0.0009906291961669922 seconds +17: Time to load utils op: 0.0009188652038574219 seconds +17: Time to load utils op: 0.0008587837219238281 secondsTime to load utils op: 0.000949859619140625 seconds +17: +17: Time to load utils op: 0.0009500980377197266 seconds + 6: Time to load utils op: 0.0008909702301025391 secondsTime to load utils op: 0.0009071826934814453 seconds + 6: + 6: Time to load utils op: 0.0008935928344726562 seconds + 6: Time to load utils op: 0.0009181499481201172 seconds + 2: Time to load utils op: 0.0007505416870117188 seconds + 4: Time to load utils op: 0.0007877349853515625 seconds + 2: Time to load utils op: 0.0007898807525634766 secondsTime to load utils op: 0.0007791519165039062 seconds + 2: + 2: Time to load utils op: 0.0008795261383056641 seconds + 4: Time to load utils op: 0.0009844303131103516 seconds + 4: Time to load utils op: 0.0009341239929199219 seconds + 4: Time to load utils op: 0.0009453296661376953 seconds +30: Time to load utils op: 0.0009071826934814453 seconds +30: Time to load utils op: 0.0009770393371582031 secondsTime to load utils op: 0.0009801387786865234 seconds +30: +30: Time to load utils op: 0.001041412353515625 seconds +29: Time to load utils op: 0.0007543563842773438 seconds +14: Time to load utils op: 0.0008883476257324219 seconds +14: Time to load utils op: 0.0008780956268310547 seconds +14: Time to load utils op: 0.0008859634399414062 seconds +29: Time to load utils op: 0.0007545948028564453 seconds +29: Time to load utils op: 0.0008420944213867188 seconds +14: Time to load utils op: 0.0008559226989746094 seconds +29: Time to load utils op: 0.0009219646453857422 seconds +24: Time to load utils op: 0.0009479522705078125 secondsTime to load utils op: 0.0009477138519287109 seconds +24: + 7: Time to load utils op: 0.0008146762847900391 seconds +24: Time to load utils op: 0.0009102821350097656 seconds +24: Time to load utils op: 0.0009341239929199219 seconds + 7: Time to load utils op: 0.0008385181427001953 secondsTime to load utils op: 0.0007977485656738281 seconds + 7: + 9: Time to load utils op: 0.0005373954772949219 secondsTime to load utils op: 0.0005304813385009766 seconds + 9: + 9: Time to load utils op: 0.0005097389221191406 seconds + 7: Time to load utils op: 0.0009453296661376953 seconds +25: Time to load utils op: 0.0009350776672363281 secondsTime to load utils op: 0.0009305477142333984 secondsTime to load utils op: 0.0009257793426513672 seconds +25: +25: + 8: Time to load utils op: 0.0010035037994384766 secondsTime to load utils op: 0.0009932518005371094 seconds + 8: +25: Time to load utils op: 0.0009045600891113281 seconds + 9: Time to load utils op: 0.0003452301025390625 seconds + 8: Time to load utils op: 0.0010230541229248047 seconds +11: Time to load utils op: 0.0008101463317871094 seconds + 8: Time to load utils op: 0.0010135173797607422 seconds +11: Time to load utils op: 0.000881195068359375 seconds +11: Time to load utils op: 0.0008661746978759766 seconds +22: Time to load utils op: 0.0005726814270019531 seconds +22: Time to load utils op: 0.0007452964782714844 seconds +11: Time to load utils op: 0.0009050369262695312 seconds +22: Time to load utils op: 0.0007369518280029297 seconds + 3: Time to load utils op: 0.0006864070892333984 seconds +22: Time to load utils op: 0.0008189678192138672 seconds +12: Time to load utils op: 0.0005619525909423828 seconds +13: Time to load utils op: 0.0006785392761230469 seconds +12: Time to load utils op: 0.0005099773406982422 seconds + 3: Time to load utils op: 0.0008575916290283203 seconds + 3: Time to load utils op: 0.0008990764617919922 seconds +12: Time to load utils op: 0.0006365776062011719 seconds +12: Time to load utils op: 0.0006654262542724609 seconds + 3: Time to load utils op: 0.0009360313415527344 seconds +28: Time to load utils op: 0.0007321834564208984 seconds +13: Time to load utils op: 0.0008091926574707031 secondsTime to load utils op: 0.0008633136749267578 seconds +13: +13: Time to load utils op: 0.0009253025054931641 seconds +23: Time to load utils op: 0.0007598400115966797 seconds +28: Time to load utils op: 0.0009379386901855469 seconds +28: Time to load utils op: 0.0008668899536132812 seconds +21: Time to load utils op: 0.0008535385131835938 seconds +28: Time to load utils op: 0.0009179115295410156 seconds +18: Time to load utils op: 0.0009608268737792969 seconds +31: Time to load utils op: 0.0009303092956542969 seconds +21: Time to load utils op: 0.0008847713470458984 secondsTime to load utils op: 0.0009322166442871094 seconds +21: +18: Time to load utils op: 0.0010869503021240234 seconds +23: Time to load utils op: 0.0009245872497558594 seconds +18: Time to load utils op: 0.0010411739349365234 seconds +31: Time to load utils op: 0.0009899139404296875 secondsTime to load utils op: 0.001031637191772461 seconds +31: +21: Time to load utils op: 0.0010037422180175781 seconds +23: Time to load utils op: 0.0008571147918701172 seconds +18: Time to load utils op: 0.0010533332824707031 seconds +23: Time to load utils op: 0.0009393692016601562 seconds +31: Time to load utils op: 0.0011065006256103516 seconds + 1: Time to load utils op: 0.0007016658782958984 seconds + 1: Time to load utils op: 0.0008821487426757812 secondsTime to load utils op: 0.0008707046508789062 seconds + 1: + 1: Time to load utils op: 0.0009317398071289062 seconds +10: Time to load utils op: 0.0008156299591064453 seconds +10: Time to load utils op: 0.0008904933929443359 seconds +10: Time to load utils op: 0.0008807182312011719 seconds +19: Time to load utils op: 0.0007524490356445312 seconds +10: Time to load utils op: 0.0008981227874755859 seconds +26: Time to load utils op: 0.0009455680847167969 seconds +26: Time to load utils op: 0.0009152889251708984 seconds +26: Time to load utils op: 0.000972747802734375 seconds +26: Time to load utils op: 0.0009746551513671875 seconds +19: Time to load utils op: 0.0009644031524658203 secondsTime to load utils op: 0.0009224414825439453 seconds +19: +19: Time to load utils op: 0.0009868144989013672 seconds + 5: Time to load utils op: 0.00042748451232910156 seconds + 5: Time to load utils op: 0.0005676746368408203 seconds + 5: Time to load utils op: 0.00049591064453125 seconds + 5: Time to load utils op: 0.0005521774291992188 seconds +27: Time to load utils op: 0.0010731220245361328 secondsTime to load utils op: 0.0010838508605957031 seconds +27: +27: Time to load utils op: 0.0010714530944824219 seconds +27: Time to load utils op: 0.0011417865753173828 seconds +20: Time to load utils op: 0.0008401870727539062 seconds +20: Time to load utils op: 0.0009436607360839844 seconds +20: Time to load utils op: 0.0008330345153808594 seconds +20: Time to load utils op: 0.0009713172912597656 seconds + 0: [2023-05-08 11:25:51,810] [INFO] [utils.py:827:see_memory_usage] after initializing group 0 + 0: [2023-05-08 11:25:51,810] [INFO] [utils.py:828:see_memory_usage] MA 4.79 GB Max_MA 4.79 GB CA 6.07 GB Max_CA 6 GB + 0: [2023-05-08 11:25:51,811] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.24 GB, percent = 7.8% + 0: [2023-05-08 11:25:51,919] [INFO] [utils.py:827:see_memory_usage] before initializing group 1 + 0: [2023-05-08 11:25:51,920] [INFO] [utils.py:828:see_memory_usage] MA 4.79 GB Max_MA 4.79 GB CA 6.07 GB Max_CA 6 GB + 0: [2023-05-08 11:25:51,920] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.28 GB, percent = 7.8% + 0: [2023-05-08 11:25:52,030] [INFO] [utils.py:827:see_memory_usage] after initializing group 1 + 0: [2023-05-08 11:25:52,030] [INFO] [utils.py:828:see_memory_usage] MA 7.02 GB Max_MA 7.02 GB CA 9.44 GB Max_CA 9 GB + 0: [2023-05-08 11:25:52,031] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.33 GB, percent = 7.8% + 0: [2023-05-08 11:25:52,138] [INFO] [utils.py:827:see_memory_usage] before initializing group 2 + 0: [2023-05-08 11:25:52,139] [INFO] [utils.py:828:see_memory_usage] MA 7.02 GB Max_MA 7.02 GB CA 9.44 GB Max_CA 9 GB + 0: [2023-05-08 11:25:52,139] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.38 GB, percent = 7.8% + 0: [2023-05-08 11:25:52,251] [INFO] [utils.py:827:see_memory_usage] after initializing group 2 + 0: [2023-05-08 11:25:52,251] [INFO] [utils.py:828:see_memory_usage] MA 7.02 GB Max_MA 7.02 GB CA 9.44 GB Max_CA 9 GB + 0: [2023-05-08 11:25:52,251] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.43 GB, percent = 7.8% + 0: [2023-05-08 11:25:52,359] [INFO] [utils.py:827:see_memory_usage] before initialize_optimizer + 0: [2023-05-08 11:25:52,359] [INFO] [utils.py:828:see_memory_usage] MA 7.02 GB Max_MA 7.02 GB CA 9.44 GB Max_CA 9 GB + 0: [2023-05-08 11:25:52,360] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.47 GB, percent = 7.8% + 0: [2023-05-08 11:25:52,646] [INFO] [utils.py:827:see_memory_usage] end initialize_optimizer + 0: [2023-05-08 11:25:52,646] [INFO] [utils.py:828:see_memory_usage] MA 7.1 GB Max_MA 7.1 GB CA 9.44 GB Max_CA 9 GB + 0: [2023-05-08 11:25:52,646] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.61 GB, percent = 7.9% + 0: [2023-05-08 11:25:52,754] [INFO] [utils.py:827:see_memory_usage] end bf16_optimizer + 0: [2023-05-08 11:25:52,755] [INFO] [utils.py:828:see_memory_usage] MA 7.1 GB Max_MA 7.1 GB CA 9.44 GB Max_CA 9 GB + 0: [2023-05-08 11:25:52,755] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 39.63 GB, percent = 7.9% + 0: [2023-05-08 11:25:52,755] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam + 0: [2023-05-08 11:25:52,755] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using client LR scheduler + 0: [2023-05-08 11:25:52,755] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = + 0: [2023-05-08 11:25:52,756] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0002, 0.0002, 0.0002], mom=[(0.9, 0.999), (0.9, 0.999), (0.9, 0.999)] + 0: [2023-05-08 11:25:52,756] [INFO] [config.py:1007:print] DeepSpeedEngine configuration: + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] activation_checkpointing_config { + 0: "partition_activations": false, + 0: "contiguous_memory_optimization": false, + 0: "cpu_checkpointing": false, + 0: "number_checkpoints": null, + 0: "synchronize_checkpoint_boundary": false, + 0: "profile": false + 0: } + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] amp_enabled .................. False + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] amp_params ................... False + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] autotuning_config ............ { + 0: "enabled": false, + 0: "start_step": null, + 0: "end_step": null, + 0: "metric_path": null, + 0: "arg_mappings": null, + 0: "metric": "throughput", + 0: "model_info": null, + 0: "results_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_results", + 0: "exps_dir": "/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/autotuning_exps", + 0: "overwrite": true, + 0: "fast": true, + 0: "start_profile_step": 3, + 0: "end_profile_step": 5, + 0: "tuner_type": "gridsearch", + 0: "tuner_early_stopping": 5, + 0: "tuner_num_trials": 50, + 0: "model_info_path": null, + 0: "mp_size": 1, + 0: "max_train_batch_size": null, + 0: "min_train_batch_size": 1, + 0: "max_train_micro_batch_size_per_gpu": 1.024000e+03, + 0: "min_train_micro_batch_size_per_gpu": 1, + 0: "num_tuning_micro_batch_sizes": 3 + 0: } + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] bfloat16_enabled ............. True + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] checkpoint_parallel_write_pipeline False + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] checkpoint_tag_validation_enabled True + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] checkpoint_tag_validation_fail False + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] comms_config ................. + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] communication_data_type ...... None + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_pa + 0: rameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] curriculum_enabled ........... False + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] curriculum_params ............ False + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] dataloader_drop_last ......... False + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] disable_allgather ............ False + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] dump_state ................... False + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] dynamic_loss_scale_args ...... None + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] eigenvalue_enabled ........... False + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] eigenvalue_gas_boundary_resolution 1 + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] eigenvalue_layer_name ........ bert.encoder.layer + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] eigenvalue_layer_num ......... 0 + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] eigenvalue_max_iter .......... 100 + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] eigenvalue_stability ......... 1e-06 + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] eigenvalue_tol ............... 0.01 + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] eigenvalue_verbose ........... False + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] elasticity_enabled ........... False + 0: [2023-05-08 11:25:52,757] [INFO] [config.py:1011:print] flops_profiler_config ........ { + 0: "enabled": false, + 0: "profile_step": 1, + 0: "module_depth": -1, + 0: "top_modules": 1, + 0: "detailed": true, + 0: "output_file": null + 0: } + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] fp16_auto_cast ............... None + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] fp16_enabled ................. False + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] fp16_master_weights_and_gradients False + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] global_rank .................. 0 + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] gradient_accumulation_steps .. 4 + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] gradient_clipping ............ 1.0 + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] gradient_predivide_factor .... 1.0 + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] initial_dynamic_scale ........ 1 + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] load_universal_checkpoint .... False + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] loss_scale ................... 1.0 + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] memory_breakdown ............. False + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] monitor_config ............... + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] nebula_config ................ { + 0: "enabled": false, + 0: "persistent_storage_path": null, + 0: "persistent_time_interval": 100, + 0: "num_of_version_in_retention": 2, + 0: "enable_nebula_load": true, + 0: "load_path": null + 0: } + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] optimizer_legacy_fusion ...... False + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] optimizer_name ............... None + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] optimizer_params ............. None + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] pld_enabled .................. False + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] pld_params ................... False + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] prescale_gradients ........... False + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] scheduler_name ............... None + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] scheduler_params ............. None + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] sparse_attention ............. None + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] sparse_gradients_enabled ..... False + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] steps_per_print .............. 2000 + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] train_batch_size ............. 1024 + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] train_micro_batch_size_per_gpu 2 + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] use_node_local_storage ....... False + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] wall_clock_breakdown ......... False + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] world_size ................... 128 + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] zero_allow_untested_optimizer False + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] zero_config .................. stage=0 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500000000 allgather_partitions=True allgather_bucket_size=500000000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=None sub_group_size=1000000000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50000000 param_persistence_threshold=100000 model_persistence_threshold=9223372036854775807 max_live_parameters=1000000000 max_reuse_distance=1000000000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] zero_enabled ................. False + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:1011:print] zero_optimization_stage ...... 0 + 0: [2023-05-08 11:25:52,758] [INFO] [config.py:996:print_user_config] json = { + 0: "train_micro_batch_size_per_gpu": 2, + 0: "train_batch_size": 1.024000e+03, + 0: "gradient_clipping": 1.0, + 0: "zero_optimization": { + 0: "stage": 0 + 0: }, + 0: "bf16": { + 0: "enabled": true + 0: }, + 0: "steps_per_print": 2.000000e+03, + 0: "wall_clock_breakdown": false + 0: } + 0: Time to load utils op: 0.00562596321105957 seconds + 0: [2023-05-08 11:25:52,764] [INFO] [engine.py:87:__init__] CONFIG: micro_batches=4 micro_batch_size=2 + 0: ninja: no work to do. + 0: Time to load utils op: 0.27188920974731445 seconds + 6: Time to load utils op: 0.2251279354095459 seconds +15: Time to load utils op: 0.2253885269165039 seconds + 2: Time to load utils op: 0.2882423400878906 seconds + 1: Time to load utils op: 0.2899150848388672 seconds +21: Time to load utils op: 0.2400679588317871 seconds +13: Time to load utils op: 0.23734426498413086 seconds + 6: Time to load utils op: 0.23096132278442383 seconds + 7: Time to load utils op: 0.29054951667785645 seconds +15: Time to load utils op: 0.2313094139099121 seconds + 2: Time to load utils op: 0.29399991035461426 seconds + 1: Time to load utils op: 0.2951169013977051 seconds +21: Time to load utils op: 0.2278902530670166 seconds +13: Time to load utils op: 0.22507762908935547 seconds + 4: Time to load utils op: 0.2973189353942871 seconds +10: Time to load utils op: 0.27512693405151367 seconds +10: Time to load utils op: 0.2763690948486328 seconds + 6: Time to load utils op: 0.24969148635864258 seconds + 7: Time to load utils op: 0.2966651916503906 seconds +15: Time to load utils op: 0.23772430419921875 seconds + 2: Time to load utils op: 0.29968714714050293 seconds +11: Time to load utils op: 0.2889595031738281 seconds + 1: Time to load utils op: 0.30213332176208496 seconds +21: Time to load utils op: 0.2337024211883545 seconds +13: Time to load utils op: 0.23113632202148438 seconds +11: Time to load utils op: 0.29221272468566895 seconds + 4: Time to load utils op: 0.3035883903503418 seconds +11: Time to load utils op: 0.29216742515563965 seconds +11: Time to load utils op: 0.2926616668701172 seconds +10: Time to load utils op: 0.2823147773742676 seconds +10: Time to load utils op: 0.2826380729675293 seconds + 3: Time to load utils op: 0.2679870128631592 seconds + 6: Time to load utils op: 0.24351930618286133 seconds + 7: Time to load utils op: 0.3026878833770752 seconds + 2: Time to load utils op: 0.30618906021118164 seconds +15: Time to load utils op: 0.26236844062805176 seconds + 1: Time to load utils op: 0.30797648429870605 seconds +21: Time to load utils op: 0.23971176147460938 seconds +12: Time to load utils op: 0.30022692680358887 seconds +13: Time to load utils op: 0.2372119426727295 seconds + 8: Time to load utils op: 0.3050990104675293 seconds +14: Time to load utils op: 0.30226588249206543 seconds + 9: Time to load utils op: 0.3060882091522217 seconds + 5: Time to load utils op: 0.30420374870300293 seconds +16: Time to load utils op: 0.302579402923584 seconds + 4: Time to load utils op: 0.30966615676879883 seconds + 7: Time to load utils op: 0.30967283248901367 seconds + 3: Time to load utils op: 0.2782282829284668 seconds +22: Time to load utils op: 0.3016629219055176 seconds +17: Time to load utils op: 0.3045518398284912 seconds +24: Time to load utils op: 0.3013918399810791 seconds +12: Time to load utils op: 0.30592823028564453 seconds + 8: Time to load utils op: 0.31107139587402344 seconds + 0: Time to load utils op: 0.3157334327697754 seconds +14: Time to load utils op: 0.30790185928344727 seconds +16: Time to load utils op: 0.3082609176635742 seconds + 5: Time to load utils op: 0.3103346824645996 seconds + 9: Time to load utils op: 0.31286191940307617 seconds +24: Time to load utils op: 0.3039252758026123 seconds +23: Time to load utils op: 0.30530667304992676 seconds + 4: Time to load utils op: 0.31621646881103516 seconds + 3: Time to load utils op: 0.28417301177978516 seconds +20: Time to load utils op: 0.3067903518676758 seconds + 0: Time to load utils op: 0.3194911479949951 seconds +29: Time to load utils op: 0.30396533012390137 seconds +22: Time to load utils op: 0.3078734874725342 seconds +25: Time to load utils op: 0.30416250228881836 seconds +17: Time to load utils op: 0.31064486503601074 seconds +30: Time to load utils op: 0.25562500953674316 seconds + 9: Time to load utils op: 0.31595635414123535 seconds +12: Time to load utils op: 0.3121199607849121 seconds + 8: Time to load utils op: 0.3172280788421631 secondsTime to load utils op: 0.3177003860473633 seconds + 8: +14: Time to load utils op: 0.3147416114807129 seconds +14: Time to load utils op: 0.3140740394592285 seconds + 5: Time to load utils op: 0.31729865074157715 seconds +16: Time to load utils op: 0.3139617443084717 seconds +23: Time to load utils op: 0.3112215995788574 seconds + 0: Time to load utils op: 0.3232393264770508 seconds +24: Time to load utils op: 0.3096127510070801 seconds + 3: Time to load utils op: 0.28316450119018555 seconds +20: Time to load utils op: 0.3129732608795166 seconds +29: Time to load utils op: 0.3107614517211914 seconds +22: Time to load utils op: 0.3134324550628662 seconds +25: Time to load utils op: 0.30977296829223633 seconds +17: Time to load utils op: 0.31659793853759766 seconds +30: Time to load utils op: 0.2736325263977051 seconds + 9: Time to load utils op: 0.3218848705291748 seconds +26: Time to load utils op: 0.2641918659210205 seconds +12: Time to load utils op: 0.31835103034973145 seconds +16: Time to load utils op: 0.31986570358276367 seconds + 5: Time to load utils op: 0.3231964111328125 seconds +23: Time to load utils op: 0.3163149356842041 seconds +24: Time to load utils op: 0.3159477710723877 seconds +29: Time to load utils op: 0.3168199062347412 seconds +22: Time to load utils op: 0.31966447830200195 seconds +20: Time to load utils op: 0.3192331790924072 seconds +18: Time to load utils op: 0.2614152431488037 seconds +25: Time to load utils op: 0.3161485195159912 seconds +17: Time to load utils op: 0.32320404052734375 seconds +19: Time to load utils op: 0.2733776569366455 seconds +30: Time to load utils op: 0.26765942573547363 seconds +26: Time to load utils op: 0.2702338695526123 seconds +25: Time to load utils op: 0.3187680244445801 seconds +23: Time to load utils op: 0.3229846954345703 seconds +27: Time to load utils op: 0.26821160316467285 seconds +20: Time to load utils op: 0.3259413242340088 seconds +29: Time to load utils op: 0.3217809200286865 seconds +18: Time to load utils op: 0.26150059700012207 seconds +30: Time to load utils op: 0.28611254692077637 seconds +19: Time to load utils op: 0.2828342914581299 seconds +31: Time to load utils op: 0.2467794418334961 seconds +26: Time to load utils op: 0.27649807929992676 seconds +27: Time to load utils op: 0.25885891914367676 seconds +28: Time to load utils op: 0.24294185638427734 seconds +18: Time to load utils op: 0.27663731575012207 seconds +19: Time to load utils op: 0.2858278751373291 seconds +31: Time to load utils op: 0.25298142433166504 seconds +18: Time to load utils op: 0.27092456817626953 seconds +26: Time to load utils op: 0.2950305938720703 seconds +27: Time to load utils op: 0.26508021354675293 seconds +28: Time to load utils op: 0.248809814453125 seconds +19: Time to load utils op: 0.29221653938293457 seconds +31: Time to load utils op: 0.2591536045074463 seconds +27: Time to load utils op: 0.2715463638305664 seconds +28: Time to load utils op: 0.2548699378967285 seconds +31: Time to load utils op: 0.26525115966796875 seconds +28: Time to load utils op: 0.2612423896789551 seconds +11: Time to load utils op: 0.00580596923828125 secondsTime to load utils op: 0.005353689193725586 seconds +11: +11: Time to load utils op: 0.0055294036865234375 seconds + 8: Time to load utils op: 0.003490447998046875 seconds + 8: Time to load utils op: 0.003467559814453125 seconds +11: Time to load utils op: 0.0004572868347167969 seconds + 8: Time to load utils op: 0.00044989585876464844 seconds + 8: Time to load utils op: 0.00048041343688964844 seconds +10: Time to load utils op: 0.0037076473236083984 seconds +10: Time to load utils op: 0.006104946136474609 seconds +10: Time to load utils op: 0.0003991127014160156 seconds +10: Time to load utils op: 0.0004825592041015625 seconds + 0: Time to load utils op: 0.0063664913177490234 seconds + 0: [2023-05-08 11:25:53,229] [INFO] [engine.py:145:__init__] RANK=0 STAGE=0 LAYERS=37 [0, 37) STAGE_PARAMS=1250178560 (1250.179M) TOTAL_PARAMS=2500357120 (2500.357M) UNIQUE_PARAMS=2500357120 (2500.357M) + 0: [2023-05-08 11:25:53,229] [INFO] [engine.py:145:__init__] RANK=1 STAGE=0 LAYERS=37 [0, 37) STAGE_PARAMS=1250178560 (1250.179M) TOTAL_PARAMS=2500357120 (2500.357M) UNIQUE_PARAMS=2500357120 (2500.357M) + 0: Time to load utils op: 0.00046324729919433594 seconds + 0: Time to load utils op: 0.0003986358642578125 seconds + 0: Time to load utils op: 0.0004107952117919922 seconds +15: Time to load utils op: 0.006606578826904297 secondsTime to load utils op: 0.0065670013427734375 seconds +15: +15: Time to load utils op: 0.006689548492431641 seconds + 2: Time to load utils op: 0.003717184066772461 seconds + 2: Time to load utils op: 0.0037140846252441406 seconds + 4: Time to load utils op: 0.004329204559326172 seconds +19: Time to load utils op: 0.003640890121459961 seconds + 2: Time to load utils op: 0.0066699981689453125 seconds +21: Time to load utils op: 0.0041081905364990234 seconds +21: Time to load utils op: 0.003998279571533203 seconds +13: Time to load utils op: 0.006635427474975586 seconds + 4: Time to load utils op: 0.004099369049072266 seconds +20: Time to load utils op: 0.004137992858886719 seconds +19: Time to load utils op: 0.00414586067199707 secondsTime to load utils op: 0.0042650699615478516 secondsTime to load utils op: 0.004208803176879883 seconds +19: +19: +14: Time to load utils op: 0.0009942054748535156 seconds +13: Time to load utils op: 0.00044155120849609375 seconds + 7: Time to load utils op: 0.003917694091796875 seconds + 6: Time to load utils op: 0.0036361217498779297 seconds +21: Time to load utils op: 0.0004298686981201172 seconds + 4: Time to load utils op: 0.00035691261291503906 seconds +12: Time to load utils op: 0.0038366317749023438 secondsTime to load utils op: 0.006583690643310547 seconds +12: +12: Time to load utils op: 0.0065996646881103516 seconds +14: Time to load utils op: 0.000392913818359375 seconds + 7: Time to load utils op: 0.00047397613525390625 seconds + 6: Time to load utils op: 0.0003948211669921875 secondsTime to load utils op: 0.0004286766052246094 seconds + 6: + 2: Time to load utils op: 0.0004756450653076172 seconds + 1: Time to load utils op: 0.004299163818359375 seconds +20: Time to load utils op: 0.00034689903259277344 seconds + 1: Time to load utils op: 0.004265308380126953 seconds + 4: Time to load utils op: 0.0005071163177490234 seconds +20: Time to load utils op: 0.0005314350128173828 seconds +20: Time to load utils op: 0.0004432201385498047 seconds +14: Time to load utils op: 0.0004792213439941406 seconds +21: Time to load utils op: 0.00046539306640625 seconds +15: Time to load utils op: 0.00034117698669433594 seconds + 1: Time to load utils op: 0.0004620552062988281 seconds + 1: Time to load utils op: 0.0004012584686279297 seconds +25: Time to load utils op: 0.0011739730834960938 seconds +13: Time to load utils op: 0.0004563331604003906 seconds +13: Time to load utils op: 0.00047469139099121094 seconds +30: Time to load utils op: 0.006576061248779297 secondsTime to load utils op: 0.006658792495727539 secondsTime to load utils op: 0.0066411495208740234 seconds +30: +30: + 7: Time to load utils op: 0.0004951953887939453 seconds +25: Time to load utils op: 0.004376649856567383 seconds +16: Time to load utils op: 0.0036668777465820312 seconds + 5: Time to load utils op: 0.00405573844909668 seconds + 5: Time to load utils op: 0.004019260406494141 seconds + 9: Time to load utils op: 0.00417327880859375 secondsTime to load utils op: 0.004157304763793945 seconds + 9: + 9: Time to load utils op: 0.0005407333374023438 seconds +25: Time to load utils op: 0.00048232078552246094 seconds +16: Time to load utils op: 0.0004029273986816406 seconds +24: Time to load utils op: 0.004194736480712891 seconds +17: Time to load utils op: 0.006591796875 seconds +17: Time to load utils op: 0.004247426986694336 seconds + 3: Time to load utils op: 0.004193544387817383 seconds +25: Time to load utils op: 0.0004277229309082031 seconds +22: Time to load utils op: 0.0038466453552246094 seconds +22: Time to load utils op: 0.0004904270172119141 seconds +14: Time to load utils op: 0.0005388259887695312 seconds +24: Time to load utils op: 0.0003781318664550781 seconds + 7: Time to load utils op: 0.0003352165222167969 seconds +30: Time to load utils op: 0.0005002021789550781 seconds +28: Time to load utils op: 0.006684541702270508 seconds + 6: Time to load utils op: 0.00048351287841796875 seconds + 5: Time to load utils op: 0.0004742145538330078 seconds + 9: Time to load utils op: 0.0004923343658447266 seconds +12: Time to load utils op: 0.00047850608825683594 seconds + 3: Time to load utils op: 0.00046563148498535156 seconds +28: Time to load utils op: 0.00039958953857421875 seconds +28: Time to load utils op: 0.0003952980041503906 seconds +16: Time to load utils op: 0.0005004405975341797 seconds +16: Time to load utils op: 0.0005574226379394531 seconds +24: Time to load utils op: 0.00036787986755371094 seconds +22: Time to load utils op: 0.00041985511779785156 seconds + 3: Time to load utils op: 0.0004153251647949219 secondsTime to load utils op: 0.00041484832763671875 seconds + 3: +18: Time to load utils op: 0.0041351318359375 seconds +24: Time to load utils op: 0.0004899501800537109 seconds +17: Time to load utils op: 0.00048542022705078125 seconds +23: Time to load utils op: 0.004096269607543945 seconds +18: Time to load utils op: 0.0004947185516357422 seconds +23: Time to load utils op: 0.004271030426025391 secondsTime to load utils op: 0.004257917404174805 seconds +23: +23: Time to load utils op: 0.0004937648773193359 seconds + 5: Time to load utils op: 0.00040602684020996094 seconds +28: Time to load utils op: 0.0004818439483642578 seconds +17: Time to load utils op: 0.0003724098205566406 seconds +27: Time to load utils op: 0.004186391830444336 secondsTime to load utils op: 0.004238605499267578 seconds +27: +22: Time to load utils op: 0.00046324729919433594 seconds +27: Time to load utils op: 0.0004649162292480469 seconds +26: Time to load utils op: 0.0042803287506103516 seconds +18: Time to load utils op: 0.00046563148498535156 seconds +26: Time to load utils op: 0.004094123840332031 seconds +26: Time to load utils op: 0.00038886070251464844 seconds +29: Time to load utils op: 0.004189014434814453 seconds +29: Time to load utils op: 0.003983020782470703 seconds +26: Time to load utils op: 0.00047850608825683594 seconds +29: Time to load utils op: 0.004236936569213867 seconds +18: Time to load utils op: 0.00046753883361816406 seconds +27: Time to load utils op: 0.00042819976806640625 seconds +31: Time to load utils op: 0.006572246551513672 seconds +31: Time to load utils op: 0.0005018711090087891 secondsTime to load utils op: 0.0004944801330566406 seconds +31: +29: Time to load utils op: 0.000476837158203125 seconds +31: Time to load utils op: 0.0004680156707763672 seconds +30: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +30: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +30: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +30: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +30: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +30: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +30: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +22: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +26: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +14: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +28: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 0: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 0: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 0: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +14: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 0: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 0: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 0: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +14: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +15: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +15: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 0: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +14: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +14: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +14: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +14: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +22: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +22: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +22: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +28: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +15: [2023-05-08 11:25:56,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +15: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +15: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +15: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +22: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +22: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +15: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +22: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +23: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +23: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +23: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +23: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +23: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 0: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +23: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +23: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +14: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +22: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +24: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +24: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +24: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +24: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +24: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +15: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +24: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +24: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +26: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +23: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 1: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +26: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +26: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +26: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +21: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +28: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +28: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +28: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +28: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +28: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +19: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +20: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +17: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +17: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +17: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +17: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +17: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +26: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +26: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +17: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +19: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +19: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +19: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +19: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +21: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +21: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +17: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +19: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +21: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +21: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +19: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +20: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +20: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +20: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +20: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +20: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +24: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +30: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 1: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +21: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +20: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +21: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 1: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 1: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 1: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 1: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 1: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +31: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +31: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +31: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +31: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +31: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +31: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +25: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +25: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +25: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +25: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +25: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +31: [2023-05-08 11:25:56,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +25: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +10: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +12: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +11: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +11: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +11: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +11: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +11: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +25: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +27: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 9: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 9: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 9: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 9: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 9: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +28: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +11: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +11: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +12: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +12: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +12: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +12: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +26: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 9: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 9: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +12: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +10: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +10: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +10: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +12: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +17: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +25: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 1: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +10: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +10: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +10: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +21: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +19: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +13: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 9: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +18: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +20: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +27: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +27: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +27: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +27: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +27: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +13: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +11: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +27: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +12: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +13: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +29: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +18: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +13: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +13: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +13: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 7: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 7: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 7: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 7: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 7: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 7: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 7: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +13: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +27: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +29: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +29: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +29: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +29: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +29: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +29: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +13: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +18: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +18: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +18: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +18: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +18: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +31: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:25:56,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:25:56,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:25:56,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:25:56,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:25:56,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:25:56,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:25:56,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 7: [2023-05-08 11:25:56,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +29: [2023-05-08 11:25:56,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 3: [2023-05-08 11:25:56,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +18: [2023-05-08 11:25:56,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:25:56,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 3: [2023-05-08 11:25:56,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 3: [2023-05-08 11:25:56,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 3: [2023-05-08 11:25:56,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 3: [2023-05-08 11:25:56,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 3: [2023-05-08 11:25:56,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 3: [2023-05-08 11:25:56,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:25:56,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 3: [2023-05-08 11:25:56,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 6: [2023-05-08 11:25:56,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 6: [2023-05-08 11:25:56,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 6: [2023-05-08 11:25:56,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 6: [2023-05-08 11:25:56,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 6: [2023-05-08 11:25:56,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 6: [2023-05-08 11:25:56,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 6: [2023-05-08 11:25:56,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 6: [2023-05-08 11:25:56,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +16: [2023-05-08 11:25:56,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +16: [2023-05-08 11:25:56,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +16: [2023-05-08 11:25:56,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +16: [2023-05-08 11:25:56,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +16: [2023-05-08 11:25:56,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +16: [2023-05-08 11:25:56,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +16: [2023-05-08 11:25:56,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +22: [2023-05-08 11:25:56,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +22: [2023-05-08 11:25:56,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +22: [2023-05-08 11:25:56,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +22: [2023-05-08 11:25:56,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +22: [2023-05-08 11:25:56,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +22: [2023-05-08 11:25:56,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +22: [2023-05-08 11:25:56,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +22: [2023-05-08 11:25:56,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +22: [2023-05-08 11:25:56,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +22: [2023-05-08 11:25:56,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +22: [2023-05-08 11:25:56,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +22: [2023-05-08 11:25:56,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +22: [2023-05-08 11:25:56,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +22: [2023-05-08 11:25:56,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +22: [2023-05-08 11:25:56,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +22: [2023-05-08 11:25:56,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +22: [2023-05-08 11:25:56,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +22: [2023-05-08 11:25:56,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +22: [2023-05-08 11:25:56,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +22: [2023-05-08 11:25:56,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +22: [2023-05-08 11:25:56,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:25:56,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:25:56,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +14: [2023-05-08 11:25:56,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +14: [2023-05-08 11:25:56,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +14: [2023-05-08 11:25:56,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +14: [2023-05-08 11:25:56,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +14: [2023-05-08 11:25:56,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +22: [2023-05-08 11:25:56,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +14: [2023-05-08 11:25:56,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +14: [2023-05-08 11:25:56,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +14: [2023-05-08 11:25:56,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +14: [2023-05-08 11:25:56,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +14: [2023-05-08 11:25:56,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +14: [2023-05-08 11:25:56,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +14: [2023-05-08 11:25:56,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +14: [2023-05-08 11:25:56,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +14: [2023-05-08 11:25:56,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +14: [2023-05-08 11:25:56,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +14: [2023-05-08 11:25:56,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +17: [2023-05-08 11:25:56,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +17: [2023-05-08 11:25:56,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +17: [2023-05-08 11:25:56,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +17: [2023-05-08 11:25:56,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +17: [2023-05-08 11:25:56,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +17: [2023-05-08 11:25:56,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +17: [2023-05-08 11:25:56,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +17: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +17: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +17: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +17: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +17: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +14: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 8: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 8: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 8: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 8: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 8: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +14: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +28: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +28: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +28: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +28: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +28: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +26: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +26: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +26: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +26: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +26: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 8: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 8: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +28: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +17: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +17: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +17: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +17: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +14: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +28: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 8: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +28: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +28: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +26: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +26: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 8: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +28: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +28: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +26: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +26: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +28: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +26: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +26: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +11: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +11: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +11: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +11: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +11: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +26: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +23: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +23: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +23: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +23: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +23: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +14: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +11: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +23: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +23: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +11: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +23: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +23: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +11: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +11: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +11: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +11: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +23: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +11: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +28: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 8: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +23: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +28: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +28: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +14: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +28: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 8: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 8: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 8: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +14: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +26: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +14: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +26: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +26: [2023-05-08 11:25:56,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +23: [2023-05-08 11:25:56,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +23: [2023-05-08 11:25:56,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +23: [2023-05-08 11:25:56,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +23: [2023-05-08 11:25:56,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +11: [2023-05-08 11:25:56,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +11: [2023-05-08 11:25:56,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +11: [2023-05-08 11:25:56,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +11: [2023-05-08 11:25:56,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +14: [2023-05-08 11:25:56,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +17: [2023-05-08 11:25:56,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +17: [2023-05-08 11:25:56,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +17: [2023-05-08 11:25:56,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +17: [2023-05-08 11:25:56,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +28: [2023-05-08 11:25:56,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 8: [2023-05-08 11:25:56,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +26: [2023-05-08 11:25:56,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +26: [2023-05-08 11:25:56,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +11: [2023-05-08 11:25:56,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +28: [2023-05-08 11:25:56,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 8: [2023-05-08 11:25:56,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +23: [2023-05-08 11:25:56,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +26: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +11: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +28: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 8: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +26: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +23: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 8: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +11: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +31: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +31: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +31: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +31: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +31: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +17: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +17: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +31: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +28: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +31: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +19: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +19: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +19: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +19: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +19: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +11: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +31: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +31: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +31: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +31: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +31: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +17: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +19: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +28: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +17: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +19: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +19: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +26: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 8: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +19: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +26: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +28: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +12: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +11: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +12: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +12: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +12: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +12: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +31: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +31: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +19: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +11: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +12: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 8: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +28: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +11: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 9: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 9: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 9: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 9: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +12: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +19: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +19: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +19: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +26: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 8: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +12: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +30: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +30: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +30: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +30: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +30: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +11: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 9: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 9: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 9: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +28: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +30: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +30: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 9: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +12: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +30: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +30: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +23: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +30: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 9: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +23: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +12: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +12: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +12: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +23: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +23: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 9: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 9: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 9: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +30: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +30: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +30: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +30: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +31: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +19: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +31: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +20: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +20: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +20: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +20: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +20: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +20: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +20: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +20: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +20: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +20: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +20: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +31: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +12: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +19: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +20: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +31: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 9: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +19: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +12: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +12: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +20: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +19: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +20: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +20: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +20: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 9: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +31: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +31: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 9: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +31: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 2: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +12: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +30: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +20: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +30: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +25: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +25: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +25: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +25: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +25: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +29: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +29: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +29: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +29: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +29: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +25: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +25: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +29: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +20: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +29: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +25: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +29: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +29: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +29: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +30: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +25: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +25: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +25: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 2: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 2: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 2: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +20: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +29: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +20: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +25: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +24: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +24: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +24: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +24: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +24: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +24: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +24: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +21: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +21: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +21: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +21: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +21: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +29: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +21: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +21: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +24: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +21: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +21: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +25: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +25: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +25: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +13: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +13: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +13: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +13: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +21: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +25: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +21: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +24: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +24: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +24: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +13: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +13: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +13: [2023-05-08 11:25:56,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +29: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +21: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +13: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +13: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +29: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +29: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +29: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +13: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +13: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +27: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +27: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +27: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +27: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +27: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +13: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +24: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +27: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +27: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 6: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 6: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 6: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 6: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 6: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +20: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +20: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 5: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +27: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +27: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +27: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +27: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +20: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +24: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +24: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +24: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +27: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 6: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 6: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +21: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +21: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 6: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +21: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +20: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +13: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +13: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +13: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +13: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 6: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +21: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 6: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 6: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +30: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 5: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 5: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 5: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +30: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +27: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +27: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +27: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +27: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +25: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 6: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +30: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 6: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 6: [2023-05-08 11:25:56,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +25: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +25: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +21: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +25: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +21: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +18: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +18: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +18: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +18: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +18: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +18: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +18: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +21: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +18: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +18: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +18: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +25: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +25: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +29: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +21: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +13: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +29: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +27: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +18: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +25: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 5: [2023-05-08 11:25:56,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +25: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 2: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +29: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +18: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +18: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +18: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 2: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +13: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +27: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 2: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +27: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +21: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 2: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +29: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +21: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +27: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +21: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +21: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 5: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +24: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +13: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +13: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +27: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +27: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +13: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 2: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +13: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +27: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +29: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 6: [2023-05-08 11:25:56,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +29: [2023-05-08 11:25:56,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +10: [2023-05-08 11:25:56,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +10: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +10: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +10: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +10: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +10: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +10: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +10: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +10: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +10: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +10: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +27: [2023-05-08 11:25:56,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +10: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +10: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +10: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +10: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +10: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +10: [2023-05-08 11:25:56,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +10: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +10: [2023-05-08 11:25:56,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +10: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +10: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +29: [2023-05-08 11:25:56,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 2: [2023-05-08 11:25:56,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +10: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +10: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +10: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +10: [2023-05-08 11:25:56,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 2: [2023-05-08 11:25:56,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:25:56,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 6: [2023-05-08 11:25:56,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +29: [2023-05-08 11:25:56,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:25:56,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +18: [2023-05-08 11:25:56,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 6: [2023-05-08 11:25:56,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +13: [2023-05-08 11:25:56,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:25:56,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +13: [2023-05-08 11:25:56,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:25:56,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +24: [2023-05-08 11:25:56,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:25:56,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:25:56,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:25:56,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:25:56,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:25:56,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:25:56,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:25:56,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:25:56,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:25:56,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:25:56,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:25:56,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +16: [2023-05-08 11:25:56,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +22: [2023-05-08 11:25:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +22: [2023-05-08 11:25:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +22: [2023-05-08 11:25:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +22: [2023-05-08 11:25:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +17: [2023-05-08 11:25:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +17: [2023-05-08 11:25:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +17: [2023-05-08 11:25:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +22: [2023-05-08 11:25:56,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:25:56,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:25:56,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +14: [2023-05-08 11:25:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +14: [2023-05-08 11:25:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +14: [2023-05-08 11:25:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +22: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +25: [2023-05-08 11:25:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 8: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 8: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 8: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +17: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +14: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +28: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +25: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +25: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 8: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +11: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +31: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +31: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +26: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +26: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +26: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +16: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +16: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +16: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +16: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +16: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +28: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +28: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 9: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +12: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +12: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +12: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +11: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +17: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +17: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +17: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +19: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +19: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +25: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 9: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +28: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +26: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +31: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +11: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 9: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +12: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +31: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +19: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +14: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +11: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +25: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 8: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 8: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +21: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +14: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +14: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +17: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 8: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +14: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +21: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +21: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 2: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +11: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +25: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +25: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 8: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +23: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +23: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 5: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 5: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 5: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +28: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +11: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +26: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +28: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +28: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +25: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +20: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +19: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +11: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +21: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +18: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +18: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +12: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +28: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 9: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 0: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 5: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +15: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +15: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +15: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +15: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +15: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +31: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 0: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 0: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +20: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +20: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +19: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +11: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +27: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +15: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +13: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 0: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +15: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +15: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +15: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +21: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +21: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 2: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 2: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +18: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +21: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 0: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 5: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 5: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +13: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +15: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 5: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +20: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +30: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +13: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 9: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +21: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +29: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 0: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 0: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 0: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 0: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +18: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +20: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 5: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +13: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +29: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +29: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +20: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +13: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 6: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 6: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +18: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +20: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +15: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +15: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +15: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +30: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +30: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +24: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +24: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +29: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +27: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +27: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +27: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +27: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 2: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +27: [2023-05-08 11:25:56,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +13: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 2: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +20: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +27: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 2: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 0: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +29: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +13: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 0: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +15: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +13: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +29: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +29: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +29: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +27: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +24: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 0: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 7: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 7: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 7: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 7: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 2: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 7: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 7: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 7: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 7: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 7: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +15: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 6: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +30: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 0: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 0: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +24: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 0: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 7: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 7: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 7: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +15: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 4: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 4: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 7: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 1: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 1: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 1: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 4: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 7: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 4: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 0: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 1: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +15: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +15: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 0: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 7: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +15: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 1: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +15: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 1: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 0: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 1: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 1: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 7: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 3: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 3: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 3: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 3: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 3: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 3: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 3: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 3: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 3: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 0: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 3: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... + 3: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 0: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 3: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 7: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 3: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... + 4: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 0: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 1: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 1: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 3: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 4: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 1: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 1: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 3: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 7: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 1: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 4: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 1: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 7: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 4: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 4: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. + 3: [2023-05-08 11:25:56,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:25:56,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:25:56,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:25:56,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 1: [2023-05-08 11:25:56,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 1: [2023-05-08 11:25:56,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 1: [2023-05-08 11:25:56,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 1: [2023-05-08 11:25:56,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 4: [2023-05-08 11:25:56,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 4: [2023-05-08 11:25:56,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 4: [2023-05-08 11:25:56,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 1: [2023-05-08 11:25:56,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 1: [2023-05-08 11:25:56,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 4: [2023-05-08 11:25:56,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 1: [2023-05-08 11:25:56,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:25:56,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 1: [2023-05-08 11:25:56,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:25:56,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 3: [2023-05-08 11:25:56,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 4: [2023-05-08 11:25:56,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 4: [2023-05-08 11:25:56,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 4: [2023-05-08 11:25:56,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:25:56,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. + 4: [2023-05-08 11:25:56,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:25:56,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:25:56,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:25:56,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:25:56,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +10: [2023-05-08 11:25:56,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +10: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +10: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +10: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +10: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +10: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +10: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +10: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +16: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +16: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +16: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +16: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +16: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +16: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +16: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt... +16: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +16: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +16: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +16: [2023-05-08 11:25:56,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt... +16: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +16: [2023-05-08 11:25:56,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +16: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +16: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt. +16: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +16: [2023-05-08 11:25:56,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +16: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +16: [2023-05-08 11:25:56,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +16: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +16: [2023-05-08 11:25:56,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +16: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +16: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt. +16: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +16: [2023-05-08 11:25:56,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +16: [2023-05-08 11:25:56,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +16: [2023-05-08 11:25:56,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:25:56,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:25:56,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:25:56,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:25:56,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:25:56,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:25:56,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:25:56,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:25:56,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:25:56,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:25:56,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +15: [2023-05-08 11:25:56,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +15: [2023-05-08 11:25:56,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +15: [2023-05-08 11:25:56,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:25:56,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:25:56,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:25:56,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +13: [2023-05-08 11:25:56,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:25:56,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:25:56,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:25:56,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:25:56,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:25:56,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:25:56,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:25:56,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:25:56,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:25:56,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:25:56,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:25:56,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:25:56,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:25:56,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:25:56,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:25:56,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:25:56,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +13: [2023-05-08 11:25:56,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +13: [2023-05-08 11:25:56,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +13: [2023-05-08 11:25:56,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +18: [2023-05-08 11:25:56,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:25:56,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:25:56,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:25:56,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:25:56,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:25:56,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:25:56,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:25:56,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:25:56,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:25:56,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:25:56,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:25:56,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:25:56,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:25:56,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:25:56,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 9: [2023-05-08 11:25:56,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:25:56,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +13: [2023-05-08 11:25:56,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +13: [2023-05-08 11:25:56,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 9: [2023-05-08 11:25:56,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:25:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +10: [2023-05-08 11:25:56,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:25:56,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:25:56,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:25:56,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:25:56,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:25:56,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:25:56,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:25:56,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:25:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +27: [2023-05-08 11:25:56,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +10: [2023-05-08 11:25:56,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:25:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +27: [2023-05-08 11:25:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +27: [2023-05-08 11:25:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +27: [2023-05-08 11:25:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 9: [2023-05-08 11:25:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:25:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 9: [2023-05-08 11:25:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 9: [2023-05-08 11:25:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 9: [2023-05-08 11:25:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 9: [2023-05-08 11:25:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:25:56,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 9: [2023-05-08 11:25:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:25:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:25:56,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:25:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:25:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:25:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:25:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:25:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +18: [2023-05-08 11:25:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +18: [2023-05-08 11:25:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +18: [2023-05-08 11:25:56,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +31: [2023-05-08 11:25:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:25:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:25:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:25:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:25:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:25:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:25:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:25:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:25:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:25:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:25:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 0: [2023-05-08 11:25:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:25:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:25:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:25:56,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:25:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:25:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:25:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:25:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:25:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:25:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:25:56,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 2: [2023-05-08 11:25:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:25:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:25:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:25:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:25:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:25:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:25:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:25:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:25:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:25:56,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:25:56,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 2: [2023-05-08 11:25:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:25:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:25:56,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:25:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:25:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:25:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:25:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:25:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:25:56,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 9: [2023-05-08 11:25:56,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:25:56,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 9: [2023-05-08 11:25:56,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 9: [2023-05-08 11:25:56,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +17: [2023-05-08 11:25:56,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +10: [2023-05-08 11:25:56,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +17: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +10: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +10: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 5: [2023-05-08 11:25:56,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 5: [2023-05-08 11:25:56,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 5: [2023-05-08 11:25:56,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 7: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 7: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 1: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 1: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 1: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 1: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 1: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 5: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +10: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 1: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 1: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 7: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 7: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +28: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +28: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +28: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 9: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 5: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 5: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 5: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +23: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +23: [2023-05-08 11:25:56,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 9: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +10: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +23: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +23: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +23: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 1: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 5: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +23: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 7: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 9: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +28: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +28: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +28: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +28: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 9: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 0: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 9: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +10: [2023-05-08 11:25:56,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +22: [2023-05-08 11:25:56,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 0: [2023-05-08 11:25:56,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 0: [2023-05-08 11:25:56,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +28: [2023-05-08 11:25:56,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:25:56,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:25:56,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 6: [2023-05-08 11:25:56,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 4: [2023-05-08 11:25:56,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 4: [2023-05-08 11:25:56,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 4: [2023-05-08 11:25:56,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +23: [2023-05-08 11:25:56,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:25:56,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 0: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 2: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 2: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 6: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 6: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 6: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +22: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 4: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 4: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 4: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 4: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +16: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 7: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 3: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 3: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 3: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 3: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 3: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 3: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 6: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:25:56,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 2: [2023-05-08 11:25:56,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:25:56,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:25:56,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 6: [2023-05-08 11:25:56,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 3: [2023-05-08 11:25:56,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:25:56,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +24: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +24: [2023-05-08 11:25:56,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 6: [2023-05-08 11:25:56,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:25:56,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:25:56,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:25:56,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +19: [2023-05-08 11:25:56,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +22: [2023-05-08 11:25:56,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 4: [2023-05-08 11:25:56,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:25:56,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:25:56,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 2: [2023-05-08 11:25:56,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +19: [2023-05-08 11:25:56,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 7: [2023-05-08 11:25:56,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:25:56,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 7: [2023-05-08 11:25:56,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +19: [2023-05-08 11:25:56,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 7: [2023-05-08 11:25:56,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 5: [2023-05-08 11:25:56,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:25:56,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +24: [2023-05-08 11:25:56,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:25:56,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:25:56,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +24: [2023-05-08 11:25:56,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +24: [2023-05-08 11:25:56,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +24: [2023-05-08 11:25:56,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:25:56,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:25:56,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 5: [2023-05-08 11:25:56,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 5: [2023-05-08 11:25:56,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +22: [2023-05-08 11:25:56,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:25:56,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 4: [2023-05-08 11:25:56,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 4: [2023-05-08 11:25:56,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +28: [2023-05-08 11:25:56,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:25:56,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +28: [2023-05-08 11:25:56,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 4: [2023-05-08 11:25:56,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 2: [2023-05-08 11:25:56,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +24: [2023-05-08 11:25:56,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +28: [2023-05-08 11:25:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 3: [2023-05-08 11:25:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +28: [2023-05-08 11:25:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 1: [2023-05-08 11:25:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +28: [2023-05-08 11:25:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +29: [2023-05-08 11:25:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 1: [2023-05-08 11:25:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:25:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +29: [2023-05-08 11:25:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +29: [2023-05-08 11:25:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 1: [2023-05-08 11:25:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +28: [2023-05-08 11:25:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 7: [2023-05-08 11:25:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 1: [2023-05-08 11:25:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +28: [2023-05-08 11:25:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 1: [2023-05-08 11:25:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 1: [2023-05-08 11:25:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 5: [2023-05-08 11:25:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 1: [2023-05-08 11:25:56,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +29: [2023-05-08 11:25:56,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +28: [2023-05-08 11:25:56,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +29: [2023-05-08 11:25:56,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +29: [2023-05-08 11:25:56,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +29: [2023-05-08 11:25:56,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 1: [2023-05-08 11:25:56,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 6: [2023-05-08 11:25:56,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +29: [2023-05-08 11:25:56,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 5: [2023-05-08 11:25:56,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 5: [2023-05-08 11:25:56,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 4: [2023-05-08 11:25:56,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 5: [2023-05-08 11:25:56,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +24: [2023-05-08 11:25:56,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 5: [2023-05-08 11:25:56,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 4: [2023-05-08 11:25:56,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:25:56,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 4: [2023-05-08 11:25:56,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 6: [2023-05-08 11:25:56,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 4: [2023-05-08 11:25:56,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 6: [2023-05-08 11:25:56,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:25:56,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:25:56,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +23: [2023-05-08 11:25:56,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 4: [2023-05-08 11:25:56,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:25:56,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +23: [2023-05-08 11:25:56,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:25:56,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 6: [2023-05-08 11:25:56,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +15: [2023-05-08 11:25:56,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +23: [2023-05-08 11:25:56,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 6: [2023-05-08 11:25:56,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +23: [2023-05-08 11:25:56,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 6: [2023-05-08 11:25:56,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:25:56,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 6: [2023-05-08 11:25:56,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +16: [2023-05-08 11:25:56,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +16: [2023-05-08 11:25:56,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:25:56,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:25:56,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 3: [2023-05-08 11:25:56,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 3: [2023-05-08 11:25:56,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +15: [2023-05-08 11:25:56,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:25:56,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +23: [2023-05-08 11:25:56,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +11: [2023-05-08 11:25:56,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +11: [2023-05-08 11:25:56,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +11: [2023-05-08 11:25:56,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +11: [2023-05-08 11:25:56,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +11: [2023-05-08 11:25:56,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +11: [2023-05-08 11:25:56,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 3: [2023-05-08 11:25:56,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +16: [2023-05-08 11:25:56,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +11: [2023-05-08 11:25:56,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +11: [2023-05-08 11:25:56,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:25:56,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +16: [2023-05-08 11:25:56,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +16: [2023-05-08 11:25:56,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +16: [2023-05-08 11:25:56,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:25:56,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:25:56,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:25:56,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:25:56,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +24: [2023-05-08 11:25:56,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +11: [2023-05-08 11:25:56,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +11: [2023-05-08 11:25:56,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +11: [2023-05-08 11:25:56,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +24: [2023-05-08 11:25:56,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +11: [2023-05-08 11:25:56,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +11: [2023-05-08 11:25:56,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +11: [2023-05-08 11:25:56,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +11: [2023-05-08 11:25:56,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +11: [2023-05-08 11:25:56,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +15: [2023-05-08 11:25:56,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +20: [2023-05-08 11:25:56,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:25:56,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:25:56,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +14: [2023-05-08 11:25:56,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +14: [2023-05-08 11:25:56,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +14: [2023-05-08 11:25:56,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:25:56,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +14: [2023-05-08 11:25:56,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +14: [2023-05-08 11:25:56,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:25:56,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:25:56,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:25:56,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:25:56,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:25:56,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +15: [2023-05-08 11:25:56,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:25:56,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:25:56,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +14: [2023-05-08 11:25:56,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:25:56,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:25:56,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +14: [2023-05-08 11:25:56,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:25:56,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:25:56,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:25:56,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:25:56,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:25:56,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:25:56,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:25:56,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:25:56,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:25:56,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:25:56,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:25:56,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:25:56,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:25:56,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:25:56,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:25:56,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +14: [2023-05-08 11:25:56,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +20: [2023-05-08 11:25:56,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +27: [2023-05-08 11:25:56,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +14: [2023-05-08 11:25:56,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +14: [2023-05-08 11:25:56,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +14: [2023-05-08 11:25:56,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +12: [2023-05-08 11:25:56,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +13: [2023-05-08 11:25:56,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:25:56,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +14: [2023-05-08 11:25:56,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +17: [2023-05-08 11:25:56,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +14: [2023-05-08 11:25:56,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +14: [2023-05-08 11:25:56,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:25:56,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +18: [2023-05-08 11:25:56,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:25:56,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +21: [2023-05-08 11:25:56,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +20: [2023-05-08 11:25:56,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +14: [2023-05-08 11:25:56,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +21: [2023-05-08 11:25:56,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +20: [2023-05-08 11:25:56,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +12: [2023-05-08 11:25:56,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:25:56,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +12: [2023-05-08 11:25:56,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +20: [2023-05-08 11:25:56,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:25:56,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +12: [2023-05-08 11:25:56,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +20: [2023-05-08 11:25:56,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +20: [2023-05-08 11:25:56,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +20: [2023-05-08 11:25:56,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +13: [2023-05-08 11:25:56,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:25:56,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:25:56,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:25:56,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:25:56,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:25:56,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:25:56,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 9: [2023-05-08 11:25:56,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:25:56,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:25:56,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:25:56,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:25:56,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:25:56,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 8: [2023-05-08 11:25:56,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:25:56,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:25:56,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:25:56,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:25:56,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:25:56,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 8: [2023-05-08 11:25:56,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:25:56,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:25:56,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +21: [2023-05-08 11:25:56,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 8: [2023-05-08 11:25:56,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:25:56,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +18: [2023-05-08 11:25:56,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:25:56,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:25:56,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +23: [2023-05-08 11:25:56,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:25:56,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +19: [2023-05-08 11:25:56,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 4: [2023-05-08 11:25:56,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 5: [2023-05-08 11:25:56,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 3: [2023-05-08 11:25:56,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:25:56,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:25:56,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:25:56,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 9: [2023-05-08 11:25:56,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 9: [2023-05-08 11:25:56,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 9: [2023-05-08 11:25:56,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 7: [2023-05-08 11:25:56,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 7: [2023-05-08 11:25:56,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:25:56,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 3: [2023-05-08 11:25:56,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 1: [2023-05-08 11:25:56,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 1: [2023-05-08 11:25:56,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:25:56,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +31: [2023-05-08 11:25:56,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:25:56,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:25:56,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 4: [2023-05-08 11:25:56,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:25:56,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:25:56,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:25:56,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 0: [2023-05-08 11:25:56,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:25:56,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:25:56,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:25:56,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:25:56,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:25:56,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +24: [2023-05-08 11:25:56,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +28: [2023-05-08 11:25:56,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +28: [2023-05-08 11:25:56,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:25:56,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 8: [2023-05-08 11:25:56,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 2: [2023-05-08 11:25:56,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:25:56,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:25:56,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +13: [2023-05-08 11:25:56,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +22: [2023-05-08 11:25:56,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 6: [2023-05-08 11:25:56,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:25:56,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 8: [2023-05-08 11:25:56,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 8: [2023-05-08 11:25:56,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +10: [2023-05-08 11:25:56,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 1: [2023-05-08 11:25:56,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +28: [2023-05-08 11:25:56,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 1: [2023-05-08 11:25:56,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 6: [2023-05-08 11:25:56,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 5: [2023-05-08 11:25:56,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:25:56,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:25:56,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +27: [2023-05-08 11:25:56,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:25:56,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 4: [2023-05-08 11:25:56,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:25:56,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 7: [2023-05-08 11:25:56,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 5: [2023-05-08 11:25:56,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 5: [2023-05-08 11:25:56,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:25:56,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 8: [2023-05-08 11:25:56,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 2: [2023-05-08 11:25:56,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:25:56,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +13: [2023-05-08 11:25:56,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:25:56,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:25:56,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:25:56,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +18: [2023-05-08 11:25:56,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +18: [2023-05-08 11:25:56,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:25:56,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:25:56,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:25:56,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +19: [2023-05-08 11:25:56,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:25:56,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +18: [2023-05-08 11:25:56,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +18: [2023-05-08 11:25:56,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:25:56,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 6: [2023-05-08 11:25:56,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 3: [2023-05-08 11:25:56,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:25:56,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:25:56,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +25: [2023-05-08 11:25:56,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:25:56,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:25:56,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:25:56,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:25:56,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:25:56,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:25:56,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +23: [2023-05-08 11:25:56,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +17: [2023-05-08 11:25:56,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +25: [2023-05-08 11:25:56,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 3: [2023-05-08 11:25:56,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +30: [2023-05-08 11:25:56,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +30: [2023-05-08 11:25:56,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +30: [2023-05-08 11:25:56,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +30: [2023-05-08 11:25:56,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +30: [2023-05-08 11:25:56,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +30: [2023-05-08 11:25:56,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +30: [2023-05-08 11:25:56,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +30: [2023-05-08 11:25:56,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 4: [2023-05-08 11:25:56,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 9: [2023-05-08 11:25:56,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:25:56,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +26: [2023-05-08 11:25:56,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +26: [2023-05-08 11:25:56,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +26: [2023-05-08 11:25:56,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 3: [2023-05-08 11:25:56,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:25:56,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +26: [2023-05-08 11:25:56,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +26: [2023-05-08 11:25:56,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:25:56,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +19: [2023-05-08 11:25:56,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:25:56,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +19: [2023-05-08 11:25:56,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:25:56,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:25:56,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:25:56,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:25:56,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +25: [2023-05-08 11:25:56,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +25: [2023-05-08 11:25:56,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +25: [2023-05-08 11:25:56,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +25: [2023-05-08 11:25:56,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +25: [2023-05-08 11:25:56,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +25: [2023-05-08 11:25:56,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +25: [2023-05-08 11:25:56,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +16: [2023-05-08 11:25:56,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:25:56,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 9: [2023-05-08 11:25:56,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:25:56,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 3: [2023-05-08 11:25:56,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 9: [2023-05-08 11:25:56,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:25:56,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 9: [2023-05-08 11:25:56,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +31: [2023-05-08 11:25:56,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +31: [2023-05-08 11:25:56,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:25:56,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +26: [2023-05-08 11:25:56,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +26: [2023-05-08 11:25:56,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 0: [2023-05-08 11:25:56,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:25:56,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... + 0: [2023-05-08 11:25:56,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:25:56,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:25:56,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:25:56,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:25:56,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:25:56,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +22: [2023-05-08 11:25:56,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:25:56,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +30: [2023-05-08 11:25:56,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +24: [2023-05-08 11:25:56,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 0: [2023-05-08 11:25:56,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:25:56,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 0: [2023-05-08 11:25:56,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +22: [2023-05-08 11:25:56,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:25:56,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:25:56,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:25:56,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +26: [2023-05-08 11:25:56,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 6: [2023-05-08 11:25:56,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:25:56,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... + 5: [2023-05-08 11:25:56,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:25:56,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +31: [2023-05-08 11:25:56,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:25:56,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +28: [2023-05-08 11:25:56,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:25:56,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:25:56,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:25:56,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +26: [2023-05-08 11:25:56,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt... +23: [2023-05-08 11:25:56,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +11: [2023-05-08 11:25:56,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +11: [2023-05-08 11:25:56,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +22: [2023-05-08 11:25:56,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:25:56,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt... +12: [2023-05-08 11:25:56,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +11: [2023-05-08 11:25:56,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +29: [2023-05-08 11:25:56,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +31: [2023-05-08 11:25:56,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +11: [2023-05-08 11:25:56,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +28: [2023-05-08 11:25:56,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:25:56,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:25:56,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:25:56,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 3: [2023-05-08 11:25:56,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:25:56,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +22: [2023-05-08 11:25:56,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:25:56,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +24: [2023-05-08 11:25:56,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:25:56,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 3: [2023-05-08 11:25:56,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:25:56,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:25:56,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:25:56,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:25:56,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:25:56,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:25:56,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 1: [2023-05-08 11:25:56,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:25:56,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:25:56,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:25:56,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:25:56,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 2: [2023-05-08 11:25:56,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 7: [2023-05-08 11:25:56,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:25:56,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 2: [2023-05-08 11:25:56,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +12: [2023-05-08 11:25:56,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:25:56,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:25:56,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +14: [2023-05-08 11:25:56,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:25:56,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:25:56,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:25:56,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:25:56,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +12: [2023-05-08 11:25:56,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:25:56,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +10: [2023-05-08 11:25:56,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +20: [2023-05-08 11:25:56,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:25:56,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:25:56,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:25:56,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +16: [2023-05-08 11:25:57,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:25:57,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:25:57,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +11: [2023-05-08 11:25:57,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +11: [2023-05-08 11:25:57,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +11: [2023-05-08 11:25:57,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:25:57,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:25:57,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:25:57,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:25:57,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:25:57,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +12: [2023-05-08 11:25:57,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:25:57,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +12: [2023-05-08 11:25:57,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:25:57,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +12: [2023-05-08 11:25:57,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +11: [2023-05-08 11:25:57,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:25:57,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:25:57,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:25:57,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +25: [2023-05-08 11:25:57,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +26: [2023-05-08 11:25:57,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +26: [2023-05-08 11:25:57,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:25:57,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:25:57,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:25:57,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +26: [2023-05-08 11:25:57,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +26: [2023-05-08 11:25:57,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +30: [2023-05-08 11:25:57,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +20: [2023-05-08 11:25:57,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:25:57,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +25: [2023-05-08 11:25:57,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +20: [2023-05-08 11:25:57,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +20: [2023-05-08 11:25:57,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +20: [2023-05-08 11:25:57,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:25:57,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:25:57,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:25:57,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. +21: [2023-05-08 11:25:57,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:25:57,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +25: [2023-05-08 11:25:57,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +25: [2023-05-08 11:25:57,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +25: [2023-05-08 11:25:57,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:25:57,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:25:57,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:25:57,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:25:57,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:25:57,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:25:57,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:25:57,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:25:57,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +27: [2023-05-08 11:25:57,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +27: [2023-05-08 11:25:57,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +27: [2023-05-08 11:25:57,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +27: [2023-05-08 11:25:57,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:25:57,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:25:57,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:25:57,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:25:57,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +27: [2023-05-08 11:25:57,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +27: [2023-05-08 11:25:57,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +27: [2023-05-08 11:25:57,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +27: [2023-05-08 11:25:57,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +19: [2023-05-08 11:25:57,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +19: [2023-05-08 11:25:57,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +19: [2023-05-08 11:25:57,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +19: [2023-05-08 11:25:57,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +27: [2023-05-08 11:25:57,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:25:57,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:25:57,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +19: [2023-05-08 11:25:57,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +19: [2023-05-08 11:25:57,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +19: [2023-05-08 11:25:57,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +19: [2023-05-08 11:25:57,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 5: [2023-05-08 11:25:57,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 5: [2023-05-08 11:25:57,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 5: [2023-05-08 11:25:57,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 3: [2023-05-08 11:25:57,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +29: [2023-05-08 11:25:57,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 9: [2023-05-08 11:25:57,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 9: [2023-05-08 11:25:57,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 9: [2023-05-08 11:25:57,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +29: [2023-05-08 11:25:57,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +29: [2023-05-08 11:25:57,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 3: [2023-05-08 11:25:57,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +17: [2023-05-08 11:25:57,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +17: [2023-05-08 11:25:57,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +17: [2023-05-08 11:25:57,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 3: [2023-05-08 11:25:57,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 9: [2023-05-08 11:25:57,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +17: [2023-05-08 11:25:57,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 3: [2023-05-08 11:25:57,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +29: [2023-05-08 11:25:57,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +29: [2023-05-08 11:25:57,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +29: [2023-05-08 11:25:57,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:25:57,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:25:57,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +29: [2023-05-08 11:25:57,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:25:57,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:25:57,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:25:57,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:25:57,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:25:57,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:25:57,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:25:57,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:25:57,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:25:57,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:25:57,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:25:57,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:25:57,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +18: [2023-05-08 11:25:57,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +18: [2023-05-08 11:25:57,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +18: [2023-05-08 11:25:57,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +18: [2023-05-08 11:25:57,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 3: [2023-05-08 11:25:57,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:25:57,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:25:57,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:25:57,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:25:57,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:25:57,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:25:57,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:25:57,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 4: [2023-05-08 11:25:57,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 4: [2023-05-08 11:25:57,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 4: [2023-05-08 11:25:57,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +22: [2023-05-08 11:25:57,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +22: [2023-05-08 11:25:57,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +22: [2023-05-08 11:25:57,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +22: [2023-05-08 11:25:57,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 6: [2023-05-08 11:25:57,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 6: [2023-05-08 11:25:57,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 6: [2023-05-08 11:25:57,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 6: [2023-05-08 11:25:57,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +16: [2023-05-08 11:25:57,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +16: [2023-05-08 11:25:57,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +16: [2023-05-08 11:25:57,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +18: [2023-05-08 11:25:57,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:25:57,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +13: [2023-05-08 11:25:57,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:25:57,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +16: [2023-05-08 11:25:57,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 0: [2023-05-08 11:25:57,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 0: [2023-05-08 11:25:57,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +18: [2023-05-08 11:25:57,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:25:57,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:25:57,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +13: [2023-05-08 11:25:57,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:25:57,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +18: [2023-05-08 11:25:57,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +25: [2023-05-08 11:25:57,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +25: [2023-05-08 11:25:57,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +25: [2023-05-08 11:25:57,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +25: [2023-05-08 11:25:57,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +13: [2023-05-08 11:25:57,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +20: [2023-05-08 11:25:57,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +20: [2023-05-08 11:25:57,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +20: [2023-05-08 11:25:57,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +13: [2023-05-08 11:25:57,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +20: [2023-05-08 11:25:57,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +13: [2023-05-08 11:25:57,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:25:57,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:25:57,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +14: [2023-05-08 11:25:57,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +14: [2023-05-08 11:25:57,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +14: [2023-05-08 11:25:57,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +30: [2023-05-08 11:25:57,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +17: [2023-05-08 11:25:57,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:25:57,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:25:57,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +30: [2023-05-08 11:25:57,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 7: [2023-05-08 11:25:57,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 7: [2023-05-08 11:25:57,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 7: [2023-05-08 11:25:57,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 7: [2023-05-08 11:25:57,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +30: [2023-05-08 11:25:57,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +23: [2023-05-08 11:25:57,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +23: [2023-05-08 11:25:57,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +23: [2023-05-08 11:25:57,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +23: [2023-05-08 11:25:57,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +28: [2023-05-08 11:25:57,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +28: [2023-05-08 11:25:57,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +28: [2023-05-08 11:25:57,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:25:57,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:25:57,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +28: [2023-05-08 11:25:57,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +28: [2023-05-08 11:25:57,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +28: [2023-05-08 11:25:57,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +28: [2023-05-08 11:25:57,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +28: [2023-05-08 11:25:57,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +29: [2023-05-08 11:25:57,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +29: [2023-05-08 11:25:57,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +29: [2023-05-08 11:25:57,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:25:57,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:25:57,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:25:57,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +22: [2023-05-08 11:25:57,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +22: [2023-05-08 11:25:57,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:25:57,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +22: [2023-05-08 11:25:57,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +22: [2023-05-08 11:25:57,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:25:57,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:25:57,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +25: [2023-05-08 11:25:57,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +25: [2023-05-08 11:25:57,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +20: [2023-05-08 11:25:57,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +20: [2023-05-08 11:25:57,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:25:57,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +25: [2023-05-08 11:25:57,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +25: [2023-05-08 11:25:57,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:25:57,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:25:57,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:25:57,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:25:57,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:25:57,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:25:57,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +17: [2023-05-08 11:25:57,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +24: [2023-05-08 11:25:57,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:25:57,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:25:57,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:25:57,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:25:57,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +24: [2023-05-08 11:25:57,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +24: [2023-05-08 11:25:57,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:25:57,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:25:57,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +24: [2023-05-08 11:25:57,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +17: [2023-05-08 11:25:57,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +24: [2023-05-08 11:25:57,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 0: [2023-05-08 11:25:57,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:25:57,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:25:57,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:25:57,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:25:57,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:25:57,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:25:57,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:25:57,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:25:57,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +24: [2023-05-08 11:25:57,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +17: [2023-05-08 11:25:57,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:25:57,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +14: [2023-05-08 11:25:57,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:25:57,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:25:57,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:25:57,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:25:57,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +17: [2023-05-08 11:25:57,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +23: [2023-05-08 11:25:57,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +17: [2023-05-08 11:25:57,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:25:57,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +30: [2023-05-08 11:25:57,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:25:57,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +13: [2023-05-08 11:25:57,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +23: [2023-05-08 11:25:57,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:25:57,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +23: [2023-05-08 11:25:57,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +23: [2023-05-08 11:25:57,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:25:57,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:25:57,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:25:57,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:25:57,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +23: [2023-05-08 11:25:57,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:25:57,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:25:57,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:25:57,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:25:57,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:25:57,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +13: [2023-05-08 11:25:57,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:25:57,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:25:57,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +29: [2023-05-08 11:25:57,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +23: [2023-05-08 11:25:57,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:25:57,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:25:57,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +29: [2023-05-08 11:25:57,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:25:57,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +29: [2023-05-08 11:25:57,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +29: [2023-05-08 11:25:57,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 7: [2023-05-08 11:25:57,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +29: [2023-05-08 11:25:57,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 7: [2023-05-08 11:25:57,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:25:57,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:25:57,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:25:57,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +10: [2023-05-08 11:25:57,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +10: [2023-05-08 11:25:57,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:25:57,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +28: [2023-05-08 11:25:57,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:25:57,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 2: [2023-05-08 11:25:57,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 2: [2023-05-08 11:25:57,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 2: [2023-05-08 11:25:57,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 2: [2023-05-08 11:25:57,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +10: [2023-05-08 11:25:57,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +10: [2023-05-08 11:25:57,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +21: [2023-05-08 11:25:57,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +21: [2023-05-08 11:25:57,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +10: [2023-05-08 11:25:57,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:25:57,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:25:57,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +10: [2023-05-08 11:25:57,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:25:57,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +21: [2023-05-08 11:25:57,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +31: [2023-05-08 11:25:57,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +31: [2023-05-08 11:25:57,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +31: [2023-05-08 11:25:57,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +10: [2023-05-08 11:25:57,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:25:57,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +31: [2023-05-08 11:25:57,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +24: [2023-05-08 11:25:57,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:25:57,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:25:57,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:25:57,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:25:57,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +13: [2023-05-08 11:25:57,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:25:57,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:25:57,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +11: [2023-05-08 11:25:57,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +11: [2023-05-08 11:25:57,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +11: [2023-05-08 11:25:57,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +11: [2023-05-08 11:25:57,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +26: [2023-05-08 11:25:57,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +26: [2023-05-08 11:25:57,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +26: [2023-05-08 11:25:57,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +13: [2023-05-08 11:25:57,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +13: [2023-05-08 11:25:57,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +13: [2023-05-08 11:25:57,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +26: [2023-05-08 11:25:57,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +13: [2023-05-08 11:25:57,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +24: [2023-05-08 11:25:57,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:25:57,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:25:57,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +24: [2023-05-08 11:25:57,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:25:57,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +28: [2023-05-08 11:25:57,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +23: [2023-05-08 11:25:57,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:25:57,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 2: [2023-05-08 11:25:57,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +12: [2023-05-08 11:25:57,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +12: [2023-05-08 11:25:57,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +23: [2023-05-08 11:25:57,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +31: [2023-05-08 11:25:57,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +12: [2023-05-08 11:25:57,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 0: [2023-05-08 11:25:57,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +10: [2023-05-08 11:25:57,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +31: [2023-05-08 11:25:57,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:25:57,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 0: [2023-05-08 11:25:57,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:25:57,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:25:57,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:25:57,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 2: [2023-05-08 11:25:57,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:25:57,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:25:57,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 0: [2023-05-08 11:25:57,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 0: [2023-05-08 11:25:57,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +31: [2023-05-08 11:25:57,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +31: [2023-05-08 11:25:57,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 1: [2023-05-08 11:25:57,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 1: [2023-05-08 11:25:57,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +13: [2023-05-08 11:25:57,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +21: [2023-05-08 11:25:57,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:25:57,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +28: [2023-05-08 11:25:57,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +10: [2023-05-08 11:25:57,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 8: [2023-05-08 11:25:57,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 8: [2023-05-08 11:25:57,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. + 8: [2023-05-08 11:25:57,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +11: [2023-05-08 11:25:57,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 2: [2023-05-08 11:25:57,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 2: [2023-05-08 11:25:57,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +15: [2023-05-08 11:25:57,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +15: [2023-05-08 11:25:57,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +15: [2023-05-08 11:25:57,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +15: [2023-05-08 11:25:57,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +23: [2023-05-08 11:25:57,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:25:57,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:25:57,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:25:57,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:25:57,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +11: [2023-05-08 11:25:57,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:25:57,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:25:57,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +23: [2023-05-08 11:25:57,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:25:57,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:25:57,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:25:57,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:25:57,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +27: [2023-05-08 11:25:57,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +27: [2023-05-08 11:25:57,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +27: [2023-05-08 11:25:57,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt. +23: [2023-05-08 11:25:57,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +24: [2023-05-08 11:25:57,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +24: [2023-05-08 11:25:57,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +26: [2023-05-08 11:25:57,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:25:57,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +12: [2023-05-08 11:25:57,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +24: [2023-05-08 11:25:57,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +24: [2023-05-08 11:25:57,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +10: [2023-05-08 11:25:57,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:25:57,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:25:57,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +23: [2023-05-08 11:25:57,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +23: [2023-05-08 11:25:57,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:25:57,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:25:57,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:25:57,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +10: [2023-05-08 11:25:57,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +26: [2023-05-08 11:25:57,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:25:57,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +24: [2023-05-08 11:25:57,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:25:57,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:25:57,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +23: [2023-05-08 11:25:57,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +23: [2023-05-08 11:25:57,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +12: [2023-05-08 11:25:57,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:25:57,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +14: [2023-05-08 11:25:57,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +14: [2023-05-08 11:25:57,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +14: [2023-05-08 11:25:57,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +14: [2023-05-08 11:25:57,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +14: [2023-05-08 11:25:57,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +14: [2023-05-08 11:25:57,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +14: [2023-05-08 11:25:57,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +28: [2023-05-08 11:25:57,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +24: [2023-05-08 11:25:57,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 0: [2023-05-08 11:25:57,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:25:57,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:25:57,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:25:57,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +14: [2023-05-08 11:25:57,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:25:57,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:25:57,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +14: [2023-05-08 11:25:57,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +14: [2023-05-08 11:25:57,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +14: [2023-05-08 11:25:57,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +14: [2023-05-08 11:25:57,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:25:57,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:25:57,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +28: [2023-05-08 11:25:57,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +28: [2023-05-08 11:25:57,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:25:57,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:25:57,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:25:57,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:25:57,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:25:57,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:25:57,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:25:57,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:25:57,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:25:57,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:25:57,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:25:57,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:25:57,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +23: [2023-05-08 11:25:57,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +28: [2023-05-08 11:25:57,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +28: [2023-05-08 11:25:57,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +28: [2023-05-08 11:25:57,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +28: [2023-05-08 11:25:57,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +23: [2023-05-08 11:25:57,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 3: [2023-05-08 11:25:57,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 3: [2023-05-08 11:25:57,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 3: [2023-05-08 11:25:57,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 3: [2023-05-08 11:25:57,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:25:57,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:25:57,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:25:57,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:25:57,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:25:57,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:25:57,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:25:57,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +10: [2023-05-08 11:25:57,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +13: [2023-05-08 11:25:57,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +24: [2023-05-08 11:25:57,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +10: [2023-05-08 11:25:57,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +24: [2023-05-08 11:25:57,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +24: [2023-05-08 11:25:57,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 9: [2023-05-08 11:25:57,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 9: [2023-05-08 11:25:57,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 0: [2023-05-08 11:25:57,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +10: [2023-05-08 11:25:57,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +10: [2023-05-08 11:25:57,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 9: [2023-05-08 11:25:57,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 9: [2023-05-08 11:25:57,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 9: [2023-05-08 11:25:57,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 9: [2023-05-08 11:25:57,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +10: [2023-05-08 11:25:57,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +13: [2023-05-08 11:25:57,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +10: [2023-05-08 11:25:57,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:25:57,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +10: [2023-05-08 11:25:57,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 9: [2023-05-08 11:25:57,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 9: [2023-05-08 11:25:57,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +13: [2023-05-08 11:25:57,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +13: [2023-05-08 11:25:57,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +13: [2023-05-08 11:25:57,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 9: [2023-05-08 11:25:57,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 9: [2023-05-08 11:25:57,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 9: [2023-05-08 11:25:57,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +13: [2023-05-08 11:25:57,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +24: [2023-05-08 11:25:57,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +24: [2023-05-08 11:25:57,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +24: [2023-05-08 11:25:57,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:25:57,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:25:57,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:25:57,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:25:57,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +14: [2023-05-08 11:25:57,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +14: [2023-05-08 11:25:57,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +24: [2023-05-08 11:25:57,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +24: [2023-05-08 11:25:57,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +24: [2023-05-08 11:25:57,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +12: [2023-05-08 11:25:57,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +14: [2023-05-08 11:25:57,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +10: [2023-05-08 11:25:57,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +10: [2023-05-08 11:25:57,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:25:57,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:25:57,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:25:57,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:25:57,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +12: [2023-05-08 11:25:57,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +10: [2023-05-08 11:25:57,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +10: [2023-05-08 11:25:57,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +12: [2023-05-08 11:25:57,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +14: [2023-05-08 11:25:57,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:25:57,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:25:57,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:25:57,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +12: [2023-05-08 11:25:57,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +12: [2023-05-08 11:25:57,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +12: [2023-05-08 11:25:57,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +12: [2023-05-08 11:25:57,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 3: [2023-05-08 11:25:57,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:25:57,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +14: [2023-05-08 11:25:57,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +14: [2023-05-08 11:25:57,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +14: [2023-05-08 11:25:57,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:25:57,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +27: [2023-05-08 11:25:57,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +27: [2023-05-08 11:25:57,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:25:57,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +14: [2023-05-08 11:25:57,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:25:57,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:25:57,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:25:57,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:25:57,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:25:57,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:25:57,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +27: [2023-05-08 11:25:57,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +27: [2023-05-08 11:25:57,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +27: [2023-05-08 11:25:57,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 5: [2023-05-08 11:25:57,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:25:57,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:25:57,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:25:57,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:25:57,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +18: [2023-05-08 11:25:57,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:25:57,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:25:57,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:25:57,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:25:57,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +18: [2023-05-08 11:25:57,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:25:57,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 5: [2023-05-08 11:25:57,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +18: [2023-05-08 11:25:57,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:25:57,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +15: [2023-05-08 11:25:57,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +15: [2023-05-08 11:25:57,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +15: [2023-05-08 11:25:57,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +15: [2023-05-08 11:25:57,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +27: [2023-05-08 11:25:57,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 5: [2023-05-08 11:25:57,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +15: [2023-05-08 11:25:57,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +15: [2023-05-08 11:25:57,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 5: [2023-05-08 11:25:57,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 6: [2023-05-08 11:25:57,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 5: [2023-05-08 11:25:57,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +15: [2023-05-08 11:25:57,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +18: [2023-05-08 11:25:57,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +18: [2023-05-08 11:25:57,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +18: [2023-05-08 11:25:57,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:25:57,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 5: [2023-05-08 11:25:57,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +18: [2023-05-08 11:25:57,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:25:57,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +18: [2023-05-08 11:25:57,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +18: [2023-05-08 11:25:57,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +18: [2023-05-08 11:25:57,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +18: [2023-05-08 11:25:57,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 7: [2023-05-08 11:25:57,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 6: [2023-05-08 11:25:57,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 7: [2023-05-08 11:25:57,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 7: [2023-05-08 11:25:57,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 7: [2023-05-08 11:25:57,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:25:57,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 9: [2023-05-08 11:25:57,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 9: [2023-05-08 11:25:57,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 9: [2023-05-08 11:25:57,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:25:57,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +31: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +31: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +31: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +31: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +31: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +31: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +31: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 3: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +31: [2023-05-08 11:25:57,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 7: [2023-05-08 11:25:57,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 7: [2023-05-08 11:25:57,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 7: [2023-05-08 11:25:57,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +19: [2023-05-08 11:25:57,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +19: [2023-05-08 11:25:57,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:25:57,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:25:57,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:25:57,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:25:57,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:25:57,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +31: [2023-05-08 11:25:57,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +31: [2023-05-08 11:25:57,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:25:57,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:25:57,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:25:57,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:25:57,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:25:57,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:25:57,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:25:57,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:25:57,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 7: [2023-05-08 11:25:57,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +19: [2023-05-08 11:25:57,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +31: [2023-05-08 11:25:57,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +31: [2023-05-08 11:25:57,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +31: [2023-05-08 11:25:57,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +30: [2023-05-08 11:25:57,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:25:57,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:25:57,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:25:57,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:25:57,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:25:57,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:25:57,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:25:57,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:25:57,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:25:57,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +31: [2023-05-08 11:25:57,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +11: [2023-05-08 11:25:57,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:25:57,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +31: [2023-05-08 11:25:57,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +30: [2023-05-08 11:25:57,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +30: [2023-05-08 11:25:57,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +30: [2023-05-08 11:25:57,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +30: [2023-05-08 11:25:57,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +31: [2023-05-08 11:25:57,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +30: [2023-05-08 11:25:57,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +11: [2023-05-08 11:25:57,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:25:57,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:25:57,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:25:57,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:25:57,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 1: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +25: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +25: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +25: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +25: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +25: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +25: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +25: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 1: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +19: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +25: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 2: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 1: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +11: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +19: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +21: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 2: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 2: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 2: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +22: [2023-05-08 11:25:57,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 2: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 2: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 2: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +22: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +22: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +22: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +22: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 2: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +22: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +22: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +22: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +19: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +11: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +12: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +11: [2023-05-08 11:25:57,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:25:57,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +19: [2023-05-08 11:25:57,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +19: [2023-05-08 11:25:57,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +19: [2023-05-08 11:25:57,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +21: [2023-05-08 11:25:57,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +21: [2023-05-08 11:25:57,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +16: [2023-05-08 11:25:57,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +22: [2023-05-08 11:25:57,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +21: [2023-05-08 11:25:57,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +16: [2023-05-08 11:25:57,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +11: [2023-05-08 11:25:57,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +11: [2023-05-08 11:25:57,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:25:57,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +21: [2023-05-08 11:25:57,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +16: [2023-05-08 11:25:57,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +22: [2023-05-08 11:25:57,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +22: [2023-05-08 11:25:57,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +16: [2023-05-08 11:25:57,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 2: [2023-05-08 11:25:57,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +22: [2023-05-08 11:25:57,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +25: [2023-05-08 11:25:57,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +11: [2023-05-08 11:25:57,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +22: [2023-05-08 11:25:57,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +11: [2023-05-08 11:25:57,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +22: [2023-05-08 11:25:57,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +25: [2023-05-08 11:25:57,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +25: [2023-05-08 11:25:57,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +22: [2023-05-08 11:25:57,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 8: [2023-05-08 11:25:57,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +11: [2023-05-08 11:25:57,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 8: [2023-05-08 11:25:57,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +22: [2023-05-08 11:25:57,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +11: [2023-05-08 11:25:57,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 2: [2023-05-08 11:25:57,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... + 2: [2023-05-08 11:25:57,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +25: [2023-05-08 11:25:57,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 2: [2023-05-08 11:25:57,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 2: [2023-05-08 11:25:57,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +25: [2023-05-08 11:25:57,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +25: [2023-05-08 11:25:57,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +25: [2023-05-08 11:25:57,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 2: [2023-05-08 11:25:57,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 2: [2023-05-08 11:25:57,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 2: [2023-05-08 11:25:57,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +25: [2023-05-08 11:25:57,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +26: [2023-05-08 11:25:57,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +26: [2023-05-08 11:25:57,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +26: [2023-05-08 11:25:57,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +26: [2023-05-08 11:25:57,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +26: [2023-05-08 11:25:57,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +26: [2023-05-08 11:25:57,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +26: [2023-05-08 11:25:57,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +26: [2023-05-08 11:25:57,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 9: [2023-05-08 11:25:57,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +12: [2023-05-08 11:25:57,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:25:57,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:25:57,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:25:57,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +26: [2023-05-08 11:25:57,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +26: [2023-05-08 11:25:57,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +26: [2023-05-08 11:25:57,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +26: [2023-05-08 11:25:57,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +26: [2023-05-08 11:25:57,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:25:57,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:25:57,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:25:57,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:25:57,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:25:57,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:25:57,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +15: [2023-05-08 11:25:57,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:25:57,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +12: [2023-05-08 11:25:57,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +12: [2023-05-08 11:25:57,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 7: [2023-05-08 11:25:57,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +31: [2023-05-08 11:25:57,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:25:57,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:25:57,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +31: [2023-05-08 11:25:57,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +31: [2023-05-08 11:25:57,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +31: [2023-05-08 11:25:57,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:25:57,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:25:57,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:25:57,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:25:57,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:25:57,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +15: [2023-05-08 11:25:57,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +15: [2023-05-08 11:25:57,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:25:57,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:25:57,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:25:57,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:25:57,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +19: [2023-05-08 11:25:57,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:25:57,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +30: [2023-05-08 11:25:57,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:25:57,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +18: [2023-05-08 11:25:57,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:25:57,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +21: [2023-05-08 11:25:57,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:25:57,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +25: [2023-05-08 11:25:57,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:25:57,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 2: [2023-05-08 11:25:57,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +18: [2023-05-08 11:25:57,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:25:57,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 2: [2023-05-08 11:25:57,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 2: [2023-05-08 11:25:57,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:25:57,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +22: [2023-05-08 11:25:57,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 6: [2023-05-08 11:25:57,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:25:57,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +22: [2023-05-08 11:25:57,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +22: [2023-05-08 11:25:57,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +18: [2023-05-08 11:25:57,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +15: [2023-05-08 11:25:57,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +15: [2023-05-08 11:25:57,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 6: [2023-05-08 11:25:57,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 6: [2023-05-08 11:25:57,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +11: [2023-05-08 11:25:57,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +25: [2023-05-08 11:25:57,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +25: [2023-05-08 11:25:57,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +25: [2023-05-08 11:25:57,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +11: [2023-05-08 11:25:57,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:25:57,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +26: [2023-05-08 11:25:57,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +26: [2023-05-08 11:25:57,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +31: [2023-05-08 11:25:57,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:25:57,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +16: [2023-05-08 11:25:57,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +16: [2023-05-08 11:25:57,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +15: [2023-05-08 11:25:57,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +19: [2023-05-08 11:25:57,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 7: [2023-05-08 11:25:57,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:25:57,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 7: [2023-05-08 11:25:57,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:25:57,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +30: [2023-05-08 11:25:57,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 7: [2023-05-08 11:25:57,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 7: [2023-05-08 11:25:57,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +31: [2023-05-08 11:25:57,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +31: [2023-05-08 11:25:57,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +31: [2023-05-08 11:25:57,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +30: [2023-05-08 11:25:57,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +19: [2023-05-08 11:25:57,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +19: [2023-05-08 11:25:57,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +16: [2023-05-08 11:25:57,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +11: [2023-05-08 11:25:57,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +16: [2023-05-08 11:25:57,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +19: [2023-05-08 11:25:57,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +30: [2023-05-08 11:25:57,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:25:57,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +11: [2023-05-08 11:25:57,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:25:57,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +30: [2023-05-08 11:25:57,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:25:57,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +25: [2023-05-08 11:25:57,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:25:57,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:25:57,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:25:57,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:25:57,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +11: [2023-05-08 11:25:57,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +25: [2023-05-08 11:25:57,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:25:57,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +25: [2023-05-08 11:25:57,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +25: [2023-05-08 11:25:57,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:25:57,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:25:57,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:25:57,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:25:57,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:25:57,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:25:57,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:25:57,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:25:57,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:25:57,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:25:57,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:25:57,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:25:57,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:25:57,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:25:57,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:25:57,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +20: [2023-05-08 11:25:57,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +20: [2023-05-08 11:25:57,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +20: [2023-05-08 11:25:57,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +20: [2023-05-08 11:25:57,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +20: [2023-05-08 11:25:57,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +20: [2023-05-08 11:25:57,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt... +20: [2023-05-08 11:25:57,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt... +20: [2023-05-08 11:25:57,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:25:57,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:25:57,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:25:57,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt. +20: [2023-05-08 11:25:57,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:25:57,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:25:57,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:25:57,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:25:57,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:25:57,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:25:57,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:25:57,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:25:57,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:25:57,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:25:57,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:25:57,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:25:57,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:25:57,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:25:57,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:25:57,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:25:57,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:25:57,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:25:57,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:25:57,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:25:57,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:25:57,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:25:57,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:25:57,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:25:57,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:25:57,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:25:57,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +23: [2023-05-08 11:25:57,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +23: [2023-05-08 11:25:57,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:25:57,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:25:57,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:25:57,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +23: [2023-05-08 11:25:57,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:25:57,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +28: [2023-05-08 11:25:57,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +28: [2023-05-08 11:25:57,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +28: [2023-05-08 11:25:57,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:25:57,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:25:57,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:25:57,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:25:57,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:25:57,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +17: [2023-05-08 11:25:57,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:25:57,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 3: [2023-05-08 11:25:57,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:25:57,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:25:57,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:25:57,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +28: [2023-05-08 11:25:57,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:25:57,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:25:57,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:25:57,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:25:57,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:25:57,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +24: [2023-05-08 11:25:57,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:25:57,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:25:57,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:25:57,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +24: [2023-05-08 11:25:57,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 3: [2023-05-08 11:25:57,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:25:57,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +14: [2023-05-08 11:25:57,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:25:57,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +10: [2023-05-08 11:25:57,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:25:57,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:25:57,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +10: [2023-05-08 11:25:57,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 3: [2023-05-08 11:25:57,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:25:57,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:25:57,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:25:57,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:25:57,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:25:57,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +14: [2023-05-08 11:25:57,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +24: [2023-05-08 11:25:57,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +24: [2023-05-08 11:25:57,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +24: [2023-05-08 11:25:57,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +10: [2023-05-08 11:25:57,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +10: [2023-05-08 11:25:57,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:25:57,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +27: [2023-05-08 11:25:57,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:25:57,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:25:57,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:25:57,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:25:57,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:25:57,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:25:57,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:25:57,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:25:57,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:25:57,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:25:57,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:25:57,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:25:57,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:25:57,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:25:57,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:25:57,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +18: [2023-05-08 11:25:57,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +18: [2023-05-08 11:25:57,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +18: [2023-05-08 11:25:57,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +18: [2023-05-08 11:25:57,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:25:57,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +18: [2023-05-08 11:25:57,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +18: [2023-05-08 11:25:57,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +18: [2023-05-08 11:25:57,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +12: [2023-05-08 11:25:57,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:25:57,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:25:57,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +18: [2023-05-08 11:25:57,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +12: [2023-05-08 11:25:57,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +12: [2023-05-08 11:25:57,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +12: [2023-05-08 11:25:57,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +12: [2023-05-08 11:25:57,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +23: [2023-05-08 11:25:57,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:25:57,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:25:57,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:25:57,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:25:57,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:25:57,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:25:57,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:25:57,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:25:57,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 3: [2023-05-08 11:25:57,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:25:57,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:25:57,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:25:57,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:25:57,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:25:57,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:25:57,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:25:57,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:25:57,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:25:57,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +19: [2023-05-08 11:25:57,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +23: [2023-05-08 11:25:57,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +13: [2023-05-08 11:25:57,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:25:57,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:25:57,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:25:57,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:25:57,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:25:57,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +28: [2023-05-08 11:25:57,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:25:57,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 3: [2023-05-08 11:25:57,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 3: [2023-05-08 11:25:57,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:25:57,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:25:57,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 6: [2023-05-08 11:25:57,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 6: [2023-05-08 11:25:57,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 6: [2023-05-08 11:25:57,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +10: [2023-05-08 11:25:57,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +28: [2023-05-08 11:25:57,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:25:57,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:25:57,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:25:57,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:25:57,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:25:57,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +28: [2023-05-08 11:25:57,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:25:57,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:25:57,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:25:57,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +27: [2023-05-08 11:25:57,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:25:57,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:25:57,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:25:57,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:25:57,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +15: [2023-05-08 11:25:57,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:25:57,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:25:57,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:25:57,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:25:57,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +24: [2023-05-08 11:25:57,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:25:57,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:25:57,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:25:57,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +10: [2023-05-08 11:25:57,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +27: [2023-05-08 11:25:57,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +27: [2023-05-08 11:25:57,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +14: [2023-05-08 11:25:57,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:25:57,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:25:57,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:25:57,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:25:57,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:25:57,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +13: [2023-05-08 11:25:57,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +18: [2023-05-08 11:25:57,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:25:57,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +24: [2023-05-08 11:25:57,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:25:57,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:25:57,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:25:57,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:25:57,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +31: [2023-05-08 11:25:57,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +31: [2023-05-08 11:25:57,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +31: [2023-05-08 11:25:57,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:25:57,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:25:57,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:25:57,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +31: [2023-05-08 11:25:57,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:25:57,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:25:57,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +11: [2023-05-08 11:25:57,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +31: [2023-05-08 11:25:57,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +31: [2023-05-08 11:25:57,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +11: [2023-05-08 11:25:57,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +14: [2023-05-08 11:25:57,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:25:57,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:25:57,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:25:57,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +10: [2023-05-08 11:25:57,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:25:57,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +12: [2023-05-08 11:25:57,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +18: [2023-05-08 11:25:57,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +18: [2023-05-08 11:25:57,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +31: [2023-05-08 11:25:57,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:25:57,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +18: [2023-05-08 11:25:57,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +27: [2023-05-08 11:25:57,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +24: [2023-05-08 11:25:57,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +24: [2023-05-08 11:25:57,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +28: [2023-05-08 11:25:57,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +10: [2023-05-08 11:25:57,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +10: [2023-05-08 11:25:57,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:25:57,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:25:57,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:25:57,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +28: [2023-05-08 11:25:57,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +18: [2023-05-08 11:25:57,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +27: [2023-05-08 11:25:57,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +27: [2023-05-08 11:25:57,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:25:57,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:25:57,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:25:57,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:25:57,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +18: [2023-05-08 11:25:57,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +18: [2023-05-08 11:25:57,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +18: [2023-05-08 11:25:57,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:25:57,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +24: [2023-05-08 11:25:57,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:25:57,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +16: [2023-05-08 11:25:57,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +16: [2023-05-08 11:25:57,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +16: [2023-05-08 11:25:57,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:25:57,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:25:57,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:25:57,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:25:57,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:25:57,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:25:57,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:25:57,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:25:57,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 6: [2023-05-08 11:25:57,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:25:57,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:25:57,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:25:57,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:25:57,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 6: [2023-05-08 11:25:57,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:25:57,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:25:57,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:25:57,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 6: [2023-05-08 11:25:57,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 6: [2023-05-08 11:25:57,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:25:57,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:25:57,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:25:57,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:25:57,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:25:57,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:25:57,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +31: [2023-05-08 11:25:57,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:25:57,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:25:57,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +31: [2023-05-08 11:25:57,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:25:57,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:25:57,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:25:57,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:25:57,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:25:57,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:25:57,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:25:57,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:25:57,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:25:57,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:25:57,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:25:57,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:25:57,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:25:57,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:25:57,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:25:57,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +29: [2023-05-08 11:25:57,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +25: [2023-05-08 11:25:57,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:25:57,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +29: [2023-05-08 11:25:57,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +16: [2023-05-08 11:25:57,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:25:57,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:25:57,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +25: [2023-05-08 11:25:57,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +25: [2023-05-08 11:25:57,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +25: [2023-05-08 11:25:57,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +31: [2023-05-08 11:25:57,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:25:57,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:25:57,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:25:57,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:25:57,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:25:57,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:25:57,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:25:57,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:25:57,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:25:57,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:25:57,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:25:57,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:25:57,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:25:57,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:25:57,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:25:57,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 2: [2023-05-08 11:25:57,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:25:57,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 2: [2023-05-08 11:25:57,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:25:57,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 2: [2023-05-08 11:25:57,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:25:57,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +20: [2023-05-08 11:25:57,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:25:57,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +23: [2023-05-08 11:25:57,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +23: [2023-05-08 11:25:57,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 0: [2023-05-08 11:25:57,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 0: [2023-05-08 11:25:57,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 0: [2023-05-08 11:25:57,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 0: [2023-05-08 11:25:57,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +28: [2023-05-08 11:25:57,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +28: [2023-05-08 11:25:57,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +28: [2023-05-08 11:25:57,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +21: [2023-05-08 11:25:57,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:25:57,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +21: [2023-05-08 11:25:57,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 7: [2023-05-08 11:25:57,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:25:57,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:25:57,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +25: [2023-05-08 11:25:57,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:25:57,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:25:57,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:25:57,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:25:57,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:25:57,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:25:57,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +17: [2023-05-08 11:25:57,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +17: [2023-05-08 11:25:57,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 2: [2023-05-08 11:25:57,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:25:57,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +31: [2023-05-08 11:25:57,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 2: [2023-05-08 11:25:57,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +31: [2023-05-08 11:25:57,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +31: [2023-05-08 11:25:57,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 2: [2023-05-08 11:25:57,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +31: [2023-05-08 11:25:57,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 8: [2023-05-08 11:25:57,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:25:57,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 7: [2023-05-08 11:25:57,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 7: [2023-05-08 11:25:57,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 7: [2023-05-08 11:25:57,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:25:57,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:25:57,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:25:57,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:25:57,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:25:57,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:25:57,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:25:57,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:25:57,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:25:57,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:25:57,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:25:57,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +26: [2023-05-08 11:25:57,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +26: [2023-05-08 11:25:57,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +26: [2023-05-08 11:25:57,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:25:57,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +23: [2023-05-08 11:25:57,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:25:57,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +22: [2023-05-08 11:25:57,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:25:57,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:25:57,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:25:57,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:25:57,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:25:57,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:25:57,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:25:57,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:25:57,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:25:57,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:25:57,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +29: [2023-05-08 11:25:57,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:25:57,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:25:57,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +25: [2023-05-08 11:25:57,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +28: [2023-05-08 11:25:57,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:25:57,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:25:57,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:25:57,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:25:57,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +31: [2023-05-08 11:25:57,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +29: [2023-05-08 11:25:57,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +29: [2023-05-08 11:25:57,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +29: [2023-05-08 11:25:57,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 1: [2023-05-08 11:25:57,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:25:57,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +25: [2023-05-08 11:25:57,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:25:57,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:25:57,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:25:57,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:25:57,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +21: [2023-05-08 11:25:57,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +21: [2023-05-08 11:25:57,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +31: [2023-05-08 11:25:57,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:25:57,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 8: [2023-05-08 11:25:57,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:25:57,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 2: [2023-05-08 11:25:57,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 2: [2023-05-08 11:25:57,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:25:57,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:25:57,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:25:57,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:25:57,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:25:57,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:25:57,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 2: [2023-05-08 11:25:57,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:25:57,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 2: [2023-05-08 11:25:57,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:25:57,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +25: [2023-05-08 11:25:57,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +25: [2023-05-08 11:25:57,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +25: [2023-05-08 11:25:57,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +23: [2023-05-08 11:25:57,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:25:57,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +23: [2023-05-08 11:25:57,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +30: [2023-05-08 11:25:57,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:25:57,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:25:57,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +30: [2023-05-08 11:25:57,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:25:57,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:25:57,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +21: [2023-05-08 11:25:57,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +21: [2023-05-08 11:25:57,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +30: [2023-05-08 11:25:57,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:25:57,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:25:57,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +30: [2023-05-08 11:25:57,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:25:57,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:25:57,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:25:57,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:25:57,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +30: [2023-05-08 11:25:57,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:25:57,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:25:57,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +30: [2023-05-08 11:25:57,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +23: [2023-05-08 11:25:57,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:25:57,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +23: [2023-05-08 11:25:57,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:25:57,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:25:57,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:25:57,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +17: [2023-05-08 11:25:57,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:25:57,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:25:57,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 7: [2023-05-08 11:25:57,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +21: [2023-05-08 11:25:57,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +26: [2023-05-08 11:25:57,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +26: [2023-05-08 11:25:57,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:25:57,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +28: [2023-05-08 11:25:57,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 8: [2023-05-08 11:25:57,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 7: [2023-05-08 11:25:57,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:25:57,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:25:57,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +17: [2023-05-08 11:25:57,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +21: [2023-05-08 11:25:57,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +17: [2023-05-08 11:25:57,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 7: [2023-05-08 11:25:57,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:25:57,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +23: [2023-05-08 11:25:57,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +17: [2023-05-08 11:25:57,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +23: [2023-05-08 11:25:57,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 8: [2023-05-08 11:25:57,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +28: [2023-05-08 11:25:57,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 2: [2023-05-08 11:25:57,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:25:57,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:25:57,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:25:57,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +31: [2023-05-08 11:25:57,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +28: [2023-05-08 11:25:57,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +28: [2023-05-08 11:25:57,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +28: [2023-05-08 11:25:57,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +31: [2023-05-08 11:25:57,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:25:57,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +31: [2023-05-08 11:25:57,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 0: [2023-05-08 11:25:57,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 0: [2023-05-08 11:25:57,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 2: [2023-05-08 11:25:57,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:25:57,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +31: [2023-05-08 11:25:57,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +28: [2023-05-08 11:25:57,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +28: [2023-05-08 11:25:57,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +26: [2023-05-08 11:25:57,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +28: [2023-05-08 11:25:57,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +31: [2023-05-08 11:25:57,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +31: [2023-05-08 11:25:57,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +26: [2023-05-08 11:25:57,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +22: [2023-05-08 11:25:57,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:25:57,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +25: [2023-05-08 11:25:57,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:25:57,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +22: [2023-05-08 11:25:57,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:25:57,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +11: [2023-05-08 11:25:57,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +11: [2023-05-08 11:25:57,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +25: [2023-05-08 11:25:57,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +11: [2023-05-08 11:25:57,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +25: [2023-05-08 11:25:57,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:25:57,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:25:57,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:25:57,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +26: [2023-05-08 11:25:57,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +30: [2023-05-08 11:25:57,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:25:57,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:25:57,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +11: [2023-05-08 11:25:57,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +11: [2023-05-08 11:25:57,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +11: [2023-05-08 11:25:57,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +21: [2023-05-08 11:25:57,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +30: [2023-05-08 11:25:57,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:25:57,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:25:57,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:25:57,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:25:57,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +21: [2023-05-08 11:25:57,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:25:57,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +21: [2023-05-08 11:25:57,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +21: [2023-05-08 11:25:57,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +21: [2023-05-08 11:25:57,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +30: [2023-05-08 11:25:57,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:25:57,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:25:57,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:25:57,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:25:57,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +25: [2023-05-08 11:25:57,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:25:57,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:25:57,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +25: [2023-05-08 11:25:57,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +25: [2023-05-08 11:25:57,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +25: [2023-05-08 11:25:57,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +11: [2023-05-08 11:25:57,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:25:57,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +22: [2023-05-08 11:25:57,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +22: [2023-05-08 11:25:57,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +22: [2023-05-08 11:25:57,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +11: [2023-05-08 11:25:57,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:25:57,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:25:57,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +11: [2023-05-08 11:25:57,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +11: [2023-05-08 11:25:57,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +11: [2023-05-08 11:25:57,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +11: [2023-05-08 11:25:57,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +24: [2023-05-08 11:25:57,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 6: [2023-05-08 11:25:57,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 6: [2023-05-08 11:25:57,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 6: [2023-05-08 11:25:57,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +24: [2023-05-08 11:25:57,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +24: [2023-05-08 11:25:57,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 7: [2023-05-08 11:25:57,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 6: [2023-05-08 11:25:57,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 7: [2023-05-08 11:25:57,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 7: [2023-05-08 11:25:57,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +24: [2023-05-08 11:25:57,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 7: [2023-05-08 11:25:57,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +14: [2023-05-08 11:25:57,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +14: [2023-05-08 11:25:57,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +14: [2023-05-08 11:25:57,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +14: [2023-05-08 11:25:57,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +27: [2023-05-08 11:25:57,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +30: [2023-05-08 11:25:57,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +30: [2023-05-08 11:25:57,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +27: [2023-05-08 11:25:57,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +27: [2023-05-08 11:25:57,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +30: [2023-05-08 11:25:57,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 9: [2023-05-08 11:25:57,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 9: [2023-05-08 11:25:57,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +27: [2023-05-08 11:25:57,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +30: [2023-05-08 11:25:57,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 9: [2023-05-08 11:25:57,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 9: [2023-05-08 11:25:57,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +22: [2023-05-08 11:25:57,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +18: [2023-05-08 11:25:57,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +18: [2023-05-08 11:25:57,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +18: [2023-05-08 11:25:57,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +22: [2023-05-08 11:25:57,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 4: [2023-05-08 11:25:57,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 4: [2023-05-08 11:25:57,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +22: [2023-05-08 11:25:57,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +22: [2023-05-08 11:25:57,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +18: [2023-05-08 11:25:57,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +26: [2023-05-08 11:25:57,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +26: [2023-05-08 11:25:57,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +26: [2023-05-08 11:25:57,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +12: [2023-05-08 11:25:57,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +26: [2023-05-08 11:25:57,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +12: [2023-05-08 11:25:57,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +12: [2023-05-08 11:25:57,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +12: [2023-05-08 11:25:57,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +19: [2023-05-08 11:25:57,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +19: [2023-05-08 11:25:57,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +19: [2023-05-08 11:25:57,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 1: [2023-05-08 11:25:57,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 1: [2023-05-08 11:25:57,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +19: [2023-05-08 11:25:57,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 1: [2023-05-08 11:25:57,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 1: [2023-05-08 11:25:57,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 5: [2023-05-08 11:25:57,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 5: [2023-05-08 11:25:57,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 5: [2023-05-08 11:25:57,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 5: [2023-05-08 11:25:57,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 3: [2023-05-08 11:25:57,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 3: [2023-05-08 11:25:57,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 3: [2023-05-08 11:25:57,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 3: [2023-05-08 11:25:57,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +13: [2023-05-08 11:25:57,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +13: [2023-05-08 11:25:57,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +13: [2023-05-08 11:25:57,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +13: [2023-05-08 11:25:57,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +15: [2023-05-08 11:25:57,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +15: [2023-05-08 11:25:57,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +15: [2023-05-08 11:25:57,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +15: [2023-05-08 11:25:57,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +10: [2023-05-08 11:25:57,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +10: [2023-05-08 11:25:57,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +10: [2023-05-08 11:25:57,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +10: [2023-05-08 11:25:57,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 2: [2023-05-08 11:25:57,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 2: [2023-05-08 11:25:57,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 2: [2023-05-08 11:25:57,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 2: [2023-05-08 11:25:57,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +16: [2023-05-08 11:25:57,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +16: [2023-05-08 11:25:57,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +16: [2023-05-08 11:25:57,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 8: [2023-05-08 11:25:57,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 8: [2023-05-08 11:25:57,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 8: [2023-05-08 11:25:57,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +16: [2023-05-08 11:25:57,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. + 8: [2023-05-08 11:25:57,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +14: [2023-05-08 11:25:57,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +24: [2023-05-08 11:25:57,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +14: [2023-05-08 11:25:57,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 6: [2023-05-08 11:25:57,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 7: [2023-05-08 11:25:57,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 6: [2023-05-08 11:25:57,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 6: [2023-05-08 11:25:57,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +14: [2023-05-08 11:25:57,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +14: [2023-05-08 11:25:57,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 6: [2023-05-08 11:25:57,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +18: [2023-05-08 11:25:57,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:25:57,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +20: [2023-05-08 11:25:57,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +20: [2023-05-08 11:25:57,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +30: [2023-05-08 11:25:57,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +18: [2023-05-08 11:25:57,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:25:57,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt. +26: [2023-05-08 11:25:57,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:25:57,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +12: [2023-05-08 11:25:57,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +24: [2023-05-08 11:25:57,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 9: [2023-05-08 11:25:57,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:25:57,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +18: [2023-05-08 11:25:57,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +18: [2023-05-08 11:25:57,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 7: [2023-05-08 11:25:57,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +24: [2023-05-08 11:25:57,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +24: [2023-05-08 11:25:57,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 4: [2023-05-08 11:25:57,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 7: [2023-05-08 11:25:57,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +30: [2023-05-08 11:25:57,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:25:57,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:25:57,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:25:57,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +19: [2023-05-08 11:25:57,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +12: [2023-05-08 11:25:57,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +15: [2023-05-08 11:25:57,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +15: [2023-05-08 11:25:57,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +15: [2023-05-08 11:25:57,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +12: [2023-05-08 11:25:57,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +10: [2023-05-08 11:25:57,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +12: [2023-05-08 11:25:57,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +27: [2023-05-08 11:25:57,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +19: [2023-05-08 11:25:57,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +19: [2023-05-08 11:25:57,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +19: [2023-05-08 11:25:57,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:25:57,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 7: [2023-05-08 11:25:57,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +16: [2023-05-08 11:25:57,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:25:57,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +13: [2023-05-08 11:25:57,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:25:57,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 5: [2023-05-08 11:25:57,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +10: [2023-05-08 11:25:57,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 1: [2023-05-08 11:25:57,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +26: [2023-05-08 11:25:57,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +10: [2023-05-08 11:25:57,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +10: [2023-05-08 11:25:57,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +15: [2023-05-08 11:25:57,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +30: [2023-05-08 11:25:57,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:25:57,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 3: [2023-05-08 11:25:57,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +30: [2023-05-08 11:25:57,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 8: [2023-05-08 11:25:57,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +16: [2023-05-08 11:25:57,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +16: [2023-05-08 11:25:57,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +16: [2023-05-08 11:25:57,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:25:57,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:25:57,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:25:57,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:25:57,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:25:57,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:25:57,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +22: [2023-05-08 11:25:57,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 2: [2023-05-08 11:25:57,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +20: [2023-05-08 11:25:57,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... + 2: [2023-05-08 11:25:57,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +22: [2023-05-08 11:25:57,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +22: [2023-05-08 11:25:57,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +22: [2023-05-08 11:25:57,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +22: [2023-05-08 11:25:57,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 2: [2023-05-08 11:25:57,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt... +14: [2023-05-08 11:25:57,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:25:57,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:25:57,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +14: [2023-05-08 11:25:57,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:25:57,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +14: [2023-05-08 11:25:57,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 4: [2023-05-08 11:25:57,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +18: [2023-05-08 11:25:57,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:25:57,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:25:57,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:25:57,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:25:57,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 9: [2023-05-08 11:25:57,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:25:57,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 6: [2023-05-08 11:25:57,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +18: [2023-05-08 11:25:57,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:25:57,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +18: [2023-05-08 11:25:57,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +27: [2023-05-08 11:25:57,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +26: [2023-05-08 11:25:57,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +26: [2023-05-08 11:25:57,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +13: [2023-05-08 11:25:57,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 6: [2023-05-08 11:25:57,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 6: [2023-05-08 11:25:57,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 6: [2023-05-08 11:25:57,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 6: [2023-05-08 11:25:57,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +19: [2023-05-08 11:25:57,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +14: [2023-05-08 11:25:57,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +14: [2023-05-08 11:25:57,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +18: [2023-05-08 11:25:57,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +18: [2023-05-08 11:25:57,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +18: [2023-05-08 11:25:57,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:25:57,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +15: [2023-05-08 11:25:57,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:25:57,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:25:57,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 7: [2023-05-08 11:25:57,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:25:57,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:25:57,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:25:57,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 7: [2023-05-08 11:25:57,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +27: [2023-05-08 11:25:57,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +26: [2023-05-08 11:25:57,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +26: [2023-05-08 11:25:57,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +18: [2023-05-08 11:25:57,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +15: [2023-05-08 11:25:57,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +18: [2023-05-08 11:25:57,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +30: [2023-05-08 11:25:57,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:25:57,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 9: [2023-05-08 11:25:57,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +19: [2023-05-08 11:25:57,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 9: [2023-05-08 11:25:57,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:25:57,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:25:57,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:25:57,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:25:57,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +24: [2023-05-08 11:25:57,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +27: [2023-05-08 11:25:57,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +27: [2023-05-08 11:25:57,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +27: [2023-05-08 11:25:57,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +24: [2023-05-08 11:25:57,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +24: [2023-05-08 11:25:57,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 7: [2023-05-08 11:25:57,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 7: [2023-05-08 11:25:57,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +13: [2023-05-08 11:25:57,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:25:57,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:25:57,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:25:57,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +26: [2023-05-08 11:25:57,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:25:57,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:25:57,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +15: [2023-05-08 11:25:57,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 5: [2023-05-08 11:25:57,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:25:57,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +26: [2023-05-08 11:25:57,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 4: [2023-05-08 11:25:57,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 7: [2023-05-08 11:25:57,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:25:57,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 4: [2023-05-08 11:25:57,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +12: [2023-05-08 11:25:57,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:25:57,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +12: [2023-05-08 11:25:57,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +13: [2023-05-08 11:25:57,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +15: [2023-05-08 11:25:57,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 5: [2023-05-08 11:25:57,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +10: [2023-05-08 11:25:57,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:25:57,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 9: [2023-05-08 11:25:57,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +13: [2023-05-08 11:25:57,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +16: [2023-05-08 11:25:57,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +19: [2023-05-08 11:25:57,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:25:57,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +19: [2023-05-08 11:25:57,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +27: [2023-05-08 11:25:57,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +27: [2023-05-08 11:25:57,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +27: [2023-05-08 11:25:57,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +10: [2023-05-08 11:25:57,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +30: [2023-05-08 11:25:57,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +13: [2023-05-08 11:25:57,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 9: [2023-05-08 11:25:57,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 3: [2023-05-08 11:25:57,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 3: [2023-05-08 11:25:57,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 3: [2023-05-08 11:25:57,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +26: [2023-05-08 11:25:57,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 1: [2023-05-08 11:25:57,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 1: [2023-05-08 11:25:57,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 5: [2023-05-08 11:25:57,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 2: [2023-05-08 11:25:57,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 5: [2023-05-08 11:25:57,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +26: [2023-05-08 11:25:57,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 7: [2023-05-08 11:25:57,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 1: [2023-05-08 11:25:57,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 1: [2023-05-08 11:25:57,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:25:57,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +15: [2023-05-08 11:25:57,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 3: [2023-05-08 11:25:57,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 8: [2023-05-08 11:25:57,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +30: [2023-05-08 11:25:57,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:25:57,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 8: [2023-05-08 11:25:57,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +10: [2023-05-08 11:25:57,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 2: [2023-05-08 11:25:57,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +19: [2023-05-08 11:25:57,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +19: [2023-05-08 11:25:57,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 8: [2023-05-08 11:25:57,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +16: [2023-05-08 11:25:57,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:25:57,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +19: [2023-05-08 11:25:57,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 1: [2023-05-08 11:25:57,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 1: [2023-05-08 11:25:57,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +12: [2023-05-08 11:25:57,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +12: [2023-05-08 11:25:57,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +12: [2023-05-08 11:25:57,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 3: [2023-05-08 11:25:57,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 3: [2023-05-08 11:25:57,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 3: [2023-05-08 11:25:57,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 8: [2023-05-08 11:25:57,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +30: [2023-05-08 11:25:57,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 8: [2023-05-08 11:25:57,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 8: [2023-05-08 11:25:57,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 3: [2023-05-08 11:25:57,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +16: [2023-05-08 11:25:57,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:25:57,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:25:57,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +20: [2023-05-08 11:25:57,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:25:57,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:25:57,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +16: [2023-05-08 11:25:57,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +16: [2023-05-08 11:25:57,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +20: [2023-05-08 11:25:57,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 2: [2023-05-08 11:25:57,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. +20: [2023-05-08 11:25:57,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +20: [2023-05-08 11:25:57,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 2: [2023-05-08 11:25:57,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 2: [2023-05-08 11:25:57,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt. + 2: [2023-05-08 11:25:57,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 2: [2023-05-08 11:25:57,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... + 2: [2023-05-08 11:25:57,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt... +13: [2023-05-08 11:25:58,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:25:58,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +13: [2023-05-08 11:25:58,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:25:58,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:25:58,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +13: [2023-05-08 11:25:58,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +13: [2023-05-08 11:25:58,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:25:58,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +13: [2023-05-08 11:25:58,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:25:58,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:25:58,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:25:58,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:25:58,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:25:58,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:25:58,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:25:58,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +27: [2023-05-08 11:25:58,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +27: [2023-05-08 11:25:58,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +27: [2023-05-08 11:25:58,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:25:58,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +13: [2023-05-08 11:25:58,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +13: [2023-05-08 11:25:58,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:25:58,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:25:58,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +13: [2023-05-08 11:25:58,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:25:58,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:25:58,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:25:58,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:25:58,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:25:58,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +23: [2023-05-08 11:25:58,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +23: [2023-05-08 11:25:58,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +23: [2023-05-08 11:25:58,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +23: [2023-05-08 11:25:58,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +10: [2023-05-08 11:25:58,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +10: [2023-05-08 11:25:58,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +10: [2023-05-08 11:25:58,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +10: [2023-05-08 11:25:58,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +10: [2023-05-08 11:25:58,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +10: [2023-05-08 11:25:58,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +10: [2023-05-08 11:25:58,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +10: [2023-05-08 11:25:58,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +28: [2023-05-08 11:25:58,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +28: [2023-05-08 11:25:58,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +28: [2023-05-08 11:25:58,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +28: [2023-05-08 11:25:58,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +28: [2023-05-08 11:25:58,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +28: [2023-05-08 11:25:58,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +28: [2023-05-08 11:25:58,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +28: [2023-05-08 11:25:58,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +27: [2023-05-08 11:25:58,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:25:58,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +24: [2023-05-08 11:25:58,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:25:58,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:25:58,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +24: [2023-05-08 11:25:58,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +24: [2023-05-08 11:25:58,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +14: [2023-05-08 11:25:58,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:25:58,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:25:58,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +24: [2023-05-08 11:25:58,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:25:58,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:25:58,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +14: [2023-05-08 11:25:58,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +14: [2023-05-08 11:25:58,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +14: [2023-05-08 11:25:58,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:25:58,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:25:58,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:25:58,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:25:58,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:25:58,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:25:58,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:25:58,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:25:58,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +17: [2023-05-08 11:25:58,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:25:58,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:25:58,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +17: [2023-05-08 11:25:58,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:25:58,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:25:58,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +17: [2023-05-08 11:25:58,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:25:58,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:25:58,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:25:58,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:25:58,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:25:58,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +17: [2023-05-08 11:25:58,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:25:58,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +10: [2023-05-08 11:25:58,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +10: [2023-05-08 11:25:58,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +10: [2023-05-08 11:25:58,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +10: [2023-05-08 11:25:58,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +14: [2023-05-08 11:25:58,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +14: [2023-05-08 11:25:58,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +28: [2023-05-08 11:25:58,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +28: [2023-05-08 11:25:58,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +28: [2023-05-08 11:25:58,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +28: [2023-05-08 11:25:58,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +14: [2023-05-08 11:25:58,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +14: [2023-05-08 11:25:58,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +24: [2023-05-08 11:25:58,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:25:58,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:25:58,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:25:58,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:25:58,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:25:58,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:25:58,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:25:58,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +10: [2023-05-08 11:25:58,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:25:58,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +23: [2023-05-08 11:25:58,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:25:58,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +14: [2023-05-08 11:25:58,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:25:58,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +14: [2023-05-08 11:25:58,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:25:58,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +28: [2023-05-08 11:25:58,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:25:58,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +24: [2023-05-08 11:25:58,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +24: [2023-05-08 11:25:58,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:25:58,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +14: [2023-05-08 11:25:58,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:25:58,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +14: [2023-05-08 11:25:58,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +23: [2023-05-08 11:25:58,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +23: [2023-05-08 11:25:58,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:25:58,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +24: [2023-05-08 11:25:58,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +24: [2023-05-08 11:25:58,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 0: [2023-05-08 11:25:58,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 0: [2023-05-08 11:25:58,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +29: [2023-05-08 11:25:58,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +29: [2023-05-08 11:25:58,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +29: [2023-05-08 11:25:58,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:25:58,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +17: [2023-05-08 11:25:58,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +29: [2023-05-08 11:25:58,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +29: [2023-05-08 11:25:58,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +29: [2023-05-08 11:25:58,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:25:58,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +21: [2023-05-08 11:25:58,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +21: [2023-05-08 11:25:58,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +21: [2023-05-08 11:25:58,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +21: [2023-05-08 11:25:58,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +17: [2023-05-08 11:25:58,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +17: [2023-05-08 11:25:58,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:25:58,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:25:58,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:25:58,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:25:58,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:25:58,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:25:58,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:25:58,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:25:58,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:25:58,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:25:58,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:25:58,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:25:58,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +23: [2023-05-08 11:25:58,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:25:58,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:25:58,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +23: [2023-05-08 11:25:58,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +23: [2023-05-08 11:25:58,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:25:58,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 0: [2023-05-08 11:25:58,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:25:58,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +29: [2023-05-08 11:25:58,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +29: [2023-05-08 11:25:58,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +17: [2023-05-08 11:25:58,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:25:58,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:25:58,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +21: [2023-05-08 11:25:58,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +21: [2023-05-08 11:25:58,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +21: [2023-05-08 11:25:58,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:25:58,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +21: [2023-05-08 11:25:58,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +21: [2023-05-08 11:25:58,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +21: [2023-05-08 11:25:58,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:25:58,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +23: [2023-05-08 11:25:58,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:25:58,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:25:58,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +21: [2023-05-08 11:25:58,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:25:58,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:25:58,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +29: [2023-05-08 11:25:58,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +29: [2023-05-08 11:25:58,158] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:25:58,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:25:58,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:25:58,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +21: [2023-05-08 11:25:58,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +21: [2023-05-08 11:25:58,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +31: [2023-05-08 11:25:58,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:25:58,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +29: [2023-05-08 11:25:58,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:25:58,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:25:58,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +29: [2023-05-08 11:25:58,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:25:58,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +21: [2023-05-08 11:25:58,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +21: [2023-05-08 11:25:58,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +21: [2023-05-08 11:25:58,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 0: [2023-05-08 11:25:58,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +31: [2023-05-08 11:25:58,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:25:58,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:25:58,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:25:58,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:25:58,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:25:58,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:25:58,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +21: [2023-05-08 11:25:58,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:25:58,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +21: [2023-05-08 11:25:58,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +21: [2023-05-08 11:25:58,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:25:58,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +21: [2023-05-08 11:25:58,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +21: [2023-05-08 11:25:58,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 0: [2023-05-08 11:25:58,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +19: [2023-05-08 11:25:58,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:25:58,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +21: [2023-05-08 11:25:58,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:25:58,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:25:58,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:25:58,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +25: [2023-05-08 11:25:58,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:25:58,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +25: [2023-05-08 11:25:58,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +25: [2023-05-08 11:25:58,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +25: [2023-05-08 11:25:58,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:25:58,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:25:58,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:25:58,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +11: [2023-05-08 11:25:58,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +11: [2023-05-08 11:25:58,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:25:58,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:25:58,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:25:58,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:25:58,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:25:58,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +11: [2023-05-08 11:25:58,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +11: [2023-05-08 11:25:58,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +11: [2023-05-08 11:25:58,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:25:58,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:25:58,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +29: [2023-05-08 11:25:58,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 4: [2023-05-08 11:25:58,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +18: [2023-05-08 11:25:58,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:25:58,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:25:58,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +18: [2023-05-08 11:25:58,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:25:58,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:25:58,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:25:58,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +18: [2023-05-08 11:25:58,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:25:58,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:25:58,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +15: [2023-05-08 11:25:58,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +15: [2023-05-08 11:25:58,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +15: [2023-05-08 11:25:58,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +15: [2023-05-08 11:25:58,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:25:58,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:25:58,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:25:58,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +15: [2023-05-08 11:25:58,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +29: [2023-05-08 11:25:58,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +29: [2023-05-08 11:25:58,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +29: [2023-05-08 11:25:58,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +29: [2023-05-08 11:25:58,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 2: [2023-05-08 11:25:58,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +21: [2023-05-08 11:25:58,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +21: [2023-05-08 11:25:58,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +20: [2023-05-08 11:25:58,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:25:58,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:25:58,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +20: [2023-05-08 11:25:58,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:25:58,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:25:58,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:25:58,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:25:58,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:25:58,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:25:58,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:25:58,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +25: [2023-05-08 11:25:58,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +11: [2023-05-08 11:25:58,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +22: [2023-05-08 11:25:58,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +22: [2023-05-08 11:25:58,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +22: [2023-05-08 11:25:58,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +22: [2023-05-08 11:25:58,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +22: [2023-05-08 11:25:58,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +22: [2023-05-08 11:25:58,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:25:58,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +16: [2023-05-08 11:25:58,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +16: [2023-05-08 11:25:58,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:25:58,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +22: [2023-05-08 11:25:58,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +22: [2023-05-08 11:25:58,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:25:58,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:25:58,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:25:58,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:25:58,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:25:58,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:25:58,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:25:58,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:25:58,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:25:58,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:25:58,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +16: [2023-05-08 11:25:58,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:25:58,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:25:58,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +16: [2023-05-08 11:25:58,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +26: [2023-05-08 11:25:58,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +26: [2023-05-08 11:25:58,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +26: [2023-05-08 11:25:58,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:25:58,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:25:58,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +26: [2023-05-08 11:25:58,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:25:58,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +16: [2023-05-08 11:25:58,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:25:58,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:25:58,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:25:58,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:25:58,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +12: [2023-05-08 11:25:58,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +12: [2023-05-08 11:25:58,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +12: [2023-05-08 11:25:58,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:25:58,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:25:58,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:25:58,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:25:58,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:25:58,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:25:58,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:25:58,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +11: [2023-05-08 11:25:58,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +12: [2023-05-08 11:25:58,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:25:58,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +25: [2023-05-08 11:25:58,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:25:58,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +25: [2023-05-08 11:25:58,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:25:58,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:25:58,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 1: [2023-05-08 11:25:58,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +15: [2023-05-08 11:25:58,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +15: [2023-05-08 11:25:58,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:25:58,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:25:58,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +15: [2023-05-08 11:25:58,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +11: [2023-05-08 11:25:58,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +11: [2023-05-08 11:25:58,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:25:58,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +26: [2023-05-08 11:25:58,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +11: [2023-05-08 11:25:58,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:25:58,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +16: [2023-05-08 11:25:58,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +16: [2023-05-08 11:25:58,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +22: [2023-05-08 11:25:58,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +22: [2023-05-08 11:25:58,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +22: [2023-05-08 11:25:58,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +17: [2023-05-08 11:25:58,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +17: [2023-05-08 11:25:58,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +22: [2023-05-08 11:25:58,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +17: [2023-05-08 11:25:58,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 5: [2023-05-08 11:25:58,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:25:58,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 8: [2023-05-08 11:25:58,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:25:58,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:25:58,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:25:58,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:25:58,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +25: [2023-05-08 11:25:58,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +16: [2023-05-08 11:25:58,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:25:58,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:25:58,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:25:58,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:25:58,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +12: [2023-05-08 11:25:58,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:25:58,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:25:58,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:25:58,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +18: [2023-05-08 11:25:58,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:25:58,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 8: [2023-05-08 11:25:58,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:25:58,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +25: [2023-05-08 11:25:58,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +18: [2023-05-08 11:25:58,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +18: [2023-05-08 11:25:58,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:25:58,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 8: [2023-05-08 11:25:58,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +25: [2023-05-08 11:25:58,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:25:58,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:25:58,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +12: [2023-05-08 11:25:58,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +12: [2023-05-08 11:25:58,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:25:58,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:25:58,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:25:58,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:25:58,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:25:58,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:25:58,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:25:58,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:25:58,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:25:58,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +22: [2023-05-08 11:25:58,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:25:58,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +20: [2023-05-08 11:25:58,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +16: [2023-05-08 11:25:58,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:25:58,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +22: [2023-05-08 11:25:58,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:25:58,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:25:58,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:25:58,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:25:58,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +20: [2023-05-08 11:25:58,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +20: [2023-05-08 11:25:58,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:25:58,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:25:58,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +26: [2023-05-08 11:25:58,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +16: [2023-05-08 11:25:58,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:25:58,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +17: [2023-05-08 11:25:58,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +30: [2023-05-08 11:25:58,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +17: [2023-05-08 11:25:58,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:25:58,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 4: [2023-05-08 11:25:58,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 4: [2023-05-08 11:25:58,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 1: [2023-05-08 11:25:58,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +25: [2023-05-08 11:25:58,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:25:58,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:25:58,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +20: [2023-05-08 11:25:58,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +22: [2023-05-08 11:25:58,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 6: [2023-05-08 11:25:58,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +12: [2023-05-08 11:25:58,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +22: [2023-05-08 11:25:58,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +16: [2023-05-08 11:25:58,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:25:58,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:25:58,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +17: [2023-05-08 11:25:58,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:25:58,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +12: [2023-05-08 11:25:58,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +12: [2023-05-08 11:25:58,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +12: [2023-05-08 11:25:58,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +12: [2023-05-08 11:25:58,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:25:58,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +30: [2023-05-08 11:25:58,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +17: [2023-05-08 11:25:58,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:25:58,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +30: [2023-05-08 11:25:58,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +30: [2023-05-08 11:25:58,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +30: [2023-05-08 11:25:58,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +20: [2023-05-08 11:25:58,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:25:58,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:25:58,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +17: [2023-05-08 11:25:58,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +17: [2023-05-08 11:25:58,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +22: [2023-05-08 11:25:58,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:25:58,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +18: [2023-05-08 11:25:58,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +22: [2023-05-08 11:25:58,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +18: [2023-05-08 11:25:58,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 6: [2023-05-08 11:25:58,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:25:58,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 6: [2023-05-08 11:25:58,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +22: [2023-05-08 11:25:58,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +22: [2023-05-08 11:25:58,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +22: [2023-05-08 11:25:58,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +22: [2023-05-08 11:25:58,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +12: [2023-05-08 11:25:58,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +25: [2023-05-08 11:25:58,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +25: [2023-05-08 11:25:58,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +25: [2023-05-08 11:25:58,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +25: [2023-05-08 11:25:58,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +12: [2023-05-08 11:25:58,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +22: [2023-05-08 11:25:58,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:25:58,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +22: [2023-05-08 11:25:58,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +12: [2023-05-08 11:25:58,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:25:58,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +25: [2023-05-08 11:25:58,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:25:58,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +30: [2023-05-08 11:25:58,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +18: [2023-05-08 11:25:58,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +18: [2023-05-08 11:25:58,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +18: [2023-05-08 11:25:58,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:25:58,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:25:58,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +18: [2023-05-08 11:25:58,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +25: [2023-05-08 11:25:58,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +17: [2023-05-08 11:25:58,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +12: [2023-05-08 11:25:58,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +17: [2023-05-08 11:25:58,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 4: [2023-05-08 11:25:58,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 7: [2023-05-08 11:25:58,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 7: [2023-05-08 11:25:58,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 7: [2023-05-08 11:25:58,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 7: [2023-05-08 11:25:58,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +30: [2023-05-08 11:25:58,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +18: [2023-05-08 11:25:58,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:25:58,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +30: [2023-05-08 11:25:58,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +12: [2023-05-08 11:25:58,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:25:58,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:25:58,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:25:58,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +30: [2023-05-08 11:25:58,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +12: [2023-05-08 11:25:58,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 4: [2023-05-08 11:25:58,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 7: [2023-05-08 11:25:58,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +18: [2023-05-08 11:25:58,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:25:58,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 7: [2023-05-08 11:25:58,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +30: [2023-05-08 11:25:58,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:25:58,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 7: [2023-05-08 11:25:58,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +12: [2023-05-08 11:25:58,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +12: [2023-05-08 11:25:58,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +12: [2023-05-08 11:25:58,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +12: [2023-05-08 11:25:58,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +18: [2023-05-08 11:25:58,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +18: [2023-05-08 11:25:58,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +30: [2023-05-08 11:25:58,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +30: [2023-05-08 11:25:58,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +12: [2023-05-08 11:25:58,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +12: [2023-05-08 11:25:58,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +19: [2023-05-08 11:25:58,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +19: [2023-05-08 11:25:58,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +19: [2023-05-08 11:25:58,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +19: [2023-05-08 11:25:58,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 7: [2023-05-08 11:25:58,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +10: [2023-05-08 11:25:58,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +10: [2023-05-08 11:25:58,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +10: [2023-05-08 11:25:58,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +10: [2023-05-08 11:25:58,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 7: [2023-05-08 11:25:58,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 7: [2023-05-08 11:25:58,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 7: [2023-05-08 11:25:58,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:25:58,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +19: [2023-05-08 11:25:58,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:25:58,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +19: [2023-05-08 11:25:58,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +10: [2023-05-08 11:25:58,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +10: [2023-05-08 11:25:58,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +10: [2023-05-08 11:25:58,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +10: [2023-05-08 11:25:58,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:25:58,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +16: [2023-05-08 11:25:58,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +16: [2023-05-08 11:25:58,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +16: [2023-05-08 11:25:58,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +28: [2023-05-08 11:25:58,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +13: [2023-05-08 11:25:58,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +13: [2023-05-08 11:25:58,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +28: [2023-05-08 11:25:58,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +28: [2023-05-08 11:25:58,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +13: [2023-05-08 11:25:58,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +28: [2023-05-08 11:25:58,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +13: [2023-05-08 11:25:58,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +20: [2023-05-08 11:25:58,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +20: [2023-05-08 11:25:58,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +20: [2023-05-08 11:25:58,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +20: [2023-05-08 11:25:58,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 1: [2023-05-08 11:25:58,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 1: [2023-05-08 11:25:58,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 1: [2023-05-08 11:25:58,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +24: [2023-05-08 11:25:58,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 2: [2023-05-08 11:25:58,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +24: [2023-05-08 11:25:58,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +24: [2023-05-08 11:25:58,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 1: [2023-05-08 11:25:58,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 2: [2023-05-08 11:25:58,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 2: [2023-05-08 11:25:58,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +24: [2023-05-08 11:25:58,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +31: [2023-05-08 11:25:58,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +31: [2023-05-08 11:25:58,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +31: [2023-05-08 11:25:58,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +10: [2023-05-08 11:25:58,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:25:58,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 2: [2023-05-08 11:25:58,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +10: [2023-05-08 11:25:58,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +16: [2023-05-08 11:25:58,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +10: [2023-05-08 11:25:58,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +10: [2023-05-08 11:25:58,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +16: [2023-05-08 11:25:58,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +27: [2023-05-08 11:25:58,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +27: [2023-05-08 11:25:58,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +27: [2023-05-08 11:25:58,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +14: [2023-05-08 11:25:58,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +14: [2023-05-08 11:25:58,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +27: [2023-05-08 11:25:58,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +16: [2023-05-08 11:25:58,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:25:58,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:25:58,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +15: [2023-05-08 11:25:58,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +15: [2023-05-08 11:25:58,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +15: [2023-05-08 11:25:58,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +14: [2023-05-08 11:25:58,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +14: [2023-05-08 11:25:58,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +19: [2023-05-08 11:25:58,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +10: [2023-05-08 11:25:58,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +10: [2023-05-08 11:25:58,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:25:58,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +10: [2023-05-08 11:25:58,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +10: [2023-05-08 11:25:58,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +19: [2023-05-08 11:25:58,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:25:58,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +13: [2023-05-08 11:25:58,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +20: [2023-05-08 11:25:58,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +20: [2023-05-08 11:25:58,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +20: [2023-05-08 11:25:58,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +13: [2023-05-08 11:25:58,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +13: [2023-05-08 11:25:58,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +13: [2023-05-08 11:25:58,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +28: [2023-05-08 11:25:58,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:25:58,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +11: [2023-05-08 11:25:58,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +28: [2023-05-08 11:25:58,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:25:58,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +20: [2023-05-08 11:25:58,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:25:58,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +19: [2023-05-08 11:25:58,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:25:58,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +19: [2023-05-08 11:25:58,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +19: [2023-05-08 11:25:58,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +24: [2023-05-08 11:25:58,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +28: [2023-05-08 11:25:58,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +28: [2023-05-08 11:25:58,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:25:58,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:25:58,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +26: [2023-05-08 11:25:58,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +26: [2023-05-08 11:25:58,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +26: [2023-05-08 11:25:58,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +24: [2023-05-08 11:25:58,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +26: [2023-05-08 11:25:58,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 9: [2023-05-08 11:25:58,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 9: [2023-05-08 11:25:58,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 9: [2023-05-08 11:25:58,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +24: [2023-05-08 11:25:58,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 5: [2023-05-08 11:25:58,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 5: [2023-05-08 11:25:58,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +24: [2023-05-08 11:25:58,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 5: [2023-05-08 11:25:58,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 1: [2023-05-08 11:25:58,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 3: [2023-05-08 11:25:58,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 3: [2023-05-08 11:25:58,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 3: [2023-05-08 11:25:58,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 1: [2023-05-08 11:25:58,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:25:58,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:25:58,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:25:58,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 8: [2023-05-08 11:25:58,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. + 8: [2023-05-08 11:25:58,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +14: [2023-05-08 11:25:58,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +27: [2023-05-08 11:25:58,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt. +27: [2023-05-08 11:25:58,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +27: [2023-05-08 11:25:58,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +27: [2023-05-08 11:25:58,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:25:58,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:25:58,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:25:58,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:25:58,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 2: [2023-05-08 11:25:58,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:25:58,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +15: [2023-05-08 11:25:58,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +13: [2023-05-08 11:25:58,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +15: [2023-05-08 11:25:58,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +13: [2023-05-08 11:25:58,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +11: [2023-05-08 11:25:58,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:25:58,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +11: [2023-05-08 11:25:58,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +16: [2023-05-08 11:25:58,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +16: [2023-05-08 11:25:58,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +16: [2023-05-08 11:25:58,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +16: [2023-05-08 11:25:58,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +13: [2023-05-08 11:25:58,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:25:58,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +26: [2023-05-08 11:25:58,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +26: [2023-05-08 11:25:58,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +20: [2023-05-08 11:25:58,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +11: [2023-05-08 11:25:58,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +20: [2023-05-08 11:25:58,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +24: [2023-05-08 11:25:58,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +13: [2023-05-08 11:25:58,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:25:58,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +24: [2023-05-08 11:25:58,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +13: [2023-05-08 11:25:58,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +13: [2023-05-08 11:25:58,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 5: [2023-05-08 11:25:58,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +26: [2023-05-08 11:25:58,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +20: [2023-05-08 11:25:58,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:25:58,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +15: [2023-05-08 11:25:58,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +26: [2023-05-08 11:25:58,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +27: [2023-05-08 11:25:58,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:25:58,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +16: [2023-05-08 11:25:58,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +16: [2023-05-08 11:25:58,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +27: [2023-05-08 11:25:58,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 3: [2023-05-08 11:25:58,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +31: [2023-05-08 11:25:58,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +14: [2023-05-08 11:25:58,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +14: [2023-05-08 11:25:58,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt... +14: [2023-05-08 11:25:58,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +14: [2023-05-08 11:25:58,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 2: [2023-05-08 11:25:58,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:25:58,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +20: [2023-05-08 11:25:58,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +20: [2023-05-08 11:25:58,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +20: [2023-05-08 11:25:58,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 1: [2023-05-08 11:25:58,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 1: [2023-05-08 11:25:58,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +31: [2023-05-08 11:25:58,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:25:58,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 2: [2023-05-08 11:25:58,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 2: [2023-05-08 11:25:58,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +14: [2023-05-08 11:25:58,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +14: [2023-05-08 11:25:58,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +28: [2023-05-08 11:25:58,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +28: [2023-05-08 11:25:58,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +28: [2023-05-08 11:25:58,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +15: [2023-05-08 11:25:58,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +28: [2023-05-08 11:25:58,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 1: [2023-05-08 11:25:58,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +27: [2023-05-08 11:25:58,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:25:58,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +24: [2023-05-08 11:25:58,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:25:58,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +28: [2023-05-08 11:25:58,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +15: [2023-05-08 11:25:58,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +15: [2023-05-08 11:25:58,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +28: [2023-05-08 11:25:58,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +28: [2023-05-08 11:25:58,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +11: [2023-05-08 11:25:58,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +11: [2023-05-08 11:25:58,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +28: [2023-05-08 11:25:58,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 1: [2023-05-08 11:25:58,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +31: [2023-05-08 11:25:58,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +31: [2023-05-08 11:25:58,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +15: [2023-05-08 11:25:58,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +31: [2023-05-08 11:25:58,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +15: [2023-05-08 11:25:58,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +15: [2023-05-08 11:25:58,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +14: [2023-05-08 11:25:58,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +14: [2023-05-08 11:25:58,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +31: [2023-05-08 11:25:58,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +24: [2023-05-08 11:25:58,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:25:58,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +24: [2023-05-08 11:25:58,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +24: [2023-05-08 11:25:58,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +27: [2023-05-08 11:25:58,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:25:58,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +27: [2023-05-08 11:25:58,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +27: [2023-05-08 11:25:58,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +26: [2023-05-08 11:25:58,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:25:58,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:25:58,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +26: [2023-05-08 11:25:58,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +26: [2023-05-08 11:25:58,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:25:58,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 9: [2023-05-08 11:25:58,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +11: [2023-05-08 11:25:58,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +11: [2023-05-08 11:25:58,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +11: [2023-05-08 11:25:58,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +11: [2023-05-08 11:25:58,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 8: [2023-05-08 11:25:58,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 8: [2023-05-08 11:25:58,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 5: [2023-05-08 11:25:58,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 1: [2023-05-08 11:25:58,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 5: [2023-05-08 11:25:58,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 5: [2023-05-08 11:25:58,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +11: [2023-05-08 11:25:58,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:25:58,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +26: [2023-05-08 11:25:58,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +11: [2023-05-08 11:25:58,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 5: [2023-05-08 11:25:58,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +15: [2023-05-08 11:25:58,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. +15: [2023-05-08 11:25:58,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 3: [2023-05-08 11:25:58,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 9: [2023-05-08 11:25:58,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 9: [2023-05-08 11:25:58,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 3: [2023-05-08 11:25:58,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 3: [2023-05-08 11:25:58,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 9: [2023-05-08 11:25:58,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 8: [2023-05-08 11:25:58,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 5: [2023-05-08 11:25:58,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... + 3: [2023-05-08 11:25:58,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt... +27: [2023-05-08 11:25:58,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +27: [2023-05-08 11:25:58,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +27: [2023-05-08 11:25:58,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +27: [2023-05-08 11:25:58,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +27: [2023-05-08 11:25:58,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:25:58,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:25:58,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:25:58,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:25:58,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +27: [2023-05-08 11:25:58,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +27: [2023-05-08 11:25:58,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +27: [2023-05-08 11:25:58,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +27: [2023-05-08 11:25:58,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:25:58,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:25:58,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:25:58,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:25:58,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +29: [2023-05-08 11:25:58,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +29: [2023-05-08 11:25:58,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +29: [2023-05-08 11:25:58,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +29: [2023-05-08 11:25:58,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:25:58,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:25:58,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:25:58,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:25:58,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +21: [2023-05-08 11:25:58,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +21: [2023-05-08 11:25:58,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +23: [2023-05-08 11:25:58,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +21: [2023-05-08 11:25:58,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +23: [2023-05-08 11:25:58,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +23: [2023-05-08 11:25:58,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +23: [2023-05-08 11:25:58,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +23: [2023-05-08 11:25:58,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:25:58,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:25:58,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +23: [2023-05-08 11:25:58,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:25:58,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:25:58,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +23: [2023-05-08 11:25:58,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 0: [2023-05-08 11:25:58,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 0: [2023-05-08 11:25:58,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 0: [2023-05-08 11:25:58,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +23: [2023-05-08 11:25:58,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:25:58,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +25: [2023-05-08 11:25:58,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +25: [2023-05-08 11:25:58,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +25: [2023-05-08 11:25:58,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +25: [2023-05-08 11:25:58,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:25:58,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +10: [2023-05-08 11:25:58,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 6: [2023-05-08 11:25:58,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 6: [2023-05-08 11:25:58,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 6: [2023-05-08 11:25:58,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +10: [2023-05-08 11:25:58,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +10: [2023-05-08 11:25:58,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +10: [2023-05-08 11:25:58,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +10: [2023-05-08 11:25:58,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:25:58,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:25:58,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:25:58,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:25:58,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:25:58,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +30: [2023-05-08 11:25:58,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +30: [2023-05-08 11:25:58,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 7: [2023-05-08 11:25:58,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +30: [2023-05-08 11:25:58,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 7: [2023-05-08 11:25:58,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:25:58,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:25:58,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +30: [2023-05-08 11:25:58,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:25:58,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:25:58,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:25:58,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +14: [2023-05-08 11:25:58,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +14: [2023-05-08 11:25:58,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +14: [2023-05-08 11:25:58,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +14: [2023-05-08 11:25:58,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +14: [2023-05-08 11:25:58,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +14: [2023-05-08 11:25:58,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +14: [2023-05-08 11:25:58,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +14: [2023-05-08 11:25:58,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:25:58,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:25:58,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:25:58,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:25:58,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:25:58,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:25:58,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:25:58,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +13: [2023-05-08 11:25:58,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +13: [2023-05-08 11:25:58,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:25:58,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:25:58,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +10: [2023-05-08 11:25:58,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:25:58,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:25:58,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:25:58,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:25:58,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:25:58,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:25:58,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +10: [2023-05-08 11:25:58,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:25:58,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:25:58,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:25:58,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +14: [2023-05-08 11:25:58,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:25:58,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +10: [2023-05-08 11:25:58,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:25:58,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +14: [2023-05-08 11:25:58,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:25:58,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +17: [2023-05-08 11:25:58,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +17: [2023-05-08 11:25:58,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +17: [2023-05-08 11:25:58,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +15: [2023-05-08 11:25:58,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +23: [2023-05-08 11:25:58,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:25:58,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +23: [2023-05-08 11:25:58,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:25:58,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +23: [2023-05-08 11:25:58,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:25:58,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:25:58,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:25:58,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:25:58,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:25:58,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +14: [2023-05-08 11:25:58,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:25:58,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +23: [2023-05-08 11:25:58,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +23: [2023-05-08 11:25:58,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +23: [2023-05-08 11:25:58,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +23: [2023-05-08 11:25:58,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +23: [2023-05-08 11:25:58,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +23: [2023-05-08 11:25:58,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +14: [2023-05-08 11:25:58,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:25:58,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:25:58,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:25:58,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:25:58,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:25:58,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:25:58,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:25:58,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:25:58,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:25:58,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:25:58,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +23: [2023-05-08 11:25:58,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:25:58,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:25:58,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:25:58,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:25:58,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:25:58,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:25:58,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +22: [2023-05-08 11:25:58,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +22: [2023-05-08 11:25:58,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +22: [2023-05-08 11:25:58,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +22: [2023-05-08 11:25:58,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +28: [2023-05-08 11:25:58,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +13: [2023-05-08 11:25:58,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:25:58,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +13: [2023-05-08 11:25:58,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:25:58,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +23: [2023-05-08 11:25:58,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:25:58,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:25:58,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:25:58,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:25:58,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:25:58,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:25:58,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +23: [2023-05-08 11:25:58,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:25:58,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:25:58,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:25:58,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:25:58,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:25:58,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +23: [2023-05-08 11:25:58,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:25:58,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:25:58,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:25:58,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:25:58,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +19: [2023-05-08 11:25:58,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:25:58,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:25:58,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 0: [2023-05-08 11:25:58,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 0: [2023-05-08 11:25:58,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:25:58,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +17: [2023-05-08 11:25:58,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:25:58,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:25:58,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:25:58,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:25:58,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:25:58,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:25:58,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +17: [2023-05-08 11:25:58,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +28: [2023-05-08 11:25:58,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:25:58,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:25:58,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +21: [2023-05-08 11:25:58,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +21: [2023-05-08 11:25:58,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +21: [2023-05-08 11:25:58,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +21: [2023-05-08 11:25:58,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +21: [2023-05-08 11:25:58,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +21: [2023-05-08 11:25:58,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +21: [2023-05-08 11:25:58,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +21: [2023-05-08 11:25:58,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 0: [2023-05-08 11:25:58,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:25:58,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +21: [2023-05-08 11:25:58,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:25:58,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 9: [2023-05-08 11:25:58,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:25:58,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +21: [2023-05-08 11:25:58,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +17: [2023-05-08 11:25:58,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +21: [2023-05-08 11:25:58,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +28: [2023-05-08 11:25:58,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:25:58,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:25:58,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:25:58,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +17: [2023-05-08 11:25:58,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:25:58,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +21: [2023-05-08 11:25:58,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:25:58,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +23: [2023-05-08 11:25:58,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +23: [2023-05-08 11:25:58,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:25:58,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:25:58,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +28: [2023-05-08 11:25:58,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:25:58,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:25:58,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:25:58,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +11: [2023-05-08 11:25:58,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +11: [2023-05-08 11:25:58,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +11: [2023-05-08 11:25:58,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +11: [2023-05-08 11:25:58,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +11: [2023-05-08 11:25:58,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +17: [2023-05-08 11:25:58,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +11: [2023-05-08 11:25:58,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +11: [2023-05-08 11:25:58,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +11: [2023-05-08 11:25:58,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +10: [2023-05-08 11:25:58,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +10: [2023-05-08 11:25:58,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +10: [2023-05-08 11:25:58,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +17: [2023-05-08 11:25:58,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +10: [2023-05-08 11:25:58,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +28: [2023-05-08 11:25:58,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +28: [2023-05-08 11:25:58,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +28: [2023-05-08 11:25:58,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +19: [2023-05-08 11:25:58,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +21: [2023-05-08 11:25:58,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:25:58,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +16: [2023-05-08 11:25:58,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +16: [2023-05-08 11:25:58,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +16: [2023-05-08 11:25:58,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +19: [2023-05-08 11:25:58,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:25:58,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:25:58,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:25:58,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:25:58,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:25:58,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +11: [2023-05-08 11:25:58,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +10: [2023-05-08 11:25:58,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:25:58,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:25:58,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:25:58,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:25:58,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:25:58,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +21: [2023-05-08 11:25:58,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:25:58,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:25:58,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:25:58,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +16: [2023-05-08 11:25:58,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +21: [2023-05-08 11:25:58,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:25:58,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:25:58,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:25:58,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:25:58,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +17: [2023-05-08 11:25:58,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +21: [2023-05-08 11:25:58,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +21: [2023-05-08 11:25:58,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +19: [2023-05-08 11:25:58,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:25:58,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +11: [2023-05-08 11:25:58,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +18: [2023-05-08 11:25:58,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +18: [2023-05-08 11:25:58,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +18: [2023-05-08 11:25:58,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +18: [2023-05-08 11:25:58,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +21: [2023-05-08 11:25:58,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:25:58,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:25:58,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +19: [2023-05-08 11:25:58,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +19: [2023-05-08 11:25:58,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +19: [2023-05-08 11:25:58,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 0: [2023-05-08 11:25:58,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:25:58,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +14: [2023-05-08 11:25:58,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +14: [2023-05-08 11:25:58,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +14: [2023-05-08 11:25:58,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +17: [2023-05-08 11:25:58,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:25:58,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +24: [2023-05-08 11:25:58,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +27: [2023-05-08 11:25:58,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +27: [2023-05-08 11:25:58,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +27: [2023-05-08 11:25:58,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +24: [2023-05-08 11:25:58,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +24: [2023-05-08 11:25:58,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +27: [2023-05-08 11:25:58,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +24: [2023-05-08 11:25:58,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +24: [2023-05-08 11:25:58,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +24: [2023-05-08 11:25:58,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +20: [2023-05-08 11:25:58,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:25:58,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:25:58,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:25:58,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:25:58,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:25:58,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:25:58,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +20: [2023-05-08 11:25:58,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:25:58,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:25:58,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:25:58,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:25:58,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:25:58,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:25:58,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:25:58,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:25:58,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +29: [2023-05-08 11:25:58,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +29: [2023-05-08 11:25:58,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +29: [2023-05-08 11:25:58,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +20: [2023-05-08 11:25:58,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +18: [2023-05-08 11:25:58,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +18: [2023-05-08 11:25:58,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:25:58,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +11: [2023-05-08 11:25:58,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +19: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +24: [2023-05-08 11:25:58,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:25:58,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:25:58,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:25:58,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +20: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +24: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +31: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +31: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +19: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +24: [2023-05-08 11:25:58,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:25:58,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +17: [2023-05-08 11:25:58,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:25:58,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +31: [2023-05-08 11:25:58,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +18: [2023-05-08 11:25:58,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 7: [2023-05-08 11:25:58,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 7: [2023-05-08 11:25:58,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +31: [2023-05-08 11:25:58,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:25:58,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +19: [2023-05-08 11:25:58,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +18: [2023-05-08 11:25:58,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:25:58,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +24: [2023-05-08 11:25:58,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +24: [2023-05-08 11:25:58,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:25:58,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +10: [2023-05-08 11:25:58,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +24: [2023-05-08 11:25:58,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +14: [2023-05-08 11:25:58,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +14: [2023-05-08 11:25:58,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:25:58,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +10: [2023-05-08 11:25:58,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +24: [2023-05-08 11:25:58,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +14: [2023-05-08 11:25:58,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +10: [2023-05-08 11:25:58,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +14: [2023-05-08 11:25:58,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +24: [2023-05-08 11:25:58,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:25:58,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +24: [2023-05-08 11:25:58,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:25:58,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:25:58,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +27: [2023-05-08 11:25:58,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +24: [2023-05-08 11:25:58,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:25:58,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:25:58,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:25:58,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:25:58,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:25:58,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +31: [2023-05-08 11:25:58,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +24: [2023-05-08 11:25:58,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:25:58,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +20: [2023-05-08 11:25:58,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:25:58,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:25:58,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:25:58,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:25:58,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:25:58,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +19: [2023-05-08 11:25:58,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:25:58,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +14: [2023-05-08 11:25:58,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:25:58,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:25:58,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 7: [2023-05-08 11:25:58,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:25:58,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:25:58,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +24: [2023-05-08 11:25:58,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:25:58,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +29: [2023-05-08 11:25:58,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:25:58,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:25:58,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +27: [2023-05-08 11:25:58,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:25:58,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:25:58,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:25:58,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:25:58,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +20: [2023-05-08 11:25:58,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:25:58,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +16: [2023-05-08 11:25:58,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +16: [2023-05-08 11:25:58,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +16: [2023-05-08 11:25:58,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:25:58,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:25:58,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:25:58,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:25:58,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:25:58,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +16: [2023-05-08 11:25:58,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +26: [2023-05-08 11:25:58,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:25:58,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:25:58,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +31: [2023-05-08 11:25:58,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:25:58,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +16: [2023-05-08 11:25:58,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:25:58,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:25:58,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 1: [2023-05-08 11:25:58,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:25:58,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 1: [2023-05-08 11:25:58,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 8: [2023-05-08 11:25:58,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 8: [2023-05-08 11:25:58,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 8: [2023-05-08 11:25:58,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 8: [2023-05-08 11:25:58,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +14: [2023-05-08 11:25:58,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +14: [2023-05-08 11:25:58,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:25:58,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:25:58,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +14: [2023-05-08 11:25:58,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +20: [2023-05-08 11:25:58,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:25:58,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +14: [2023-05-08 11:25:58,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +27: [2023-05-08 11:25:58,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:25:58,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +24: [2023-05-08 11:25:58,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +24: [2023-05-08 11:25:58,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +19: [2023-05-08 11:25:58,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +24: [2023-05-08 11:25:58,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +24: [2023-05-08 11:25:58,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +14: [2023-05-08 11:25:58,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +27: [2023-05-08 11:25:58,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 4: [2023-05-08 11:25:58,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 4: [2023-05-08 11:25:58,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 4: [2023-05-08 11:25:58,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +31: [2023-05-08 11:25:58,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:25:58,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:25:58,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +24: [2023-05-08 11:25:58,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:25:58,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +14: [2023-05-08 11:25:58,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 8: [2023-05-08 11:25:58,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +24: [2023-05-08 11:25:58,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +24: [2023-05-08 11:25:58,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 3: [2023-05-08 11:25:58,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +27: [2023-05-08 11:25:58,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +29: [2023-05-08 11:25:58,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:25:58,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:25:58,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +29: [2023-05-08 11:25:58,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +20: [2023-05-08 11:25:58,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +20: [2023-05-08 11:25:58,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 1: [2023-05-08 11:25:58,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 1: [2023-05-08 11:25:58,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 4: [2023-05-08 11:25:58,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 4: [2023-05-08 11:25:58,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +12: [2023-05-08 11:25:58,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 7: [2023-05-08 11:25:58,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:25:58,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +12: [2023-05-08 11:25:58,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +12: [2023-05-08 11:25:58,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +16: [2023-05-08 11:25:58,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +24: [2023-05-08 11:25:58,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +24: [2023-05-08 11:25:58,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +31: [2023-05-08 11:25:58,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +31: [2023-05-08 11:25:58,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +31: [2023-05-08 11:25:58,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 7: [2023-05-08 11:25:58,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +24: [2023-05-08 11:25:58,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:25:58,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +16: [2023-05-08 11:25:58,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +24: [2023-05-08 11:25:58,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:25:58,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +24: [2023-05-08 11:25:58,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 1: [2023-05-08 11:25:58,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 1: [2023-05-08 11:25:58,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +12: [2023-05-08 11:25:58,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +16: [2023-05-08 11:25:58,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +16: [2023-05-08 11:25:58,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +24: [2023-05-08 11:25:58,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +24: [2023-05-08 11:25:58,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +12: [2023-05-08 11:25:58,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:25:58,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:25:58,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +13: [2023-05-08 11:25:58,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +13: [2023-05-08 11:25:58,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +13: [2023-05-08 11:25:58,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +13: [2023-05-08 11:25:58,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +20: [2023-05-08 11:25:58,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 1: [2023-05-08 11:25:58,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 5: [2023-05-08 11:25:58,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +20: [2023-05-08 11:25:58,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +20: [2023-05-08 11:25:58,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +31: [2023-05-08 11:25:58,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +20: [2023-05-08 11:25:58,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +26: [2023-05-08 11:25:58,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 3: [2023-05-08 11:25:58,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:25:58,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:25:58,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +26: [2023-05-08 11:25:58,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +26: [2023-05-08 11:25:58,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 5: [2023-05-08 11:25:58,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +28: [2023-05-08 11:25:58,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +28: [2023-05-08 11:25:58,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +28: [2023-05-08 11:25:58,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +28: [2023-05-08 11:25:58,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 3: [2023-05-08 11:25:58,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +31: [2023-05-08 11:25:58,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:25:58,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +15: [2023-05-08 11:25:58,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +11: [2023-05-08 11:25:58,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +11: [2023-05-08 11:25:58,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +11: [2023-05-08 11:25:58,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +15: [2023-05-08 11:25:58,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +15: [2023-05-08 11:25:58,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 2: [2023-05-08 11:25:58,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 2: [2023-05-08 11:25:58,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 2: [2023-05-08 11:25:58,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 2: [2023-05-08 11:25:58,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 4: [2023-05-08 11:25:58,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +11: [2023-05-08 11:25:58,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 4: [2023-05-08 11:25:58,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +16: [2023-05-08 11:25:58,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:25:58,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +13: [2023-05-08 11:25:58,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:25:58,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +20: [2023-05-08 11:25:58,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +11: [2023-05-08 11:25:58,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:25:58,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:25:58,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:25:58,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:25:58,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:25:58,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +25: [2023-05-08 11:25:58,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +25: [2023-05-08 11:25:58,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +25: [2023-05-08 11:25:58,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +25: [2023-05-08 11:25:58,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +25: [2023-05-08 11:25:58,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:25:58,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +25: [2023-05-08 11:25:58,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +11: [2023-05-08 11:25:58,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:25:58,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:25:58,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:25:58,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:25:58,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:25:58,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:25:58,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +13: [2023-05-08 11:25:58,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +13: [2023-05-08 11:25:58,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 5: [2023-05-08 11:25:58,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +11: [2023-05-08 11:25:58,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:25:58,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +26: [2023-05-08 11:25:58,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +11: [2023-05-08 11:25:58,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +20: [2023-05-08 11:25:58,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:25:58,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 1: [2023-05-08 11:25:58,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 1: [2023-05-08 11:25:58,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +28: [2023-05-08 11:25:58,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:25:58,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +25: [2023-05-08 11:25:58,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +28: [2023-05-08 11:25:58,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +15: [2023-05-08 11:25:58,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +31: [2023-05-08 11:25:58,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +31: [2023-05-08 11:25:58,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +25: [2023-05-08 11:25:58,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +25: [2023-05-08 11:25:58,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:25:58,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:25:58,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:25:58,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +30: [2023-05-08 11:25:58,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +25: [2023-05-08 11:25:58,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +25: [2023-05-08 11:25:58,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +30: [2023-05-08 11:25:58,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +30: [2023-05-08 11:25:58,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +30: [2023-05-08 11:25:58,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +30: [2023-05-08 11:25:58,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +30: [2023-05-08 11:25:58,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +30: [2023-05-08 11:25:58,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +30: [2023-05-08 11:25:58,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:25:58,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 6: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 8: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 2: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:25:58,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +22: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +22: [2023-05-08 11:25:58,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +22: [2023-05-08 11:25:58,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:25:58,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:25:58,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +22: [2023-05-08 11:25:58,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +22: [2023-05-08 11:25:58,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +30: [2023-05-08 11:25:58,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 2: [2023-05-08 11:25:58,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:25:58,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +22: [2023-05-08 11:25:58,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +30: [2023-05-08 11:25:58,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:25:58,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 6: [2023-05-08 11:25:58,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 6: [2023-05-08 11:25:58,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 6: [2023-05-08 11:25:58,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +22: [2023-05-08 11:25:58,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 8: [2023-05-08 11:25:58,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 6: [2023-05-08 11:25:58,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 6: [2023-05-08 11:25:58,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:25:58,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +18: [2023-05-08 11:25:58,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +18: [2023-05-08 11:25:58,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +18: [2023-05-08 11:25:58,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +18: [2023-05-08 11:25:58,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +18: [2023-05-08 11:25:58,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:25:58,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +18: [2023-05-08 11:25:58,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +18: [2023-05-08 11:25:58,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +22: [2023-05-08 11:25:58,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +18: [2023-05-08 11:25:58,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +18: [2023-05-08 11:25:58,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +18: [2023-05-08 11:25:58,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +28: [2023-05-08 11:25:58,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +22: [2023-05-08 11:25:58,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +26: [2023-05-08 11:25:58,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 4: [2023-05-08 11:25:58,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +11: [2023-05-08 11:25:58,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:25:58,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +22: [2023-05-08 11:25:58,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +20: [2023-05-08 11:25:58,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:25:58,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +11: [2023-05-08 11:25:58,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +11: [2023-05-08 11:25:58,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +11: [2023-05-08 11:25:58,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +18: [2023-05-08 11:25:58,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +18: [2023-05-08 11:25:58,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +18: [2023-05-08 11:25:58,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +18: [2023-05-08 11:25:58,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +22: [2023-05-08 11:25:58,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +18: [2023-05-08 11:25:58,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +18: [2023-05-08 11:25:58,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 2: [2023-05-08 11:25:58,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +31: [2023-05-08 11:25:58,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +31: [2023-05-08 11:25:58,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +31: [2023-05-08 11:25:58,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:25:58,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +15: [2023-05-08 11:25:58,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 5: [2023-05-08 11:25:58,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +31: [2023-05-08 11:25:58,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +25: [2023-05-08 11:25:58,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:25:58,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +30: [2023-05-08 11:25:58,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 5: [2023-05-08 11:25:58,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 9: [2023-05-08 11:25:58,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 9: [2023-05-08 11:25:58,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 6: [2023-05-08 11:25:58,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:25:58,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 5: [2023-05-08 11:25:58,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +15: [2023-05-08 11:25:58,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +15: [2023-05-08 11:25:58,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 5: [2023-05-08 11:25:58,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 5: [2023-05-08 11:25:58,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +26: [2023-05-08 11:25:58,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:25:58,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:25:58,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:25:58,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +27: [2023-05-08 11:25:58,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +27: [2023-05-08 11:25:58,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +27: [2023-05-08 11:25:58,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +27: [2023-05-08 11:25:58,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:25:58,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:25:58,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +27: [2023-05-08 11:25:58,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:25:58,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +27: [2023-05-08 11:25:58,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:25:58,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 5: [2023-05-08 11:25:58,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:25:58,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +26: [2023-05-08 11:25:58,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +22: [2023-05-08 11:25:58,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +22: [2023-05-08 11:25:58,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:25:58,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 3: [2023-05-08 11:25:58,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 3: [2023-05-08 11:25:58,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. + 3: [2023-05-08 11:25:58,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +20: [2023-05-08 11:25:58,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt. +13: [2023-05-08 11:25:58,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +11: [2023-05-08 11:25:58,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +20: [2023-05-08 11:25:58,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:25:58,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:25:58,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 6: [2023-05-08 11:25:58,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +20: [2023-05-08 11:25:58,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 9: [2023-05-08 11:25:58,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +11: [2023-05-08 11:25:58,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +13: [2023-05-08 11:25:58,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +11: [2023-05-08 11:25:58,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +20: [2023-05-08 11:25:58,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +30: [2023-05-08 11:25:58,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +13: [2023-05-08 11:25:58,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +25: [2023-05-08 11:25:58,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +25: [2023-05-08 11:25:58,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:25:58,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +15: [2023-05-08 11:25:58,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:25:58,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +28: [2023-05-08 11:25:58,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:25:58,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +28: [2023-05-08 11:25:58,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:25:58,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 8: [2023-05-08 11:25:58,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +28: [2023-05-08 11:25:58,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 5: [2023-05-08 11:25:58,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +28: [2023-05-08 11:25:58,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +28: [2023-05-08 11:25:58,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +30: [2023-05-08 11:25:58,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +18: [2023-05-08 11:25:58,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +18: [2023-05-08 11:25:58,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +30: [2023-05-08 11:25:58,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +27: [2023-05-08 11:25:58,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +27: [2023-05-08 11:25:58,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +22: [2023-05-08 11:25:58,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:25:58,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:25:58,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:25:58,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:25:58,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +12: [2023-05-08 11:25:58,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:25:58,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:25:58,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:25:58,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:25:58,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:25:58,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +12: [2023-05-08 11:25:58,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +12: [2023-05-08 11:25:58,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:25:58,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:25:58,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +18: [2023-05-08 11:25:58,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +18: [2023-05-08 11:25:58,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +25: [2023-05-08 11:25:58,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +27: [2023-05-08 11:25:58,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +27: [2023-05-08 11:25:58,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:25:58,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +25: [2023-05-08 11:25:58,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:25:58,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +30: [2023-05-08 11:25:58,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 9: [2023-05-08 11:25:58,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +22: [2023-05-08 11:25:58,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:25:58,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +12: [2023-05-08 11:25:58,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +12: [2023-05-08 11:25:58,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 2: [2023-05-08 11:25:58,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +12: [2023-05-08 11:25:58,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt... +30: [2023-05-08 11:25:58,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 6: [2023-05-08 11:25:58,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +30: [2023-05-08 11:25:58,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:25:58,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +18: [2023-05-08 11:25:58,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +18: [2023-05-08 11:25:58,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:25:58,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:25:58,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +18: [2023-05-08 11:25:58,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +18: [2023-05-08 11:25:58,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:25:58,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:25:58,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +25: [2023-05-08 11:25:58,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:25:58,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +22: [2023-05-08 11:25:58,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:25:58,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 9: [2023-05-08 11:25:58,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 3: [2023-05-08 11:25:58,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +22: [2023-05-08 11:25:58,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +12: [2023-05-08 11:25:58,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:25:58,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +10: [2023-05-08 11:25:58,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +10: [2023-05-08 11:25:58,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +10: [2023-05-08 11:25:58,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:25:58,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +10: [2023-05-08 11:25:58,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:25:58,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:25:58,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +10: [2023-05-08 11:25:58,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... + 3: [2023-05-08 11:25:58,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt... +12: [2023-05-08 11:25:58,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:25:58,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +12: [2023-05-08 11:25:58,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt. +10: [2023-05-08 11:25:58,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +10: [2023-05-08 11:25:58,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +10: [2023-05-08 11:25:58,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +10: [2023-05-08 11:25:58,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 7: [2023-05-08 11:25:58,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +12: [2023-05-08 11:25:58,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:25:58,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:25:58,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +10: [2023-05-08 11:25:58,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +10: [2023-05-08 11:25:58,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +10: [2023-05-08 11:25:58,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:25:58,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +14: [2023-05-08 11:25:58,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:25:58,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +14: [2023-05-08 11:25:58,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +14: [2023-05-08 11:25:58,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:25:58,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:25:58,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +14: [2023-05-08 11:25:58,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +23: [2023-05-08 11:25:58,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +14: [2023-05-08 11:25:58,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +23: [2023-05-08 11:25:58,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +23: [2023-05-08 11:25:58,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +23: [2023-05-08 11:25:58,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +14: [2023-05-08 11:25:58,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +14: [2023-05-08 11:25:58,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +14: [2023-05-08 11:25:58,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:25:58,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +17: [2023-05-08 11:25:58,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +17: [2023-05-08 11:25:58,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +17: [2023-05-08 11:25:58,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +23: [2023-05-08 11:25:58,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +23: [2023-05-08 11:25:58,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +23: [2023-05-08 11:25:58,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +23: [2023-05-08 11:25:58,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:25:58,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:25:58,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +21: [2023-05-08 11:25:58,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +21: [2023-05-08 11:25:58,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +21: [2023-05-08 11:25:58,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +14: [2023-05-08 11:25:58,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:25:58,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:25:58,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:25:58,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:25:58,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:25:58,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +21: [2023-05-08 11:25:58,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +21: [2023-05-08 11:25:58,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +21: [2023-05-08 11:25:58,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:25:58,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +21: [2023-05-08 11:25:58,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:25:58,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:25:58,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:25:58,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:25:58,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:25:58,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:25:58,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:25:58,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:25:58,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +23: [2023-05-08 11:25:58,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +23: [2023-05-08 11:25:58,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +23: [2023-05-08 11:25:58,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +23: [2023-05-08 11:25:58,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:25:58,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +23: [2023-05-08 11:25:58,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +23: [2023-05-08 11:25:58,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +23: [2023-05-08 11:25:58,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +23: [2023-05-08 11:25:58,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +23: [2023-05-08 11:25:58,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +23: [2023-05-08 11:25:58,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +23: [2023-05-08 11:25:58,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +23: [2023-05-08 11:25:58,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +23: [2023-05-08 11:25:58,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +23: [2023-05-08 11:25:58,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +23: [2023-05-08 11:25:58,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +23: [2023-05-08 11:25:58,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:25:58,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:25:58,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:25:58,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:25:58,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:25:58,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:25:58,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:25:58,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:25:58,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:25:58,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:25:58,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:25:58,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +28: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +28: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +28: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:25:58,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:25:58,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:25:58,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:25:58,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:25:58,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:25:58,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:25:58,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +19: [2023-05-08 11:25:58,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:25:58,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:25:58,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:25:58,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:25:58,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:25:58,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +19: [2023-05-08 11:25:58,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:25:58,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:25:58,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:25:58,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +23: [2023-05-08 11:25:58,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:25:58,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:25:58,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:25:58,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:25:58,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:25:58,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:25:58,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:25:58,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:25:58,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:25:58,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:25:58,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +17: [2023-05-08 11:25:58,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:25:58,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +17: [2023-05-08 11:25:58,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +17: [2023-05-08 11:25:58,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:25:58,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +17: [2023-05-08 11:25:58,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +21: [2023-05-08 11:25:58,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +21: [2023-05-08 11:25:58,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +21: [2023-05-08 11:25:58,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +21: [2023-05-08 11:25:58,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +21: [2023-05-08 11:25:58,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +21: [2023-05-08 11:25:58,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +21: [2023-05-08 11:25:58,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +21: [2023-05-08 11:25:58,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +21: [2023-05-08 11:25:58,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +21: [2023-05-08 11:25:58,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +21: [2023-05-08 11:25:58,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +21: [2023-05-08 11:25:58,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +21: [2023-05-08 11:25:58,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +21: [2023-05-08 11:25:58,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +21: [2023-05-08 11:25:58,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +21: [2023-05-08 11:25:58,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +11: [2023-05-08 11:25:58,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:25:58,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:25:58,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:25:58,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:25:58,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:25:58,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:25:58,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +24: [2023-05-08 11:25:58,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +24: [2023-05-08 11:25:58,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:25:58,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:25:58,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +24: [2023-05-08 11:25:58,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:25:58,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:25:58,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +23: [2023-05-08 11:25:58,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +23: [2023-05-08 11:25:58,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:25:58,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:25:58,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +23: [2023-05-08 11:25:58,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:25:58,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:25:58,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:25:58,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:25:58,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:25:58,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:25:58,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:25:58,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:25:58,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:25:58,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:25:58,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:25:58,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:25:58,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +23: [2023-05-08 11:25:58,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +23: [2023-05-08 11:25:58,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:25:58,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:25:58,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:25:58,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +13: [2023-05-08 11:25:58,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:25:58,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:25:58,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:25:58,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:25:58,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:25:58,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 9: [2023-05-08 11:25:58,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:25:58,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:25:58,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +21: [2023-05-08 11:25:58,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +21: [2023-05-08 11:25:58,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +21: [2023-05-08 11:25:58,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +23: [2023-05-08 11:25:58,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +10: [2023-05-08 11:25:58,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +10: [2023-05-08 11:25:58,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +10: [2023-05-08 11:25:58,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +10: [2023-05-08 11:25:58,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +15: [2023-05-08 11:25:58,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:25:58,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:25:58,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:25:58,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:25:58,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:25:58,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:25:58,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:25:58,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:25:58,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:25:58,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:25:58,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:25:58,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:25:58,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:25:58,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +17: [2023-05-08 11:25:58,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:25:58,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:25:58,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:25:58,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:25:58,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:25:58,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:25:58,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +10: [2023-05-08 11:25:58,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:25:58,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +10: [2023-05-08 11:25:58,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:25:58,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:25:58,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +21: [2023-05-08 11:25:58,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:25:58,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:25:58,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:25:58,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:25:58,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:25:58,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:25:58,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +15: [2023-05-08 11:25:58,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:25:58,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +15: [2023-05-08 11:25:58,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 0: [2023-05-08 11:25:58,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 0: [2023-05-08 11:25:58,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +24: [2023-05-08 11:25:58,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:25:58,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +24: [2023-05-08 11:25:58,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:25:58,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +10: [2023-05-08 11:25:58,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +10: [2023-05-08 11:25:58,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +10: [2023-05-08 11:25:58,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +10: [2023-05-08 11:25:58,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +10: [2023-05-08 11:25:58,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +10: [2023-05-08 11:25:58,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +10: [2023-05-08 11:25:58,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 6: [2023-05-08 11:25:58,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +19: [2023-05-08 11:25:58,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +19: [2023-05-08 11:25:58,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +19: [2023-05-08 11:25:58,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +19: [2023-05-08 11:25:58,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 0: [2023-05-08 11:25:58,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:25:58,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +14: [2023-05-08 11:25:58,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +14: [2023-05-08 11:25:58,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +14: [2023-05-08 11:25:58,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 3: [2023-05-08 11:25:58,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:25:58,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:25:58,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +27: [2023-05-08 11:25:58,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +27: [2023-05-08 11:25:58,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +27: [2023-05-08 11:25:58,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +27: [2023-05-08 11:25:58,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +19: [2023-05-08 11:25:58,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +19: [2023-05-08 11:25:58,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +19: [2023-05-08 11:25:58,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +19: [2023-05-08 11:25:58,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:25:58,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:25:58,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:25:58,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:25:58,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +20: [2023-05-08 11:25:58,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +20: [2023-05-08 11:25:58,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +20: [2023-05-08 11:25:58,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +14: [2023-05-08 11:25:58,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +20: [2023-05-08 11:25:58,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:25:58,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +14: [2023-05-08 11:25:58,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:25:58,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 6: [2023-05-08 11:25:58,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 5: [2023-05-08 11:25:58,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:25:58,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +16: [2023-05-08 11:25:58,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +27: [2023-05-08 11:25:58,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 5: [2023-05-08 11:25:58,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:25:58,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:25:58,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:25:58,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:25:58,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +16: [2023-05-08 11:25:58,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:25:58,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:25:58,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:25:58,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:25:58,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +16: [2023-05-08 11:25:58,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:25:58,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +27: [2023-05-08 11:25:58,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 0: [2023-05-08 11:25:58,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +27: [2023-05-08 11:25:58,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:25:58,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +29: [2023-05-08 11:25:58,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +29: [2023-05-08 11:25:58,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +29: [2023-05-08 11:25:58,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 0: [2023-05-08 11:25:58,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 0: [2023-05-08 11:25:58,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 0: [2023-05-08 11:25:58,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 0: [2023-05-08 11:25:58,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 5: [2023-05-08 11:25:58,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:25:58,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:25:58,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 6: [2023-05-08 11:25:58,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 6: [2023-05-08 11:25:58,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:25:58,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +14: [2023-05-08 11:25:58,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +14: [2023-05-08 11:25:58,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +14: [2023-05-08 11:25:58,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 5: [2023-05-08 11:25:58,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +31: [2023-05-08 11:25:58,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +31: [2023-05-08 11:25:58,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +31: [2023-05-08 11:25:58,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:25:58,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:25:58,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:25:58,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +19: [2023-05-08 11:25:58,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:25:58,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:25:58,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +29: [2023-05-08 11:25:58,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:25:58,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:25:58,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:25:58,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 5: [2023-05-08 11:25:58,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:25:58,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:25:58,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:25:58,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +31: [2023-05-08 11:25:58,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:25:58,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:25:58,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:25:58,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:25:58,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +14: [2023-05-08 11:25:58,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +14: [2023-05-08 11:25:58,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +27: [2023-05-08 11:25:58,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +14: [2023-05-08 11:25:58,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +14: [2023-05-08 11:25:58,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +26: [2023-05-08 11:25:58,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:25:58,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:25:58,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:25:58,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:25:58,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:25:58,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:25:58,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:25:58,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +19: [2023-05-08 11:25:58,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +19: [2023-05-08 11:25:58,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 1: [2023-05-08 11:25:58,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +24: [2023-05-08 11:25:58,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +24: [2023-05-08 11:25:58,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +18: [2023-05-08 11:25:58,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +18: [2023-05-08 11:25:58,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +18: [2023-05-08 11:25:58,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +27: [2023-05-08 11:25:58,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +27: [2023-05-08 11:25:58,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +27: [2023-05-08 11:25:58,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +27: [2023-05-08 11:25:58,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +18: [2023-05-08 11:25:58,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +18: [2023-05-08 11:25:58,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +18: [2023-05-08 11:25:58,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +18: [2023-05-08 11:25:58,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +18: [2023-05-08 11:25:58,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:25:58,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:25:58,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:25:58,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:25:58,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:25:58,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:25:58,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +24: [2023-05-08 11:25:58,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +22: [2023-05-08 11:25:58,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:25:58,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +24: [2023-05-08 11:25:58,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +22: [2023-05-08 11:25:58,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:25:58,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:25:58,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:25:58,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +12: [2023-05-08 11:25:58,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +12: [2023-05-08 11:25:58,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:25:58,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:25:58,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:25:58,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +12: [2023-05-08 11:25:58,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:25:58,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +12: [2023-05-08 11:25:58,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +16: [2023-05-08 11:25:58,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:25:58,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +30: [2023-05-08 11:25:58,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +30: [2023-05-08 11:25:58,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +30: [2023-05-08 11:25:58,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +30: [2023-05-08 11:25:58,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +30: [2023-05-08 11:25:58,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +30: [2023-05-08 11:25:58,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +30: [2023-05-08 11:25:58,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +30: [2023-05-08 11:25:58,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 4: [2023-05-08 11:25:58,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +27: [2023-05-08 11:25:58,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +27: [2023-05-08 11:25:58,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 2: [2023-05-08 11:25:58,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +24: [2023-05-08 11:25:58,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:25:58,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:25:58,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +31: [2023-05-08 11:25:58,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +29: [2023-05-08 11:25:58,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 4: [2023-05-08 11:25:58,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +26: [2023-05-08 11:25:58,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:25:58,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:25:58,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:25:58,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +24: [2023-05-08 11:25:58,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:25:58,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:25:58,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +20: [2023-05-08 11:25:58,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:25:58,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:25:58,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +31: [2023-05-08 11:25:58,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:25:58,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +29: [2023-05-08 11:25:58,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +29: [2023-05-08 11:25:58,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +26: [2023-05-08 11:25:58,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:25:58,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +26: [2023-05-08 11:25:58,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:25:58,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:25:58,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:25:58,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +20: [2023-05-08 11:25:58,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +18: [2023-05-08 11:25:58,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +18: [2023-05-08 11:25:58,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:25:58,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +18: [2023-05-08 11:25:58,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:25:58,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +29: [2023-05-08 11:25:58,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +18: [2023-05-08 11:25:58,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +31: [2023-05-08 11:25:58,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +31: [2023-05-08 11:25:58,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:25:58,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +30: [2023-05-08 11:25:58,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +30: [2023-05-08 11:25:58,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:25:58,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +30: [2023-05-08 11:25:58,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:25:58,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:25:58,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +16: [2023-05-08 11:25:58,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:25:58,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +22: [2023-05-08 11:25:58,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +31: [2023-05-08 11:25:58,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +31: [2023-05-08 11:25:58,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 8: [2023-05-08 11:25:58,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:25:58,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:25:58,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:25:58,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:25:58,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:25:58,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +12: [2023-05-08 11:25:58,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +12: [2023-05-08 11:25:58,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +12: [2023-05-08 11:25:58,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +12: [2023-05-08 11:25:58,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +30: [2023-05-08 11:25:58,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:25:58,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +26: [2023-05-08 11:25:58,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +26: [2023-05-08 11:25:58,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 2: [2023-05-08 11:25:58,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:25:58,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:25:58,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 7: [2023-05-08 11:25:58,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +18: [2023-05-08 11:25:58,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:25:58,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 7: [2023-05-08 11:25:58,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +18: [2023-05-08 11:25:58,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:25:58,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:25:58,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:25:58,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +16: [2023-05-08 11:25:58,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +24: [2023-05-08 11:25:58,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +24: [2023-05-08 11:25:58,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +30: [2023-05-08 11:25:58,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:25:58,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +18: [2023-05-08 11:25:58,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +18: [2023-05-08 11:25:58,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:25:58,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +25: [2023-05-08 11:25:58,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +25: [2023-05-08 11:25:58,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +25: [2023-05-08 11:25:58,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 4: [2023-05-08 11:25:58,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:25:58,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +25: [2023-05-08 11:25:58,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:25:58,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:25:58,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +25: [2023-05-08 11:25:58,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +25: [2023-05-08 11:25:58,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:25:58,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:25:58,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +30: [2023-05-08 11:25:59,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +30: [2023-05-08 11:25:59,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 5: [2023-05-08 11:25:59,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 5: [2023-05-08 11:25:59,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 8: [2023-05-08 11:25:59,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:25:59,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:25:59,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +30: [2023-05-08 11:25:59,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 5: [2023-05-08 11:25:59,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +30: [2023-05-08 11:25:59,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:25:59,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:25:59,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:25:59,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:25:59,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +24: [2023-05-08 11:25:59,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +24: [2023-05-08 11:25:59,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +22: [2023-05-08 11:25:59,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:25:59,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +22: [2023-05-08 11:25:59,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 2: [2023-05-08 11:25:59,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 5: [2023-05-08 11:25:59,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 3: [2023-05-08 11:25:59,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 2: [2023-05-08 11:25:59,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:25:59,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +30: [2023-05-08 11:25:59,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 8: [2023-05-08 11:25:59,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:25:59,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:25:59,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +30: [2023-05-08 11:25:59,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +30: [2023-05-08 11:25:59,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:25:59,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +25: [2023-05-08 11:25:59,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +25: [2023-05-08 11:25:59,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +25: [2023-05-08 11:25:59,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 7: [2023-05-08 11:25:59,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 5: [2023-05-08 11:25:59,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 5: [2023-05-08 11:25:59,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 5: [2023-05-08 11:25:59,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +25: [2023-05-08 11:25:59,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:25:59,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +30: [2023-05-08 11:25:59,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 7: [2023-05-08 11:25:59,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 5: [2023-05-08 11:25:59,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +25: [2023-05-08 11:25:59,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:25:59,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:25:59,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:25:59,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +30: [2023-05-08 11:25:59,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +30: [2023-05-08 11:25:59,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +30: [2023-05-08 11:25:59,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +30: [2023-05-08 11:25:59,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +30: [2023-05-08 11:25:59,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 5: [2023-05-08 11:25:59,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 3: [2023-05-08 11:25:59,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 3: [2023-05-08 11:25:59,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 3: [2023-05-08 11:25:59,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 3: [2023-05-08 11:25:59,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +20: [2023-05-08 11:25:59,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +20: [2023-05-08 11:25:59,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +31: [2023-05-08 11:25:59,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +31: [2023-05-08 11:25:59,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +20: [2023-05-08 11:25:59,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +31: [2023-05-08 11:25:59,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +20: [2023-05-08 11:25:59,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +18: [2023-05-08 11:25:59,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +18: [2023-05-08 11:25:59,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +31: [2023-05-08 11:25:59,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +18: [2023-05-08 11:25:59,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +18: [2023-05-08 11:25:59,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +26: [2023-05-08 11:25:59,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +26: [2023-05-08 11:25:59,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +26: [2023-05-08 11:25:59,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +26: [2023-05-08 11:25:59,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +20: [2023-05-08 11:25:59,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:25:59,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +20: [2023-05-08 11:25:59,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +20: [2023-05-08 11:25:59,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:25:59,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +20: [2023-05-08 11:25:59,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:25:59,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +18: [2023-05-08 11:25:59,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:25:59,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +31: [2023-05-08 11:25:59,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:25:59,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +22: [2023-05-08 11:25:59,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +22: [2023-05-08 11:25:59,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +18: [2023-05-08 11:25:59,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +18: [2023-05-08 11:25:59,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +18: [2023-05-08 11:25:59,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:25:59,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:25:59,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:25:59,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +26: [2023-05-08 11:25:59,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +20: [2023-05-08 11:25:59,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:25:59,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:25:59,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +20: [2023-05-08 11:25:59,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +22: [2023-05-08 11:25:59,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:25:59,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:25:59,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +22: [2023-05-08 11:25:59,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +18: [2023-05-08 11:25:59,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +18: [2023-05-08 11:25:59,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +31: [2023-05-08 11:25:59,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +31: [2023-05-08 11:25:59,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +31: [2023-05-08 11:25:59,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +31: [2023-05-08 11:25:59,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +31: [2023-05-08 11:25:59,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:25:59,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +31: [2023-05-08 11:25:59,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +20: [2023-05-08 11:25:59,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +31: [2023-05-08 11:25:59,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +31: [2023-05-08 11:25:59,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +18: [2023-05-08 11:25:59,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +18: [2023-05-08 11:25:59,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +20: [2023-05-08 11:25:59,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +20: [2023-05-08 11:25:59,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +26: [2023-05-08 11:25:59,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:25:59,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +18: [2023-05-08 11:25:59,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +18: [2023-05-08 11:25:59,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +26: [2023-05-08 11:25:59,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:25:59,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +18: [2023-05-08 11:25:59,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:25:59,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:25:59,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +26: [2023-05-08 11:25:59,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +26: [2023-05-08 11:25:59,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +18: [2023-05-08 11:25:59,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +22: [2023-05-08 11:25:59,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:25:59,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +22: [2023-05-08 11:25:59,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:25:59,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 6: [2023-05-08 11:25:59,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 6: [2023-05-08 11:25:59,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +13: [2023-05-08 11:25:59,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +28: [2023-05-08 11:25:59,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +13: [2023-05-08 11:25:59,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +13: [2023-05-08 11:25:59,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +28: [2023-05-08 11:25:59,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +28: [2023-05-08 11:25:59,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +13: [2023-05-08 11:25:59,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +28: [2023-05-08 11:25:59,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 8: [2023-05-08 11:25:59,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +15: [2023-05-08 11:25:59,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +15: [2023-05-08 11:25:59,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +15: [2023-05-08 11:25:59,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +15: [2023-05-08 11:25:59,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 9: [2023-05-08 11:25:59,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 9: [2023-05-08 11:25:59,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 9: [2023-05-08 11:25:59,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 6: [2023-05-08 11:25:59,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 9: [2023-05-08 11:25:59,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 6: [2023-05-08 11:25:59,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +25: [2023-05-08 11:25:59,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 8: [2023-05-08 11:25:59,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +25: [2023-05-08 11:25:59,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +25: [2023-05-08 11:25:59,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +25: [2023-05-08 11:25:59,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +22: [2023-05-08 11:25:59,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +22: [2023-05-08 11:25:59,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +27: [2023-05-08 11:25:59,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +27: [2023-05-08 11:25:59,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:25:59,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:25:59,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +27: [2023-05-08 11:25:59,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:25:59,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +27: [2023-05-08 11:25:59,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +27: [2023-05-08 11:25:59,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:25:59,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 8: [2023-05-08 11:25:59,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +12: [2023-05-08 11:25:59,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +12: [2023-05-08 11:25:59,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +12: [2023-05-08 11:25:59,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +27: [2023-05-08 11:25:59,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:25:59,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +13: [2023-05-08 11:25:59,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:25:59,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:25:59,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:25:59,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:25:59,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:25:59,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:25:59,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:25:59,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:25:59,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:25:59,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +11: [2023-05-08 11:25:59,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +15: [2023-05-08 11:25:59,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +28: [2023-05-08 11:25:59,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:25:59,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +28: [2023-05-08 11:25:59,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +28: [2023-05-08 11:25:59,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:25:59,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:25:59,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +25: [2023-05-08 11:25:59,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +11: [2023-05-08 11:25:59,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 9: [2023-05-08 11:25:59,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:25:59,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:25:59,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +11: [2023-05-08 11:25:59,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:25:59,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +11: [2023-05-08 11:25:59,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +25: [2023-05-08 11:25:59,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +25: [2023-05-08 11:25:59,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:25:59,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 2: [2023-05-08 11:25:59,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +27: [2023-05-08 11:25:59,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +28: [2023-05-08 11:25:59,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:25:59,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +12: [2023-05-08 11:25:59,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:25:59,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 1: [2023-05-08 11:25:59,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. + 1: [2023-05-08 11:25:59,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +15: [2023-05-08 11:25:59,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt. +13: [2023-05-08 11:25:59,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:25:59,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +10: [2023-05-08 11:25:59,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:25:59,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:25:59,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +10: [2023-05-08 11:25:59,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +13: [2023-05-08 11:25:59,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +15: [2023-05-08 11:25:59,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:25:59,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +13: [2023-05-08 11:25:59,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 6: [2023-05-08 11:25:59,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 6: [2023-05-08 11:25:59,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +13: [2023-05-08 11:25:59,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:25:59,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +11: [2023-05-08 11:25:59,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +11: [2023-05-08 11:25:59,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +27: [2023-05-08 11:25:59,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:25:59,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:25:59,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 8: [2023-05-08 11:25:59,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +10: [2023-05-08 11:25:59,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:25:59,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 8: [2023-05-08 11:25:59,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +11: [2023-05-08 11:25:59,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +10: [2023-05-08 11:25:59,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +12: [2023-05-08 11:25:59,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +10: [2023-05-08 11:25:59,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 9: [2023-05-08 11:25:59,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +12: [2023-05-08 11:25:59,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +25: [2023-05-08 11:25:59,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +25: [2023-05-08 11:25:59,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 6: [2023-05-08 11:25:59,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +27: [2023-05-08 11:25:59,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:25:59,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... +13: [2023-05-08 11:25:59,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +15: [2023-05-08 11:25:59,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:25:59,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 1: [2023-05-08 11:25:59,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +27: [2023-05-08 11:25:59,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +15: [2023-05-08 11:25:59,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:25:59,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 2: [2023-05-08 11:25:59,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 2: [2023-05-08 11:25:59,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 9: [2023-05-08 11:25:59,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +28: [2023-05-08 11:25:59,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:25:59,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:25:59,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +28: [2023-05-08 11:25:59,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +28: [2023-05-08 11:25:59,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 8: [2023-05-08 11:25:59,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 9: [2023-05-08 11:25:59,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:25:59,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:25:59,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +15: [2023-05-08 11:25:59,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +15: [2023-05-08 11:25:59,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 9: [2023-05-08 11:25:59,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +13: [2023-05-08 11:25:59,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +13: [2023-05-08 11:25:59,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +13: [2023-05-08 11:25:59,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +13: [2023-05-08 11:25:59,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +13: [2023-05-08 11:25:59,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:25:59,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +25: [2023-05-08 11:25:59,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +25: [2023-05-08 11:25:59,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +25: [2023-05-08 11:25:59,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +13: [2023-05-08 11:25:59,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:25:59,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +25: [2023-05-08 11:25:59,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +10: [2023-05-08 11:25:59,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:25:59,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +13: [2023-05-08 11:25:59,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:25:59,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +12: [2023-05-08 11:25:59,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +12: [2023-05-08 11:25:59,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +28: [2023-05-08 11:25:59,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +12: [2023-05-08 11:25:59,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +28: [2023-05-08 11:25:59,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +11: [2023-05-08 11:25:59,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:25:59,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:25:59,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +13: [2023-05-08 11:25:59,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:25:59,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +27: [2023-05-08 11:25:59,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:25:59,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +12: [2023-05-08 11:25:59,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 2: [2023-05-08 11:25:59,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +11: [2023-05-08 11:25:59,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 2: [2023-05-08 11:25:59,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 1: [2023-05-08 11:25:59,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:25:59,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:25:59,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +11: [2023-05-08 11:25:59,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:25:59,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 1: [2023-05-08 11:25:59,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 1: [2023-05-08 11:25:59,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 1: [2023-05-08 11:25:59,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... + 1: [2023-05-08 11:25:59,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +10: [2023-05-08 11:25:59,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +13: [2023-05-08 11:25:59,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:25:59,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:25:59,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +27: [2023-05-08 11:25:59,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:25:59,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +13: [2023-05-08 11:25:59,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +13: [2023-05-08 11:25:59,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +11: [2023-05-08 11:25:59,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt. +11: [2023-05-08 11:25:59,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt... +10: [2023-05-08 11:25:59,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:25:59,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:25:59,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:25:59,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +13: [2023-05-08 11:25:59,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +13: [2023-05-08 11:25:59,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:25:59,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +13: [2023-05-08 11:25:59,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:25:59,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +13: [2023-05-08 11:25:59,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:25:59,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +19: [2023-05-08 11:25:59,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +19: [2023-05-08 11:25:59,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +19: [2023-05-08 11:25:59,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 9: [2023-05-08 11:25:59,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +13: [2023-05-08 11:25:59,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:25:59,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:25:59,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:25:59,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:25:59,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:25:59,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:25:59,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:25:59,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +10: [2023-05-08 11:25:59,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +10: [2023-05-08 11:25:59,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +10: [2023-05-08 11:25:59,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +19: [2023-05-08 11:25:59,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:25:59,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:25:59,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:25:59,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:25:59,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:25:59,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:25:59,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:25:59,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:25:59,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:25:59,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:25:59,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:25:59,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:25:59,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +10: [2023-05-08 11:25:59,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +10: [2023-05-08 11:25:59,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:25:59,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +10: [2023-05-08 11:25:59,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:25:59,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:25:59,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:25:59,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:25:59,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:25:59,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:25:59,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +23: [2023-05-08 11:25:59,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:25:59,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:25:59,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:25:59,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:25:59,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:25:59,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:25:59,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:25:59,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:25:59,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:25:59,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:25:59,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:25:59,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +10: [2023-05-08 11:25:59,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:25:59,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:25:59,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +14: [2023-05-08 11:25:59,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:25:59,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 3: [2023-05-08 11:25:59,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:25:59,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 0: [2023-05-08 11:25:59,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 0: [2023-05-08 11:25:59,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 0: [2023-05-08 11:25:59,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 3: [2023-05-08 11:25:59,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:25:59,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:25:59,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +15: [2023-05-08 11:25:59,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +15: [2023-05-08 11:25:59,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +15: [2023-05-08 11:25:59,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +15: [2023-05-08 11:25:59,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +15: [2023-05-08 11:25:59,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +17: [2023-05-08 11:25:59,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +17: [2023-05-08 11:25:59,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:25:59,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +17: [2023-05-08 11:25:59,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +17: [2023-05-08 11:25:59,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:25:59,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:25:59,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +15: [2023-05-08 11:25:59,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +15: [2023-05-08 11:25:59,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:25:59,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +15: [2023-05-08 11:25:59,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +17: [2023-05-08 11:25:59,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +17: [2023-05-08 11:25:59,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +15: [2023-05-08 11:25:59,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +17: [2023-05-08 11:25:59,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +17: [2023-05-08 11:25:59,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +17: [2023-05-08 11:25:59,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:25:59,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +21: [2023-05-08 11:25:59,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +21: [2023-05-08 11:25:59,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +14: [2023-05-08 11:25:59,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +16: [2023-05-08 11:25:59,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +17: [2023-05-08 11:25:59,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:25:59,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 4: [2023-05-08 11:25:59,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:25:59,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +16: [2023-05-08 11:25:59,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +14: [2023-05-08 11:25:59,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +16: [2023-05-08 11:25:59,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +16: [2023-05-08 11:25:59,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 4: [2023-05-08 11:25:59,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +29: [2023-05-08 11:25:59,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +29: [2023-05-08 11:25:59,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +29: [2023-05-08 11:25:59,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +29: [2023-05-08 11:25:59,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +15: [2023-05-08 11:25:59,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +21: [2023-05-08 11:25:59,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 7: [2023-05-08 11:25:59,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 5: [2023-05-08 11:25:59,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 5: [2023-05-08 11:25:59,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 5: [2023-05-08 11:25:59,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 7: [2023-05-08 11:25:59,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 7: [2023-05-08 11:25:59,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 7: [2023-05-08 11:25:59,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +23: [2023-05-08 11:25:59,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +23: [2023-05-08 11:25:59,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +23: [2023-05-08 11:25:59,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 5: [2023-05-08 11:25:59,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +23: [2023-05-08 11:25:59,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 4: [2023-05-08 11:25:59,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 4: [2023-05-08 11:25:59,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 4: [2023-05-08 11:25:59,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +21: [2023-05-08 11:25:59,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +27: [2023-05-08 11:25:59,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +27: [2023-05-08 11:25:59,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +27: [2023-05-08 11:25:59,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +27: [2023-05-08 11:25:59,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 4: [2023-05-08 11:25:59,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 4: [2023-05-08 11:25:59,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +17: [2023-05-08 11:25:59,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +17: [2023-05-08 11:25:59,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +17: [2023-05-08 11:25:59,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +17: [2023-05-08 11:25:59,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +17: [2023-05-08 11:25:59,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:25:59,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +14: [2023-05-08 11:25:59,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:25:59,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:25:59,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +15: [2023-05-08 11:25:59,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:25:59,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:25:59,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:25:59,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:25:59,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:25:59,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:25:59,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:25:59,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:25:59,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:25:59,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +15: [2023-05-08 11:25:59,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:25:59,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:25:59,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:25:59,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:25:59,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:25:59,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:25:59,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:25:59,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:25:59,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:25:59,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:25:59,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:25:59,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:25:59,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:25:59,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:25:59,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +24: [2023-05-08 11:25:59,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +24: [2023-05-08 11:25:59,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +24: [2023-05-08 11:25:59,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +17: [2023-05-08 11:25:59,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:25:59,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +17: [2023-05-08 11:25:59,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:25:59,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:25:59,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +15: [2023-05-08 11:25:59,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:25:59,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:25:59,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:25:59,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +24: [2023-05-08 11:25:59,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:25:59,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:25:59,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:25:59,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +24: [2023-05-08 11:25:59,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +23: [2023-05-08 11:25:59,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +17: [2023-05-08 11:25:59,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +17: [2023-05-08 11:25:59,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +17: [2023-05-08 11:25:59,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +17: [2023-05-08 11:25:59,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +27: [2023-05-08 11:25:59,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +27: [2023-05-08 11:25:59,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +27: [2023-05-08 11:25:59,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +27: [2023-05-08 11:25:59,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 4: [2023-05-08 11:25:59,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 4: [2023-05-08 11:25:59,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 4: [2023-05-08 11:25:59,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +14: [2023-05-08 11:25:59,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:25:59,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:25:59,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +14: [2023-05-08 11:25:59,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 4: [2023-05-08 11:25:59,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:25:59,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +27: [2023-05-08 11:25:59,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 3: [2023-05-08 11:25:59,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 3: [2023-05-08 11:25:59,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 4: [2023-05-08 11:25:59,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 3: [2023-05-08 11:25:59,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +27: [2023-05-08 11:25:59,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +14: [2023-05-08 11:25:59,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +17: [2023-05-08 11:25:59,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:25:59,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +17: [2023-05-08 11:25:59,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +23: [2023-05-08 11:25:59,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +27: [2023-05-08 11:25:59,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:25:59,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +18: [2023-05-08 11:25:59,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +18: [2023-05-08 11:25:59,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +18: [2023-05-08 11:25:59,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +23: [2023-05-08 11:25:59,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +27: [2023-05-08 11:25:59,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +24: [2023-05-08 11:25:59,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:25:59,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:25:59,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:25:59,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +31: [2023-05-08 11:25:59,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +31: [2023-05-08 11:25:59,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +24: [2023-05-08 11:25:59,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +24: [2023-05-08 11:25:59,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:25:59,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +16: [2023-05-08 11:25:59,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +16: [2023-05-08 11:25:59,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +16: [2023-05-08 11:25:59,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +16: [2023-05-08 11:25:59,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +16: [2023-05-08 11:25:59,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +16: [2023-05-08 11:25:59,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +16: [2023-05-08 11:25:59,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +16: [2023-05-08 11:25:59,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:25:59,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +24: [2023-05-08 11:25:59,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +31: [2023-05-08 11:25:59,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:25:59,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +16: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +16: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +16: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 0: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +24: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +24: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +24: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +24: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +24: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +24: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 0: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +24: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 0: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 0: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 0: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +18: [2023-05-08 11:25:59,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 8: [2023-05-08 11:25:59,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 7: [2023-05-08 11:25:59,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 7: [2023-05-08 11:25:59,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 6: [2023-05-08 11:25:59,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +31: [2023-05-08 11:25:59,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +16: [2023-05-08 11:25:59,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 0: [2023-05-08 11:25:59,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +16: [2023-05-08 11:25:59,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:25:59,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:25:59,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +24: [2023-05-08 11:25:59,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +17: [2023-05-08 11:25:59,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 7: [2023-05-08 11:25:59,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 6: [2023-05-08 11:25:59,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +20: [2023-05-08 11:25:59,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +20: [2023-05-08 11:25:59,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +20: [2023-05-08 11:25:59,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +20: [2023-05-08 11:25:59,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +20: [2023-05-08 11:25:59,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +20: [2023-05-08 11:25:59,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:25:59,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +30: [2023-05-08 11:25:59,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:25:59,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:25:59,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:25:59,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +30: [2023-05-08 11:25:59,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +20: [2023-05-08 11:25:59,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +20: [2023-05-08 11:25:59,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:25:59,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +18: [2023-05-08 11:25:59,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:25:59,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:25:59,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:25:59,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:25:59,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:25:59,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:25:59,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:25:59,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:25:59,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:25:59,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:25:59,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +21: [2023-05-08 11:25:59,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:25:59,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:25:59,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:25:59,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:25:59,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:25:59,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:25:59,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:25:59,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:25:59,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:25:59,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:25:59,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:25:59,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:25:59,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +28: [2023-05-08 11:25:59,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +28: [2023-05-08 11:25:59,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:25:59,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +28: [2023-05-08 11:25:59,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:25:59,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +21: [2023-05-08 11:25:59,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +18: [2023-05-08 11:25:59,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:25:59,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +28: [2023-05-08 11:25:59,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:25:59,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +21: [2023-05-08 11:25:59,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:25:59,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:25:59,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +22: [2023-05-08 11:25:59,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:25:59,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:25:59,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +22: [2023-05-08 11:25:59,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:25:59,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:25:59,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:25:59,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:25:59,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:25:59,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +16: [2023-05-08 11:25:59,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:25:59,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:25:59,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:25:59,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:25:59,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +26: [2023-05-08 11:25:59,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:25:59,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:25:59,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:25:59,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:25:59,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:25:59,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:25:59,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +26: [2023-05-08 11:25:59,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +26: [2023-05-08 11:25:59,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +24: [2023-05-08 11:25:59,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:25:59,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:25:59,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 5: [2023-05-08 11:25:59,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 5: [2023-05-08 11:25:59,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +19: [2023-05-08 11:25:59,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:25:59,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:25:59,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:25:59,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:25:59,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +22: [2023-05-08 11:25:59,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +18: [2023-05-08 11:25:59,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:25:59,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +18: [2023-05-08 11:25:59,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:25:59,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:25:59,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:25:59,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:25:59,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:25:59,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +19: [2023-05-08 11:25:59,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:25:59,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:25:59,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:25:59,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +18: [2023-05-08 11:25:59,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:25:59,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:25:59,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +18: [2023-05-08 11:25:59,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +30: [2023-05-08 11:25:59,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:25:59,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +19: [2023-05-08 11:25:59,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:25:59,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:25:59,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +19: [2023-05-08 11:25:59,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +18: [2023-05-08 11:25:59,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:25:59,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +30: [2023-05-08 11:25:59,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +30: [2023-05-08 11:25:59,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +18: [2023-05-08 11:25:59,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +24: [2023-05-08 11:25:59,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:25:59,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:25:59,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:25:59,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:25:59,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:25:59,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:25:59,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:25:59,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:25:59,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:25:59,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:25:59,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:25:59,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:25:59,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +29: [2023-05-08 11:25:59,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +29: [2023-05-08 11:25:59,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:25:59,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +21: [2023-05-08 11:25:59,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:25:59,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +30: [2023-05-08 11:25:59,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +21: [2023-05-08 11:25:59,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +27: [2023-05-08 11:25:59,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +27: [2023-05-08 11:25:59,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +29: [2023-05-08 11:25:59,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +29: [2023-05-08 11:25:59,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +27: [2023-05-08 11:25:59,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:25:59,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +27: [2023-05-08 11:25:59,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:25:59,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:25:59,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +27: [2023-05-08 11:25:59,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +31: [2023-05-08 11:25:59,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +28: [2023-05-08 11:25:59,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +28: [2023-05-08 11:25:59,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:25:59,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:25:59,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:25:59,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:25:59,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +18: [2023-05-08 11:25:59,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +16: [2023-05-08 11:25:59,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +24: [2023-05-08 11:25:59,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:25:59,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:25:59,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:25:59,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 3: [2023-05-08 11:25:59,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 5: [2023-05-08 11:25:59,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +26: [2023-05-08 11:25:59,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:25:59,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:25:59,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:25:59,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +20: [2023-05-08 11:25:59,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +30: [2023-05-08 11:25:59,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:25:59,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +20: [2023-05-08 11:25:59,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +20: [2023-05-08 11:25:59,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +20: [2023-05-08 11:25:59,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +16: [2023-05-08 11:25:59,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:25:59,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +26: [2023-05-08 11:25:59,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:25:59,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:25:59,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:25:59,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:25:59,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:25:59,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:25:59,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:25:59,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +16: [2023-05-08 11:25:59,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +16: [2023-05-08 11:25:59,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +31: [2023-05-08 11:25:59,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:25:59,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +30: [2023-05-08 11:25:59,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:25:59,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:25:59,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +30: [2023-05-08 11:25:59,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:25:59,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +18: [2023-05-08 11:25:59,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 6: [2023-05-08 11:25:59,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 6: [2023-05-08 11:25:59,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +27: [2023-05-08 11:25:59,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +29: [2023-05-08 11:25:59,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:25:59,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:25:59,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +28: [2023-05-08 11:25:59,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:25:59,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +27: [2023-05-08 11:25:59,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +28: [2023-05-08 11:25:59,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:25:59,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +22: [2023-05-08 11:25:59,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +22: [2023-05-08 11:25:59,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +24: [2023-05-08 11:25:59,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:25:59,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 1: [2023-05-08 11:25:59,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +26: [2023-05-08 11:25:59,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +10: [2023-05-08 11:25:59,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +10: [2023-05-08 11:25:59,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +10: [2023-05-08 11:25:59,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +27: [2023-05-08 11:25:59,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:25:59,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +10: [2023-05-08 11:25:59,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +10: [2023-05-08 11:25:59,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +10: [2023-05-08 11:25:59,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +29: [2023-05-08 11:25:59,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:25:59,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:25:59,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +24: [2023-05-08 11:25:59,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:25:59,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:25:59,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +30: [2023-05-08 11:25:59,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +31: [2023-05-08 11:25:59,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:25:59,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:25:59,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:25:59,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:25:59,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:25:59,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +21: [2023-05-08 11:25:59,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:25:59,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:25:59,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:25:59,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:25:59,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +20: [2023-05-08 11:25:59,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:25:59,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +26: [2023-05-08 11:25:59,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:25:59,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +30: [2023-05-08 11:25:59,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:25:59,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +20: [2023-05-08 11:25:59,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:25:59,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +27: [2023-05-08 11:25:59,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +29: [2023-05-08 11:25:59,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +21: [2023-05-08 11:25:59,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +27: [2023-05-08 11:25:59,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +28: [2023-05-08 11:25:59,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:25:59,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:25:59,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:25:59,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +24: [2023-05-08 11:25:59,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:25:59,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:25:59,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +19: [2023-05-08 11:25:59,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:25:59,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +22: [2023-05-08 11:25:59,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +22: [2023-05-08 11:25:59,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:25:59,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +28: [2023-05-08 11:25:59,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +27: [2023-05-08 11:25:59,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:25:59,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +22: [2023-05-08 11:25:59,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +27: [2023-05-08 11:25:59,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +24: [2023-05-08 11:25:59,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +30: [2023-05-08 11:25:59,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +29: [2023-05-08 11:25:59,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:25:59,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +30: [2023-05-08 11:25:59,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 6: [2023-05-08 11:25:59,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +30: [2023-05-08 11:25:59,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +29: [2023-05-08 11:25:59,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 6: [2023-05-08 11:25:59,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:25:59,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +21: [2023-05-08 11:25:59,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 0: [2023-05-08 11:25:59,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +30: [2023-05-08 11:25:59,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +30: [2023-05-08 11:25:59,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +30: [2023-05-08 11:25:59,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +30: [2023-05-08 11:25:59,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +10: [2023-05-08 11:25:59,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:25:59,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +10: [2023-05-08 11:25:59,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +12: [2023-05-08 11:25:59,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +30: [2023-05-08 11:25:59,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:25:59,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:25:59,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:25:59,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:25:59,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +21: [2023-05-08 11:25:59,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +13: [2023-05-08 11:25:59,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +13: [2023-05-08 11:25:59,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +25: [2023-05-08 11:25:59,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +13: [2023-05-08 11:25:59,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +19: [2023-05-08 11:25:59,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +28: [2023-05-08 11:25:59,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +13: [2023-05-08 11:25:59,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +30: [2023-05-08 11:25:59,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +30: [2023-05-08 11:25:59,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +19: [2023-05-08 11:25:59,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:25:59,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:25:59,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:25:59,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +18: [2023-05-08 11:25:59,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:25:59,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +22: [2023-05-08 11:25:59,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +10: [2023-05-08 11:25:59,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:25:59,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 2: [2023-05-08 11:25:59,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +29: [2023-05-08 11:25:59,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:25:59,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:25:59,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 6: [2023-05-08 11:25:59,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +10: [2023-05-08 11:25:59,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +18: [2023-05-08 11:25:59,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:25:59,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +22: [2023-05-08 11:25:59,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:25:59,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +20: [2023-05-08 11:25:59,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 1: [2023-05-08 11:25:59,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 1: [2023-05-08 11:25:59,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 1: [2023-05-08 11:25:59,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +25: [2023-05-08 11:25:59,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +28: [2023-05-08 11:25:59,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +28: [2023-05-08 11:25:59,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +28: [2023-05-08 11:25:59,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +28: [2023-05-08 11:25:59,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 2: [2023-05-08 11:25:59,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +13: [2023-05-08 11:25:59,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +22: [2023-05-08 11:25:59,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:25:59,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:25:59,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +20: [2023-05-08 11:25:59,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:25:59,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:25:59,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +22: [2023-05-08 11:25:59,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +29: [2023-05-08 11:25:59,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:25:59,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +13: [2023-05-08 11:25:59,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:25:59,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +13: [2023-05-08 11:25:59,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +20: [2023-05-08 11:25:59,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +20: [2023-05-08 11:25:59,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 1: [2023-05-08 11:25:59,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +10: [2023-05-08 11:25:59,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +28: [2023-05-08 11:25:59,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +13: [2023-05-08 11:25:59,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +13: [2023-05-08 11:25:59,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +28: [2023-05-08 11:25:59,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +31: [2023-05-08 11:25:59,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +31: [2023-05-08 11:25:59,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +31: [2023-05-08 11:25:59,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +22: [2023-05-08 11:25:59,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +31: [2023-05-08 11:25:59,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +20: [2023-05-08 11:25:59,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:25:59,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +22: [2023-05-08 11:25:59,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +28: [2023-05-08 11:25:59,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +13: [2023-05-08 11:25:59,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +13: [2023-05-08 11:25:59,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +20: [2023-05-08 11:25:59,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +11: [2023-05-08 11:25:59,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +11: [2023-05-08 11:25:59,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +11: [2023-05-08 11:25:59,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +11: [2023-05-08 11:25:59,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 8: [2023-05-08 11:25:59,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 8: [2023-05-08 11:25:59,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 8: [2023-05-08 11:25:59,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 8: [2023-05-08 11:25:59,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +26: [2023-05-08 11:25:59,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +26: [2023-05-08 11:25:59,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +26: [2023-05-08 11:25:59,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 2: [2023-05-08 11:25:59,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:25:59,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +25: [2023-05-08 11:25:59,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +25: [2023-05-08 11:25:59,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +26: [2023-05-08 11:25:59,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +25: [2023-05-08 11:25:59,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 1: [2023-05-08 11:25:59,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 1: [2023-05-08 11:25:59,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 9: [2023-05-08 11:25:59,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 2: [2023-05-08 11:25:59,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +31: [2023-05-08 11:25:59,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +28: [2023-05-08 11:25:59,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +28: [2023-05-08 11:25:59,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +13: [2023-05-08 11:25:59,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +31: [2023-05-08 11:25:59,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +31: [2023-05-08 11:25:59,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +31: [2023-05-08 11:25:59,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +13: [2023-05-08 11:25:59,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 8: [2023-05-08 11:25:59,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:25:59,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +28: [2023-05-08 11:25:59,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +11: [2023-05-08 11:25:59,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +11: [2023-05-08 11:25:59,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +11: [2023-05-08 11:25:59,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +13: [2023-05-08 11:25:59,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:25:59,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +13: [2023-05-08 11:25:59,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +26: [2023-05-08 11:25:59,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +26: [2023-05-08 11:25:59,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:25:59,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:25:59,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:25:59,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +25: [2023-05-08 11:25:59,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:25:59,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +26: [2023-05-08 11:25:59,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +31: [2023-05-08 11:25:59,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:25:59,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +31: [2023-05-08 11:25:59,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:25:59,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +11: [2023-05-08 11:25:59,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:25:59,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +11: [2023-05-08 11:25:59,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:25:59,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 8: [2023-05-08 11:25:59,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:25:59,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:25:59,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 8: [2023-05-08 11:25:59,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 8: [2023-05-08 11:25:59,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 8: [2023-05-08 11:25:59,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 8: [2023-05-08 11:25:59,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +28: [2023-05-08 11:25:59,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +28: [2023-05-08 11:25:59,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:25:59,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +28: [2023-05-08 11:25:59,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +26: [2023-05-08 11:25:59,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:25:59,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +28: [2023-05-08 11:25:59,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +26: [2023-05-08 11:25:59,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +25: [2023-05-08 11:25:59,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:25:59,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:25:59,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:25:59,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +25: [2023-05-08 11:25:59,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +25: [2023-05-08 11:25:59,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +25: [2023-05-08 11:25:59,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +25: [2023-05-08 11:25:59,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +31: [2023-05-08 11:25:59,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:25:59,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +31: [2023-05-08 11:25:59,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +31: [2023-05-08 11:25:59,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +15: [2023-05-08 11:25:59,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +12: [2023-05-08 11:25:59,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +12: [2023-05-08 11:25:59,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +12: [2023-05-08 11:25:59,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +12: [2023-05-08 11:25:59,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +15: [2023-05-08 11:25:59,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +15: [2023-05-08 11:25:59,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +15: [2023-05-08 11:25:59,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 9: [2023-05-08 11:25:59,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 9: [2023-05-08 11:25:59,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 9: [2023-05-08 11:25:59,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. + 9: [2023-05-08 11:25:59,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt. +11: [2023-05-08 11:25:59,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +11: [2023-05-08 11:25:59,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +26: [2023-05-08 11:25:59,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:25:59,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +26: [2023-05-08 11:25:59,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +26: [2023-05-08 11:25:59,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +12: [2023-05-08 11:25:59,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:25:59,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:25:59,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +15: [2023-05-08 11:25:59,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:25:59,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +15: [2023-05-08 11:25:59,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +15: [2023-05-08 11:25:59,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +15: [2023-05-08 11:25:59,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt... +12: [2023-05-08 11:25:59,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:25:59,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 9: [2023-05-08 11:25:59,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 9: [2023-05-08 11:25:59,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... + 9: [2023-05-08 11:25:59,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +15: [2023-05-08 11:25:59,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +15: [2023-05-08 11:25:59,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +15: [2023-05-08 11:25:59,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +15: [2023-05-08 11:25:59,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +12: [2023-05-08 11:25:59,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +12: [2023-05-08 11:25:59,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +15: [2023-05-08 11:25:59,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +15: [2023-05-08 11:25:59,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +11: [2023-05-08 11:25:59,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:25:59,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:25:59,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:25:59,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +23: [2023-05-08 11:25:59,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +23: [2023-05-08 11:25:59,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:25:59,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +23: [2023-05-08 11:25:59,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +23: [2023-05-08 11:25:59,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:25:59,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:25:59,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +23: [2023-05-08 11:25:59,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:25:59,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +15: [2023-05-08 11:25:59,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:25:59,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:25:59,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +14: [2023-05-08 11:25:59,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +14: [2023-05-08 11:25:59,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +14: [2023-05-08 11:25:59,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:25:59,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +14: [2023-05-08 11:25:59,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:25:59,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:25:59,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +14: [2023-05-08 11:25:59,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:25:59,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt. +13: [2023-05-08 11:25:59,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:25:59,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:25:59,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:25:59,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +13: [2023-05-08 11:25:59,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:25:59,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +13: [2023-05-08 11:25:59,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:25:59,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +13: [2023-05-08 11:25:59,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:25:59,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +17: [2023-05-08 11:25:59,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +17: [2023-05-08 11:25:59,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +17: [2023-05-08 11:25:59,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +17: [2023-05-08 11:25:59,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +17: [2023-05-08 11:25:59,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:25:59,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +17: [2023-05-08 11:25:59,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +15: [2023-05-08 11:25:59,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +12: [2023-05-08 11:25:59,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt... +17: [2023-05-08 11:25:59,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +17: [2023-05-08 11:25:59,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:25:59,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:25:59,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +15: [2023-05-08 11:25:59,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +15: [2023-05-08 11:25:59,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:25:59,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +15: [2023-05-08 11:25:59,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:25:59,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:25:59,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +15: [2023-05-08 11:25:59,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:25:59,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:25:59,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:25:59,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:25:59,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +17: [2023-05-08 11:25:59,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +14: [2023-05-08 11:25:59,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +14: [2023-05-08 11:25:59,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +14: [2023-05-08 11:25:59,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +14: [2023-05-08 11:25:59,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +23: [2023-05-08 11:25:59,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +23: [2023-05-08 11:25:59,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +23: [2023-05-08 11:25:59,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +17: [2023-05-08 11:25:59,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:25:59,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:25:59,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:25:59,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +17: [2023-05-08 11:25:59,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +23: [2023-05-08 11:25:59,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +17: [2023-05-08 11:25:59,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:25:59,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +14: [2023-05-08 11:25:59,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +11: [2023-05-08 11:25:59,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:25:59,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:25:59,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +14: [2023-05-08 11:25:59,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +14: [2023-05-08 11:25:59,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:25:59,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:25:59,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:25:59,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +15: [2023-05-08 11:25:59,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +14: [2023-05-08 11:25:59,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:25:59,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +11: [2023-05-08 11:25:59,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:25:59,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:25:59,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:25:59,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:25:59,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:25:59,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:25:59,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +31: [2023-05-08 11:25:59,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +31: [2023-05-08 11:25:59,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +31: [2023-05-08 11:25:59,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +31: [2023-05-08 11:25:59,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +31: [2023-05-08 11:25:59,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +31: [2023-05-08 11:25:59,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +31: [2023-05-08 11:25:59,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:25:59,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +24: [2023-05-08 11:25:59,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:25:59,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +24: [2023-05-08 11:25:59,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +24: [2023-05-08 11:25:59,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:25:59,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:25:59,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +24: [2023-05-08 11:25:59,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:25:59,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:25:59,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +15: [2023-05-08 11:25:59,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:25:59,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:25:59,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:25:59,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:25:59,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:25:59,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:25:59,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +15: [2023-05-08 11:25:59,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:25:59,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:25:59,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:25:59,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:25:59,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +31: [2023-05-08 11:25:59,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:25:59,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:25:59,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:25:59,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:25:59,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:25:59,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:25:59,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +24: [2023-05-08 11:25:59,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +24: [2023-05-08 11:25:59,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +24: [2023-05-08 11:25:59,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:25:59,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:25:59,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:25:59,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:25:59,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:25:59,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:25:59,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +21: [2023-05-08 11:25:59,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +21: [2023-05-08 11:25:59,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:25:59,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +28: [2023-05-08 11:25:59,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:25:59,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:25:59,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +18: [2023-05-08 11:25:59,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:25:59,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:25:59,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:25:59,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +18: [2023-05-08 11:25:59,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +18: [2023-05-08 11:25:59,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +18: [2023-05-08 11:25:59,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:25:59,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:25:59,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:25:59,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:25:59,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:25:59,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:25:59,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:25:59,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +28: [2023-05-08 11:25:59,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:25:59,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:25:59,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +28: [2023-05-08 11:25:59,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:25:59,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:25:59,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:25:59,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:25:59,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:25:59,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:25:59,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:25:59,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:25:59,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:25:59,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:25:59,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:25:59,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +18: [2023-05-08 11:25:59,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +18: [2023-05-08 11:25:59,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:25:59,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:25:59,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:25:59,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:25:59,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:25:59,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:25:59,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:25:59,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +12: [2023-05-08 11:25:59,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +12: [2023-05-08 11:25:59,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:25:59,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +12: [2023-05-08 11:25:59,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:25:59,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +12: [2023-05-08 11:25:59,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:25:59,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +12: [2023-05-08 11:25:59,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:25:59,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:25:59,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +21: [2023-05-08 11:25:59,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:25:59,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +28: [2023-05-08 11:25:59,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:25:59,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:25:59,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +29: [2023-05-08 11:25:59,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +29: [2023-05-08 11:25:59,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:25:59,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +29: [2023-05-08 11:25:59,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +29: [2023-05-08 11:25:59,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:25:59,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:25:59,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:25:59,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:25:59,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:25:59,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:25:59,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:25:59,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +29: [2023-05-08 11:25:59,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:25:59,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +29: [2023-05-08 11:25:59,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:25:59,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:25:59,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +29: [2023-05-08 11:25:59,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +29: [2023-05-08 11:25:59,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:25:59,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:25:59,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:25:59,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +12: [2023-05-08 11:25:59,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:25:59,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +12: [2023-05-08 11:25:59,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +12: [2023-05-08 11:25:59,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +12: [2023-05-08 11:25:59,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +12: [2023-05-08 11:25:59,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:25:59,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +29: [2023-05-08 11:25:59,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +29: [2023-05-08 11:25:59,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:25:59,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:25:59,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:25:59,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +12: [2023-05-08 11:25:59,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +12: [2023-05-08 11:25:59,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +12: [2023-05-08 11:25:59,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +29: [2023-05-08 11:25:59,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +29: [2023-05-08 11:25:59,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +29: [2023-05-08 11:25:59,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:25:59,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:25:59,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:25:59,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:25:59,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +29: [2023-05-08 11:25:59,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +29: [2023-05-08 11:25:59,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:25:59,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:25:59,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:25:59,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +29: [2023-05-08 11:25:59,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:25:59,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:25:59,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +26: [2023-05-08 11:25:59,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:25:59,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:25:59,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:25:59,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:25:59,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:25:59,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +26: [2023-05-08 11:25:59,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:25:59,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +26: [2023-05-08 11:25:59,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:25:59,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:25:59,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +26: [2023-05-08 11:25:59,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:25:59,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:25:59,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:25:59,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:25:59,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:25:59,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:25:59,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:25:59,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:25:59,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:25:59,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:25:59,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:25:59,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:25:59,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:25:59,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:25:59,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:25:59,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +30: [2023-05-08 11:25:59,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:25:59,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:25:59,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +30: [2023-05-08 11:25:59,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +30: [2023-05-08 11:25:59,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:25:59,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:25:59,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +30: [2023-05-08 11:25:59,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:25:59,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +26: [2023-05-08 11:25:59,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:25:59,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:25:59,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:25:59,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:25:59,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:25:59,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:25:59,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:25:59,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:25:59,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:25:59,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:25:59,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:25:59,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:25:59,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:25:59,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:25:59,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:25:59,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:25:59,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +26: [2023-05-08 11:25:59,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +19: [2023-05-08 11:25:59,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +19: [2023-05-08 11:25:59,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +19: [2023-05-08 11:25:59,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:25:59,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +26: [2023-05-08 11:25:59,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:25:59,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:25:59,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +27: [2023-05-08 11:25:59,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +19: [2023-05-08 11:25:59,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:25:59,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:25:59,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:25:59,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +27: [2023-05-08 11:25:59,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +27: [2023-05-08 11:25:59,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +16: [2023-05-08 11:25:59,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:25:59,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +16: [2023-05-08 11:25:59,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:25:59,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +10: [2023-05-08 11:25:59,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +10: [2023-05-08 11:25:59,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +10: [2023-05-08 11:25:59,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +30: [2023-05-08 11:25:59,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:25:59,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:25:59,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +16: [2023-05-08 11:25:59,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +10: [2023-05-08 11:25:59,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +16: [2023-05-08 11:25:59,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:25:59,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:25:59,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +10: [2023-05-08 11:25:59,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +10: [2023-05-08 11:25:59,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +10: [2023-05-08 11:25:59,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +10: [2023-05-08 11:25:59,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +26: [2023-05-08 11:25:59,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:25:59,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +27: [2023-05-08 11:25:59,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:25:59,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:25:59,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +27: [2023-05-08 11:25:59,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:25:59,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +16: [2023-05-08 11:25:59,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:25:59,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +10: [2023-05-08 11:25:59,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +10: [2023-05-08 11:25:59,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:25:59,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +10: [2023-05-08 11:25:59,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:25:59,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:25:59,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:25:59,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +23: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +23: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 0: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 4: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 0: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 0: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +14: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +14: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +23: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 4: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 4: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +27: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +17: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +17: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 0: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +17: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 4: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +27: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +27: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 2: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 2: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +14: [2023-05-08 11:25:59,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +27: [2023-05-08 11:25:59,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +17: [2023-05-08 11:25:59,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +14: [2023-05-08 11:25:59,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 2: [2023-05-08 11:25:59,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +10: [2023-05-08 11:25:59,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +10: [2023-05-08 11:25:59,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 2: [2023-05-08 11:25:59,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +10: [2023-05-08 11:25:59,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +10: [2023-05-08 11:25:59,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 9: [2023-05-08 11:25:59,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +10: [2023-05-08 11:25:59,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +10: [2023-05-08 11:25:59,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:25:59,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +29: [2023-05-08 11:25:59,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +29: [2023-05-08 11:25:59,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +29: [2023-05-08 11:25:59,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +29: [2023-05-08 11:25:59,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 4: [2023-05-08 11:25:59,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:25:59,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:25:59,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:25:59,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:25:59,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:25:59,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +10: [2023-05-08 11:25:59,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:25:59,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +27: [2023-05-08 11:25:59,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +27: [2023-05-08 11:25:59,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +27: [2023-05-08 11:25:59,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:25:59,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +27: [2023-05-08 11:25:59,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:25:59,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:25:59,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 5: [2023-05-08 11:25:59,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 5: [2023-05-08 11:25:59,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 5: [2023-05-08 11:25:59,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 2: [2023-05-08 11:25:59,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +17: [2023-05-08 11:25:59,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:25:59,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:25:59,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:25:59,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:25:59,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:25:59,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +25: [2023-05-08 11:25:59,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +25: [2023-05-08 11:25:59,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +25: [2023-05-08 11:25:59,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +16: [2023-05-08 11:25:59,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +17: [2023-05-08 11:25:59,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:25:59,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +16: [2023-05-08 11:25:59,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +16: [2023-05-08 11:25:59,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +17: [2023-05-08 11:25:59,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:25:59,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:25:59,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:25:59,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +22: [2023-05-08 11:25:59,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +22: [2023-05-08 11:25:59,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +22: [2023-05-08 11:25:59,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +29: [2023-05-08 11:25:59,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +29: [2023-05-08 11:25:59,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +29: [2023-05-08 11:25:59,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:25:59,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +17: [2023-05-08 11:25:59,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:25:59,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +17: [2023-05-08 11:25:59,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +25: [2023-05-08 11:25:59,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:25:59,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:25:59,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:25:59,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +30: [2023-05-08 11:25:59,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 5: [2023-05-08 11:25:59,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +30: [2023-05-08 11:25:59,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +30: [2023-05-08 11:25:59,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +30: [2023-05-08 11:25:59,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +16: [2023-05-08 11:25:59,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:25:59,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:25:59,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +11: [2023-05-08 11:25:59,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +11: [2023-05-08 11:25:59,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +24: [2023-05-08 11:25:59,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +11: [2023-05-08 11:25:59,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +14: [2023-05-08 11:25:59,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:25:59,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:25:59,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +24: [2023-05-08 11:25:59,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +24: [2023-05-08 11:25:59,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +22: [2023-05-08 11:25:59,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +14: [2023-05-08 11:25:59,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +24: [2023-05-08 11:25:59,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +22: [2023-05-08 11:25:59,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:25:59,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 4: [2023-05-08 11:25:59,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:25:59,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +28: [2023-05-08 11:25:59,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 6: [2023-05-08 11:25:59,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 6: [2023-05-08 11:25:59,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 4: [2023-05-08 11:25:59,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +20: [2023-05-08 11:25:59,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +28: [2023-05-08 11:25:59,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +28: [2023-05-08 11:25:59,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +18: [2023-05-08 11:25:59,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 6: [2023-05-08 11:25:59,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +28: [2023-05-08 11:25:59,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +10: [2023-05-08 11:25:59,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:25:59,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +10: [2023-05-08 11:25:59,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +10: [2023-05-08 11:25:59,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +10: [2023-05-08 11:25:59,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +22: [2023-05-08 11:25:59,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:25:59,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +20: [2023-05-08 11:25:59,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +18: [2023-05-08 11:25:59,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +20: [2023-05-08 11:25:59,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +23: [2023-05-08 11:25:59,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +23: [2023-05-08 11:25:59,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +10: [2023-05-08 11:25:59,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +10: [2023-05-08 11:25:59,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 4: [2023-05-08 11:25:59,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 4: [2023-05-08 11:25:59,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 4: [2023-05-08 11:25:59,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +10: [2023-05-08 11:25:59,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +10: [2023-05-08 11:25:59,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +29: [2023-05-08 11:25:59,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:25:59,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +29: [2023-05-08 11:25:59,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 2: [2023-05-08 11:25:59,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +14: [2023-05-08 11:25:59,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +14: [2023-05-08 11:25:59,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +14: [2023-05-08 11:25:59,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +14: [2023-05-08 11:25:59,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +27: [2023-05-08 11:25:59,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 2: [2023-05-08 11:25:59,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +14: [2023-05-08 11:25:59,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:25:59,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +17: [2023-05-08 11:25:59,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:25:59,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +17: [2023-05-08 11:25:59,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +13: [2023-05-08 11:25:59,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +13: [2023-05-08 11:25:59,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +14: [2023-05-08 11:25:59,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +17: [2023-05-08 11:25:59,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +17: [2023-05-08 11:25:59,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 0: [2023-05-08 11:25:59,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 0: [2023-05-08 11:25:59,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 0: [2023-05-08 11:25:59,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 7: [2023-05-08 11:25:59,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 7: [2023-05-08 11:25:59,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 7: [2023-05-08 11:25:59,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +23: [2023-05-08 11:25:59,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +23: [2023-05-08 11:25:59,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +30: [2023-05-08 11:25:59,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +30: [2023-05-08 11:25:59,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +30: [2023-05-08 11:25:59,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 2: [2023-05-08 11:25:59,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +17: [2023-05-08 11:25:59,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 1: [2023-05-08 11:25:59,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 1: [2023-05-08 11:25:59,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +17: [2023-05-08 11:25:59,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 1: [2023-05-08 11:25:59,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +12: [2023-05-08 11:25:59,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 9: [2023-05-08 11:25:59,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +27: [2023-05-08 11:25:59,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:25:59,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 9: [2023-05-08 11:25:59,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +12: [2023-05-08 11:25:59,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +12: [2023-05-08 11:25:59,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +30: [2023-05-08 11:25:59,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:25:59,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 3: [2023-05-08 11:25:59,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 3: [2023-05-08 11:25:59,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 3: [2023-05-08 11:25:59,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +27: [2023-05-08 11:25:59,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 9: [2023-05-08 11:25:59,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 8: [2023-05-08 11:25:59,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 8: [2023-05-08 11:25:59,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 9: [2023-05-08 11:25:59,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 8: [2023-05-08 11:25:59,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 6: [2023-05-08 11:25:59,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 6: [2023-05-08 11:25:59,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:25:59,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:25:59,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:25:59,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:25:59,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +23: [2023-05-08 11:25:59,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +27: [2023-05-08 11:25:59,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 5: [2023-05-08 11:25:59,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:25:59,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:25:59,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +19: [2023-05-08 11:25:59,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +19: [2023-05-08 11:25:59,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +25: [2023-05-08 11:25:59,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +19: [2023-05-08 11:25:59,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +23: [2023-05-08 11:25:59,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +19: [2023-05-08 11:25:59,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +11: [2023-05-08 11:25:59,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:25:59,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:25:59,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +28: [2023-05-08 11:25:59,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +29: [2023-05-08 11:25:59,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +29: [2023-05-08 11:25:59,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +25: [2023-05-08 11:25:59,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +25: [2023-05-08 11:25:59,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:25:59,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:25:59,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +25: [2023-05-08 11:25:59,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +20: [2023-05-08 11:25:59,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +24: [2023-05-08 11:25:59,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +25: [2023-05-08 11:25:59,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:25:59,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:25:59,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +28: [2023-05-08 11:25:59,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +29: [2023-05-08 11:25:59,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:25:59,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +16: [2023-05-08 11:25:59,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +21: [2023-05-08 11:25:59,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +28: [2023-05-08 11:25:59,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:25:59,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +21: [2023-05-08 11:25:59,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +21: [2023-05-08 11:25:59,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +25: [2023-05-08 11:25:59,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +20: [2023-05-08 11:25:59,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 5: [2023-05-08 11:25:59,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +20: [2023-05-08 11:25:59,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +29: [2023-05-08 11:25:59,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 6: [2023-05-08 11:25:59,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +27: [2023-05-08 11:25:59,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:25:59,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +27: [2023-05-08 11:25:59,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +20: [2023-05-08 11:25:59,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +13: [2023-05-08 11:25:59,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +29: [2023-05-08 11:25:59,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +26: [2023-05-08 11:25:59,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +29: [2023-05-08 11:25:59,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +26: [2023-05-08 11:25:59,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +26: [2023-05-08 11:25:59,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +15: [2023-05-08 11:25:59,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +15: [2023-05-08 11:25:59,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +26: [2023-05-08 11:25:59,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +15: [2023-05-08 11:25:59,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +15: [2023-05-08 11:25:59,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 7: [2023-05-08 11:25:59,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:25:59,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +16: [2023-05-08 11:25:59,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +13: [2023-05-08 11:25:59,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +31: [2023-05-08 11:25:59,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 7: [2023-05-08 11:25:59,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:25:59,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +31: [2023-05-08 11:25:59,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +16: [2023-05-08 11:25:59,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +31: [2023-05-08 11:25:59,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. + 7: [2023-05-08 11:25:59,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +31: [2023-05-08 11:25:59,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt. +13: [2023-05-08 11:25:59,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +13: [2023-05-08 11:25:59,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:25:59,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:25:59,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:25:59,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +12: [2023-05-08 11:25:59,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:25:59,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:25:59,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:25:59,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:25:59,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 9: [2023-05-08 11:25:59,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +22: [2023-05-08 11:25:59,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 8: [2023-05-08 11:25:59,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:25:59,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +16: [2023-05-08 11:25:59,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +19: [2023-05-08 11:25:59,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +19: [2023-05-08 11:25:59,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +21: [2023-05-08 11:25:59,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +19: [2023-05-08 11:25:59,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:25:59,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:25:59,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +24: [2023-05-08 11:25:59,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:25:59,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +21: [2023-05-08 11:25:59,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +21: [2023-05-08 11:25:59,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +24: [2023-05-08 11:25:59,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +22: [2023-05-08 11:25:59,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:25:59,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +22: [2023-05-08 11:25:59,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +22: [2023-05-08 11:25:59,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +30: [2023-05-08 11:25:59,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:25:59,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +30: [2023-05-08 11:25:59,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +26: [2023-05-08 11:25:59,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +30: [2023-05-08 11:25:59,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:25:59,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +20: [2023-05-08 11:25:59,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:25:59,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 6: [2023-05-08 11:25:59,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +18: [2023-05-08 11:25:59,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +18: [2023-05-08 11:25:59,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +31: [2023-05-08 11:25:59,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:25:59,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +18: [2023-05-08 11:25:59,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +31: [2023-05-08 11:25:59,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +18: [2023-05-08 11:25:59,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +18: [2023-05-08 11:25:59,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +11: [2023-05-08 11:25:59,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:25:59,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +31: [2023-05-08 11:25:59,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +20: [2023-05-08 11:25:59,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +26: [2023-05-08 11:25:59,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:25:59,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +11: [2023-05-08 11:25:59,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:25:59,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:25:59,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +15: [2023-05-08 11:25:59,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +15: [2023-05-08 11:25:59,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... +31: [2023-05-08 11:25:59,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +11: [2023-05-08 11:25:59,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +30: [2023-05-08 11:25:59,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:25:59,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +24: [2023-05-08 11:25:59,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:25:59,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +24: [2023-05-08 11:25:59,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +24: [2023-05-08 11:25:59,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 6: [2023-05-08 11:25:59,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +30: [2023-05-08 11:25:59,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:25:59,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 6: [2023-05-08 11:25:59,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +28: [2023-05-08 11:25:59,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +20: [2023-05-08 11:25:59,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:25:59,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +30: [2023-05-08 11:25:59,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 9: [2023-05-08 11:25:59,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:25:59,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:25:59,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 9: [2023-05-08 11:25:59,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:25:59,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:25:59,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +20: [2023-05-08 11:25:59,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:25:59,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +15: [2023-05-08 11:25:59,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 9: [2023-05-08 11:25:59,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 9: [2023-05-08 11:25:59,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 9: [2023-05-08 11:25:59,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 1: [2023-05-08 11:25:59,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 1: [2023-05-08 11:25:59,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 7: [2023-05-08 11:25:59,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +18: [2023-05-08 11:25:59,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:25:59,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:25:59,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 8: [2023-05-08 11:25:59,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +18: [2023-05-08 11:25:59,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +28: [2023-05-08 11:25:59,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:25:59,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:25:59,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:25:59,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +28: [2023-05-08 11:25:59,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +28: [2023-05-08 11:25:59,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 6: [2023-05-08 11:25:59,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 7: [2023-05-08 11:25:59,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 6: [2023-05-08 11:25:59,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 3: [2023-05-08 11:25:59,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:25:59,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +13: [2023-05-08 11:25:59,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +21: [2023-05-08 11:25:59,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:25:59,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 1: [2023-05-08 11:25:59,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +26: [2023-05-08 11:25:59,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +28: [2023-05-08 11:25:59,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +24: [2023-05-08 11:25:59,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:25:59,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:25:59,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:25:59,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 8: [2023-05-08 11:25:59,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +21: [2023-05-08 11:25:59,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:25:59,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +21: [2023-05-08 11:25:59,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +21: [2023-05-08 11:25:59,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +28: [2023-05-08 11:25:59,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 1: [2023-05-08 11:25:59,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +24: [2023-05-08 11:25:59,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +12: [2023-05-08 11:25:59,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +12: [2023-05-08 11:25:59,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:25:59,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +20: [2023-05-08 11:25:59,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +12: [2023-05-08 11:25:59,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +31: [2023-05-08 11:25:59,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:25:59,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +26: [2023-05-08 11:25:59,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +20: [2023-05-08 11:25:59,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +31: [2023-05-08 11:25:59,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +12: [2023-05-08 11:25:59,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +12: [2023-05-08 11:25:59,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +31: [2023-05-08 11:25:59,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +31: [2023-05-08 11:25:59,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +31: [2023-05-08 11:25:59,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +31: [2023-05-08 11:25:59,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 3: [2023-05-08 11:25:59,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 3: [2023-05-08 11:25:59,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 3: [2023-05-08 11:25:59,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +19: [2023-05-08 11:25:59,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +19: [2023-05-08 11:25:59,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +12: [2023-05-08 11:25:59,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +19: [2023-05-08 11:25:59,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:25:59,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +15: [2023-05-08 11:25:59,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +19: [2023-05-08 11:25:59,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +26: [2023-05-08 11:25:59,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +15: [2023-05-08 11:25:59,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +15: [2023-05-08 11:25:59,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +26: [2023-05-08 11:25:59,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +15: [2023-05-08 11:25:59,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +12: [2023-05-08 11:25:59,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +12: [2023-05-08 11:25:59,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +15: [2023-05-08 11:25:59,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +15: [2023-05-08 11:25:59,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +15: [2023-05-08 11:25:59,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +15: [2023-05-08 11:25:59,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +26: [2023-05-08 11:25:59,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. +31: [2023-05-08 11:25:59,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +26: [2023-05-08 11:25:59,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +31: [2023-05-08 11:25:59,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... + 8: [2023-05-08 11:25:59,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt... +17: [2023-05-08 11:25:59,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +17: [2023-05-08 11:25:59,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +17: [2023-05-08 11:25:59,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:25:59,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:25:59,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +17: [2023-05-08 11:25:59,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:25:59,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +17: [2023-05-08 11:25:59,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:25:59,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +31: [2023-05-08 11:25:59,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +31: [2023-05-08 11:25:59,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +31: [2023-05-08 11:25:59,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:25:59,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:25:59,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:25:59,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:25:59,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +31: [2023-05-08 11:25:59,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:25:59,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:25:59,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +31: [2023-05-08 11:25:59,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:25:59,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:25:59,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:25:59,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:25:59,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:25:59,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:25:59,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:25:59,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:25:59,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:25:59,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:25:59,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:25:59,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:25:59,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:25:59,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:25:59,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:25:59,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +17: [2023-05-08 11:25:59,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:25:59,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:25:59,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:25:59,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:25:59,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +29: [2023-05-08 11:25:59,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +29: [2023-05-08 11:25:59,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +29: [2023-05-08 11:25:59,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +17: [2023-05-08 11:25:59,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:25:59,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:25:59,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:25:59,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:25:59,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:25:59,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:25:59,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:25:59,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:25:59,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:25:59,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:25:59,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +17: [2023-05-08 11:25:59,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:25:59,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:25:59,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:25:59,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:25:59,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:25:59,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:25:59,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:25:59,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:25:59,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:25:59,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:25:59,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:25:59,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:25:59,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:25:59,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:25:59,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:25:59,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:25:59,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +12: [2023-05-08 11:25:59,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +12: [2023-05-08 11:25:59,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +12: [2023-05-08 11:25:59,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:25:59,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +29: [2023-05-08 11:25:59,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:25:59,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:25:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:25:59,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:25:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:25:59,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:25:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:25:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:25:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:25:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:25:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:25:59,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:25:59,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:25:59,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:25:59,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +18: [2023-05-08 11:25:59,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:25:59,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +18: [2023-05-08 11:25:59,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:25:59,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:25:59,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +14: [2023-05-08 11:25:59,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:25:59,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +14: [2023-05-08 11:25:59,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +14: [2023-05-08 11:25:59,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +14: [2023-05-08 11:25:59,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:25:59,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:25:59,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +14: [2023-05-08 11:25:59,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +14: [2023-05-08 11:25:59,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:25:59,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:25:59,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:25:59,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +18: [2023-05-08 11:25:59,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:25:59,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:25:59,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:25:59,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:25:59,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:25:59,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:25:59,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +14: [2023-05-08 11:25:59,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:25:59,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:25:59,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +14: [2023-05-08 11:25:59,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:25:59,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:25:59,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:25:59,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:25:59,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:25:59,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:25:59,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:25:59,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:25:59,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:25:59,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +25: [2023-05-08 11:25:59,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +20: [2023-05-08 11:25:59,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:25:59,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:25:59,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +20: [2023-05-08 11:25:59,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:25:59,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:25:59,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +20: [2023-05-08 11:25:59,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:25:59,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:25:59,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:25:59,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:25:59,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +14: [2023-05-08 11:25:59,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:25:59,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:25:59,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +14: [2023-05-08 11:25:59,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:25:59,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:25:59,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:25:59,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:25:59,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:25:59,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +25: [2023-05-08 11:25:59,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:25:59,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +18: [2023-05-08 11:25:59,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:25:59,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:25:59,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:25:59,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +31: [2023-05-08 11:25:59,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +31: [2023-05-08 11:25:59,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +31: [2023-05-08 11:25:59,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +31: [2023-05-08 11:25:59,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +31: [2023-05-08 11:25:59,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:25:59,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:25:59,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:25:59,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +18: [2023-05-08 11:25:59,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:25:59,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 7: [2023-05-08 11:25:59,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:25:59,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:25:59,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:25:59,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:25:59,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:25:59,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +14: [2023-05-08 11:25:59,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +21: [2023-05-08 11:25:59,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 2: [2023-05-08 11:25:59,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:25:59,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:25:59,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:25:59,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:25:59,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:25:59,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:25:59,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +21: [2023-05-08 11:25:59,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +14: [2023-05-08 11:25:59,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:25:59,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:25:59,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:25:59,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:25:59,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:25:59,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:25:59,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +21: [2023-05-08 11:25:59,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:25:59,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:25:59,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:25:59,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +14: [2023-05-08 11:25:59,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:25:59,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:25:59,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:25:59,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:25:59,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:25:59,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:25:59,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:25:59,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +14: [2023-05-08 11:25:59,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:25:59,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +21: [2023-05-08 11:25:59,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +14: [2023-05-08 11:25:59,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +21: [2023-05-08 11:25:59,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:25:59,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +18: [2023-05-08 11:25:59,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +21: [2023-05-08 11:25:59,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +14: [2023-05-08 11:25:59,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:25:59,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:25:59,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +16: [2023-05-08 11:25:59,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +16: [2023-05-08 11:25:59,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:25:59,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +16: [2023-05-08 11:25:59,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +16: [2023-05-08 11:25:59,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +16: [2023-05-08 11:25:59,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +16: [2023-05-08 11:25:59,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +16: [2023-05-08 11:25:59,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +16: [2023-05-08 11:25:59,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +28: [2023-05-08 11:25:59,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +28: [2023-05-08 11:25:59,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +28: [2023-05-08 11:25:59,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +28: [2023-05-08 11:25:59,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +19: [2023-05-08 11:25:59,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +19: [2023-05-08 11:25:59,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +19: [2023-05-08 11:25:59,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +19: [2023-05-08 11:25:59,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +19: [2023-05-08 11:25:59,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +19: [2023-05-08 11:25:59,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +19: [2023-05-08 11:25:59,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +19: [2023-05-08 11:25:59,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +10: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +10: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +10: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +10: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +10: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +10: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +10: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +22: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:25:59,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +28: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +28: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +28: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +28: [2023-05-08 11:25:59,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:25:59,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +30: [2023-05-08 11:25:59,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +30: [2023-05-08 11:25:59,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:25:59,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:25:59,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +30: [2023-05-08 11:25:59,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +26: [2023-05-08 11:25:59,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +26: [2023-05-08 11:25:59,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +26: [2023-05-08 11:25:59,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +26: [2023-05-08 11:25:59,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +26: [2023-05-08 11:25:59,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +26: [2023-05-08 11:25:59,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:25:59,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +26: [2023-05-08 11:25:59,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +10: [2023-05-08 11:25:59,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +11: [2023-05-08 11:25:59,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +11: [2023-05-08 11:25:59,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +11: [2023-05-08 11:25:59,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +11: [2023-05-08 11:25:59,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +11: [2023-05-08 11:25:59,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +11: [2023-05-08 11:25:59,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +11: [2023-05-08 11:25:59,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +26: [2023-05-08 11:25:59,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:25:59,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:25:59,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +11: [2023-05-08 11:25:59,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +16: [2023-05-08 11:25:59,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +16: [2023-05-08 11:25:59,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +24: [2023-05-08 11:25:59,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +16: [2023-05-08 11:25:59,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +16: [2023-05-08 11:25:59,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +19: [2023-05-08 11:25:59,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +19: [2023-05-08 11:25:59,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +10: [2023-05-08 11:25:59,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +26: [2023-05-08 11:25:59,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +10: [2023-05-08 11:25:59,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +11: [2023-05-08 11:25:59,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +30: [2023-05-08 11:25:59,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:25:59,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +26: [2023-05-08 11:25:59,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +11: [2023-05-08 11:25:59,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +19: [2023-05-08 11:25:59,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +19: [2023-05-08 11:25:59,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +11: [2023-05-08 11:25:59,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:25:59,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:25:59,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:25:59,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:25:59,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +24: [2023-05-08 11:25:59,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:25:59,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:25:59,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +30: [2023-05-08 11:25:59,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +16: [2023-05-08 11:25:59,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:25:59,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:25:59,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +16: [2023-05-08 11:25:59,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +26: [2023-05-08 11:25:59,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:25:59,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +26: [2023-05-08 11:25:59,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +10: [2023-05-08 11:25:59,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +22: [2023-05-08 11:25:59,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +16: [2023-05-08 11:25:59,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +16: [2023-05-08 11:25:59,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:25:59,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:25:59,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:25:59,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:25:59,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +30: [2023-05-08 11:25:59,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +11: [2023-05-08 11:25:59,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +26: [2023-05-08 11:25:59,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +11: [2023-05-08 11:25:59,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +10: [2023-05-08 11:25:59,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:25:59,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:25:59,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:25:59,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 8: [2023-05-08 11:25:59,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +26: [2023-05-08 11:25:59,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:25:59,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +10: [2023-05-08 11:25:59,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:25:59,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +10: [2023-05-08 11:25:59,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:25:59,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +11: [2023-05-08 11:25:59,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +30: [2023-05-08 11:25:59,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +26: [2023-05-08 11:25:59,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:25:59,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +30: [2023-05-08 11:25:59,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +26: [2023-05-08 11:25:59,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +10: [2023-05-08 11:25:59,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 3: [2023-05-08 11:25:59,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:25:59,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:25:59,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +30: [2023-05-08 11:25:59,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 9: [2023-05-08 11:25:59,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +22: [2023-05-08 11:25:59,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:25:59,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:25:59,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 1: [2023-05-08 11:25:59,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:25:59,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:25:59,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:25:59,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 0: [2023-05-08 11:25:59,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +10: [2023-05-08 11:25:59,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:25:59,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +28: [2023-05-08 11:25:59,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +28: [2023-05-08 11:25:59,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +28: [2023-05-08 11:25:59,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +28: [2023-05-08 11:25:59,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:25:59,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:25:59,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:25:59,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 6: [2023-05-08 11:25:59,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:25:59,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 9: [2023-05-08 11:25:59,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:25:59,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:25:59,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:25:59,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:26:00,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:26:00,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:26:00,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:26:00,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:26:00,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:26:00,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:26:00,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:26:00,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:26:00,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:26:00,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:26:00,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:26:00,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:26:00,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:26:00,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:26:00,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:26:00,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:26:00,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:26:00,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:26:00,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:26:00,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:26:00,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:26:00,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:26:00,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:26:00,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:26:00,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:26:00,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:26:00,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:26:00,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:26:00,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +17: [2023-05-08 11:26:00,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +17: [2023-05-08 11:26:00,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +17: [2023-05-08 11:26:00,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +29: [2023-05-08 11:26:00,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:26:00,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:26:00,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:26:00,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:26:00,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:26:00,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:26:00,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:26:00,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:26:00,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:26:00,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:26:00,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:26:00,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:26:00,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:26:00,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +29: [2023-05-08 11:26:00,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:26:00,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:26:00,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:26:00,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +17: [2023-05-08 11:26:00,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:26:00,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:26:00,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 2: [2023-05-08 11:26:00,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:26:00,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:26:00,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:26:00,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:26:00,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:26:00,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +17: [2023-05-08 11:26:00,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +11: [2023-05-08 11:26:00,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +11: [2023-05-08 11:26:00,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +11: [2023-05-08 11:26:00,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +11: [2023-05-08 11:26:00,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 3: [2023-05-08 11:26:00,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 3: [2023-05-08 11:26:00,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 3: [2023-05-08 11:26:00,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 3: [2023-05-08 11:26:00,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 7: [2023-05-08 11:26:00,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:26:00,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +23: [2023-05-08 11:26:00,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +23: [2023-05-08 11:26:00,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +23: [2023-05-08 11:26:00,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:26:00,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:26:00,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 2: [2023-05-08 11:26:00,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +23: [2023-05-08 11:26:00,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:26:00,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:26:00,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:26:00,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:26:00,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +17: [2023-05-08 11:26:00,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +25: [2023-05-08 11:26:00,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:26:00,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:26:00,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +25: [2023-05-08 11:26:00,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:26:00,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:26:00,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:26:00,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:26:00,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +21: [2023-05-08 11:26:00,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +21: [2023-05-08 11:26:00,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +21: [2023-05-08 11:26:00,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:26:00,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +21: [2023-05-08 11:26:00,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +21: [2023-05-08 11:26:00,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +29: [2023-05-08 11:26:00,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +29: [2023-05-08 11:26:00,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +29: [2023-05-08 11:26:00,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 7: [2023-05-08 11:26:00,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +21: [2023-05-08 11:26:00,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +29: [2023-05-08 11:26:00,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +21: [2023-05-08 11:26:00,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +21: [2023-05-08 11:26:00,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +21: [2023-05-08 11:26:00,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +21: [2023-05-08 11:26:00,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:26:00,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +21: [2023-05-08 11:26:00,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +21: [2023-05-08 11:26:00,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +14: [2023-05-08 11:26:00,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +14: [2023-05-08 11:26:00,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +14: [2023-05-08 11:26:00,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 7: [2023-05-08 11:26:00,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:26:00,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +25: [2023-05-08 11:26:00,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +14: [2023-05-08 11:26:00,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +15: [2023-05-08 11:26:00,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +15: [2023-05-08 11:26:00,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +15: [2023-05-08 11:26:00,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +25: [2023-05-08 11:26:00,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +15: [2023-05-08 11:26:00,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +25: [2023-05-08 11:26:00,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +17: [2023-05-08 11:26:00,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +11: [2023-05-08 11:26:00,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 7: [2023-05-08 11:26:00,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +11: [2023-05-08 11:26:00,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +17: [2023-05-08 11:26:00,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:26:00,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +17: [2023-05-08 11:26:00,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +11: [2023-05-08 11:26:00,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +23: [2023-05-08 11:26:00,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +23: [2023-05-08 11:26:00,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 3: [2023-05-08 11:26:00,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +11: [2023-05-08 11:26:00,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +20: [2023-05-08 11:26:00,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +23: [2023-05-08 11:26:00,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +20: [2023-05-08 11:26:00,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 4: [2023-05-08 11:26:00,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:26:00,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +20: [2023-05-08 11:26:00,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 4: [2023-05-08 11:26:00,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:26:00,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:26:00,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +29: [2023-05-08 11:26:00,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:26:00,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +14: [2023-05-08 11:26:00,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:26:00,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:26:00,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:26:00,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +14: [2023-05-08 11:26:00,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:26:00,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:26:00,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +14: [2023-05-08 11:26:00,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +14: [2023-05-08 11:26:00,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:26:00,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:26:00,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:26:00,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:26:00,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:26:00,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:26:00,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:26:00,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +15: [2023-05-08 11:26:00,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:26:00,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:26:00,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:26:00,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +21: [2023-05-08 11:26:00,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +11: [2023-05-08 11:26:00,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:26:00,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +20: [2023-05-08 11:26:00,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:26:00,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:26:00,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +25: [2023-05-08 11:26:00,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +11: [2023-05-08 11:26:00,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 2: [2023-05-08 11:26:00,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +11: [2023-05-08 11:26:00,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +11: [2023-05-08 11:26:00,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 4: [2023-05-08 11:26:00,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 9: [2023-05-08 11:26:00,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 9: [2023-05-08 11:26:00,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 9: [2023-05-08 11:26:00,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 4: [2023-05-08 11:26:00,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +20: [2023-05-08 11:26:00,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +11: [2023-05-08 11:26:00,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +11: [2023-05-08 11:26:00,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +23: [2023-05-08 11:26:00,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +14: [2023-05-08 11:26:00,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +14: [2023-05-08 11:26:00,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +14: [2023-05-08 11:26:00,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +14: [2023-05-08 11:26:00,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +23: [2023-05-08 11:26:00,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:26:00,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +21: [2023-05-08 11:26:00,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +21: [2023-05-08 11:26:00,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +15: [2023-05-08 11:26:00,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:26:00,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +14: [2023-05-08 11:26:00,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +14: [2023-05-08 11:26:00,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:26:00,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:26:00,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +14: [2023-05-08 11:26:00,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +14: [2023-05-08 11:26:00,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 4: [2023-05-08 11:26:00,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 4: [2023-05-08 11:26:00,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 4: [2023-05-08 11:26:00,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +20: [2023-05-08 11:26:00,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +11: [2023-05-08 11:26:00,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:26:00,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:26:00,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 9: [2023-05-08 11:26:00,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:26:00,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:26:00,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +15: [2023-05-08 11:26:00,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +15: [2023-05-08 11:26:00,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +15: [2023-05-08 11:26:00,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +11: [2023-05-08 11:26:00,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +15: [2023-05-08 11:26:00,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +21: [2023-05-08 11:26:00,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +21: [2023-05-08 11:26:00,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +25: [2023-05-08 11:26:00,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:26:00,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:26:00,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +29: [2023-05-08 11:26:00,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:26:00,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:26:00,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +29: [2023-05-08 11:26:00,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +20: [2023-05-08 11:26:00,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:26:00,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 7: [2023-05-08 11:26:00,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +25: [2023-05-08 11:26:00,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +29: [2023-05-08 11:26:00,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:26:00,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +21: [2023-05-08 11:26:00,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:26:00,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 9: [2023-05-08 11:26:00,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +23: [2023-05-08 11:26:00,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:26:00,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +23: [2023-05-08 11:26:00,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +25: [2023-05-08 11:26:00,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:26:00,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +23: [2023-05-08 11:26:00,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:26:00,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:26:00,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +23: [2023-05-08 11:26:00,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:26:00,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +25: [2023-05-08 11:26:00,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +25: [2023-05-08 11:26:00,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +25: [2023-05-08 11:26:00,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +21: [2023-05-08 11:26:00,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +21: [2023-05-08 11:26:00,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 9: [2023-05-08 11:26:00,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +21: [2023-05-08 11:26:00,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +23: [2023-05-08 11:26:00,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:26:00,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:26:00,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:26:00,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +23: [2023-05-08 11:26:00,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +25: [2023-05-08 11:26:00,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:26:00,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +23: [2023-05-08 11:26:00,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:26:00,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:26:00,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 9: [2023-05-08 11:26:00,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +20: [2023-05-08 11:26:00,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +20: [2023-05-08 11:26:00,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +21: [2023-05-08 11:26:00,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +21: [2023-05-08 11:26:00,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +23: [2023-05-08 11:26:00,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +21: [2023-05-08 11:26:00,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +21: [2023-05-08 11:26:00,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 9: [2023-05-08 11:26:00,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +23: [2023-05-08 11:26:00,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:26:00,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 5: [2023-05-08 11:26:00,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:26:00,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:26:00,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +20: [2023-05-08 11:26:00,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:26:00,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:26:00,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:26:00,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:26:00,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:26:00,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +20: [2023-05-08 11:26:00,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:26:00,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +20: [2023-05-08 11:26:00,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +20: [2023-05-08 11:26:00,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:26:00,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:26:00,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +20: [2023-05-08 11:26:00,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:26:00,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:26:00,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:26:00,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:26:00,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:26:00,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:26:00,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:26:00,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:26:00,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:26:00,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:26:00,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:26:00,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:26:00,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:26:00,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:26:00,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +24: [2023-05-08 11:26:00,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +24: [2023-05-08 11:26:00,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +24: [2023-05-08 11:26:00,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:26:00,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:26:00,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +24: [2023-05-08 11:26:00,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:26:00,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +24: [2023-05-08 11:26:00,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:26:00,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:26:00,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:26:00,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:26:00,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:26:00,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:26:00,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:26:00,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:26:00,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +18: [2023-05-08 11:26:00,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:26:00,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:26:00,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:26:00,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:26:00,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:26:00,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +11: [2023-05-08 11:26:00,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +11: [2023-05-08 11:26:00,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +11: [2023-05-08 11:26:00,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:26:00,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:26:00,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:26:00,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:26:00,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:26:00,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +11: [2023-05-08 11:26:00,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +15: [2023-05-08 11:26:00,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +15: [2023-05-08 11:26:00,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +15: [2023-05-08 11:26:00,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:26:00,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +28: [2023-05-08 11:26:00,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:26:00,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:26:00,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:26:00,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:26:00,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +15: [2023-05-08 11:26:00,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +11: [2023-05-08 11:26:00,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:26:00,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +11: [2023-05-08 11:26:00,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:26:00,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:26:00,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +28: [2023-05-08 11:26:00,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:26:00,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:26:00,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:26:00,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:26:00,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:26:00,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:26:00,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:26:00,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +16: [2023-05-08 11:26:00,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:26:00,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:26:00,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:26:00,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +16: [2023-05-08 11:26:00,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +16: [2023-05-08 11:26:00,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:26:00,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:26:00,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:26:00,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:26:00,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:26:00,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:26:00,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:26:00,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:26:00,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:26:00,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:26:00,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:26:00,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:26:00,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:26:00,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:26:00,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:26:00,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +30: [2023-05-08 11:26:00,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +20: [2023-05-08 11:26:00,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:26:00,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +12: [2023-05-08 11:26:00,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:26:00,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:26:00,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:26:00,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +24: [2023-05-08 11:26:00,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:26:00,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +30: [2023-05-08 11:26:00,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:26:00,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:26:00,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +13: [2023-05-08 11:26:00,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +11: [2023-05-08 11:26:00,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +28: [2023-05-08 11:26:00,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +24: [2023-05-08 11:26:00,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +14: [2023-05-08 11:26:00,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:26:00,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:26:00,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:26:00,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +28: [2023-05-08 11:26:00,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +28: [2023-05-08 11:26:00,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +15: [2023-05-08 11:26:00,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:26:00,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:26:00,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:26:00,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:26:00,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:26:00,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +14: [2023-05-08 11:26:00,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:26:00,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:26:00,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:26:00,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:26:00,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:26:00,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:26:00,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +11: [2023-05-08 11:26:00,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +24: [2023-05-08 11:26:00,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:26:00,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +27: [2023-05-08 11:26:00,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +27: [2023-05-08 11:26:00,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +12: [2023-05-08 11:26:00,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:26:00,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:26:00,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:26:00,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:26:00,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:26:00,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +27: [2023-05-08 11:26:00,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:26:00,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +24: [2023-05-08 11:26:00,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +12: [2023-05-08 11:26:00,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:26:00,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:26:00,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:26:00,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:26:00,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +15: [2023-05-08 11:26:00,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +11: [2023-05-08 11:26:00,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:26:00,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +13: [2023-05-08 11:26:00,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +15: [2023-05-08 11:26:00,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +14: [2023-05-08 11:26:00,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:26:00,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:26:00,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 9: [2023-05-08 11:26:00,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +13: [2023-05-08 11:26:00,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:26:00,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +16: [2023-05-08 11:26:00,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +13: [2023-05-08 11:26:00,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:26:00,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:26:00,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:26:00,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:26:00,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +30: [2023-05-08 11:26:00,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +14: [2023-05-08 11:26:00,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +24: [2023-05-08 11:26:00,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:26:00,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +30: [2023-05-08 11:26:00,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 8: [2023-05-08 11:26:00,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 8: [2023-05-08 11:26:00,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +30: [2023-05-08 11:26:00,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 8: [2023-05-08 11:26:00,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 8: [2023-05-08 11:26:00,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +26: [2023-05-08 11:26:00,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +26: [2023-05-08 11:26:00,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +26: [2023-05-08 11:26:00,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +30: [2023-05-08 11:26:00,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:26:00,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +15: [2023-05-08 11:26:00,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:26:00,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +15: [2023-05-08 11:26:00,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:26:00,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +12: [2023-05-08 11:26:00,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:26:00,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +22: [2023-05-08 11:26:00,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:26:00,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +22: [2023-05-08 11:26:00,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +11: [2023-05-08 11:26:00,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:26:00,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:26:00,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +15: [2023-05-08 11:26:00,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:26:00,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +22: [2023-05-08 11:26:00,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +22: [2023-05-08 11:26:00,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:26:00,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:26:00,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:26:00,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:26:00,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:26:00,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +11: [2023-05-08 11:26:00,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:26:00,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +12: [2023-05-08 11:26:00,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:26:00,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:26:00,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +27: [2023-05-08 11:26:00,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:26:00,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:26:00,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:26:00,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:26:00,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +10: [2023-05-08 11:26:00,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:26:00,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:26:00,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +10: [2023-05-08 11:26:00,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +10: [2023-05-08 11:26:00,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:26:00,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:26:00,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +16: [2023-05-08 11:26:00,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:26:00,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +16: [2023-05-08 11:26:00,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +26: [2023-05-08 11:26:00,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +14: [2023-05-08 11:26:00,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +14: [2023-05-08 11:26:00,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:26:00,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +15: [2023-05-08 11:26:00,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:26:00,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:26:00,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:26:00,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:26:00,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:26:00,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +30: [2023-05-08 11:26:00,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +14: [2023-05-08 11:26:00,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:26:00,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:26:00,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:26:00,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +16: [2023-05-08 11:26:00,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +12: [2023-05-08 11:26:00,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +26: [2023-05-08 11:26:00,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:26:00,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:26:00,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:26:00,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:26:00,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +26: [2023-05-08 11:26:00,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:26:00,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +10: [2023-05-08 11:26:00,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:26:00,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:26:00,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:26:00,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:26:00,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:26:00,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:26:00,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:26:00,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +10: [2023-05-08 11:26:00,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:26:00,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:26:00,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +26: [2023-05-08 11:26:00,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +30: [2023-05-08 11:26:00,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:26:00,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +12: [2023-05-08 11:26:00,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +12: [2023-05-08 11:26:00,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +26: [2023-05-08 11:26:00,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:26:00,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +16: [2023-05-08 11:26:00,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +16: [2023-05-08 11:26:00,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +16: [2023-05-08 11:26:00,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:26:00,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:26:00,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:26:00,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:26:00,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:26:00,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:26:00,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +16: [2023-05-08 11:26:00,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +29: [2023-05-08 11:26:00,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:26:00,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:26:00,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +26: [2023-05-08 11:26:00,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +26: [2023-05-08 11:26:00,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +29: [2023-05-08 11:26:00,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:26:00,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +30: [2023-05-08 11:26:00,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +30: [2023-05-08 11:26:00,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +30: [2023-05-08 11:26:00,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +16: [2023-05-08 11:26:00,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +16: [2023-05-08 11:26:00,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +22: [2023-05-08 11:26:00,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:26:00,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:26:00,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +26: [2023-05-08 11:26:00,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +10: [2023-05-08 11:26:00,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +10: [2023-05-08 11:26:00,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 8: [2023-05-08 11:26:00,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +30: [2023-05-08 11:26:00,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +30: [2023-05-08 11:26:00,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +12: [2023-05-08 11:26:00,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:26:00,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 8: [2023-05-08 11:26:00,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +26: [2023-05-08 11:26:00,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:26:00,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +12: [2023-05-08 11:26:00,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +10: [2023-05-08 11:26:00,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:26:00,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:26:00,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:26:00,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:26:00,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:26:00,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:26:00,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:26:00,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:26:00,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:26:00,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:26:00,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:26:00,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:26:00,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:26:00,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:26:00,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +26: [2023-05-08 11:26:00,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:26:00,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:26:00,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +12: [2023-05-08 11:26:00,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:26:00,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +26: [2023-05-08 11:26:00,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:26:00,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:26:00,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 8: [2023-05-08 11:26:00,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:26:00,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 6: [2023-05-08 11:26:00,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +22: [2023-05-08 11:26:00,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +22: [2023-05-08 11:26:00,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +22: [2023-05-08 11:26:00,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +22: [2023-05-08 11:26:00,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 6: [2023-05-08 11:26:00,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 8: [2023-05-08 11:26:00,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 4: [2023-05-08 11:26:00,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:26:00,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:26:00,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:26:00,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:26:00,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:26:00,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +12: [2023-05-08 11:26:00,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +12: [2023-05-08 11:26:00,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +26: [2023-05-08 11:26:00,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:26:00,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:26:00,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:26:00,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:26:00,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:26:00,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:26:00,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:26:00,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:26:00,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:26:00,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:26:00,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:26:00,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +31: [2023-05-08 11:26:00,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +31: [2023-05-08 11:26:00,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +26: [2023-05-08 11:26:00,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:26:00,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:26:00,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +31: [2023-05-08 11:26:00,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +10: [2023-05-08 11:26:00,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +10: [2023-05-08 11:26:00,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +10: [2023-05-08 11:26:00,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +10: [2023-05-08 11:26:00,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +13: [2023-05-08 11:26:00,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +13: [2023-05-08 11:26:00,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +13: [2023-05-08 11:26:00,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +13: [2023-05-08 11:26:00,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +12: [2023-05-08 11:26:00,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +29: [2023-05-08 11:26:00,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:26:00,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +24: [2023-05-08 11:26:00,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +28: [2023-05-08 11:26:00,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +28: [2023-05-08 11:26:00,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +28: [2023-05-08 11:26:00,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +27: [2023-05-08 11:26:00,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +27: [2023-05-08 11:26:00,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +27: [2023-05-08 11:26:00,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +24: [2023-05-08 11:26:00,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 5: [2023-05-08 11:26:00,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 5: [2023-05-08 11:26:00,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +28: [2023-05-08 11:26:00,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 5: [2023-05-08 11:26:00,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +24: [2023-05-08 11:26:00,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +27: [2023-05-08 11:26:00,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 5: [2023-05-08 11:26:00,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +24: [2023-05-08 11:26:00,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 1: [2023-05-08 11:26:00,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 1: [2023-05-08 11:26:00,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 1: [2023-05-08 11:26:00,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 0: [2023-05-08 11:26:00,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 0: [2023-05-08 11:26:00,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 0: [2023-05-08 11:26:00,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 1: [2023-05-08 11:26:00,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +19: [2023-05-08 11:26:00,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 0: [2023-05-08 11:26:00,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +19: [2023-05-08 11:26:00,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +19: [2023-05-08 11:26:00,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +12: [2023-05-08 11:26:00,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +19: [2023-05-08 11:26:00,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 2: [2023-05-08 11:26:00,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 2: [2023-05-08 11:26:00,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. + 2: [2023-05-08 11:26:00,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +31: [2023-05-08 11:26:00,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +31: [2023-05-08 11:26:00,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +18: [2023-05-08 11:26:00,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +18: [2023-05-08 11:26:00,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +18: [2023-05-08 11:26:00,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +31: [2023-05-08 11:26:00,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +18: [2023-05-08 11:26:00,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt. +31: [2023-05-08 11:26:00,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +29: [2023-05-08 11:26:00,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +21: [2023-05-08 11:26:00,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +21: [2023-05-08 11:26:00,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +21: [2023-05-08 11:26:00,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +21: [2023-05-08 11:26:00,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +21: [2023-05-08 11:26:00,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:26:00,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:26:00,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:26:00,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +10: [2023-05-08 11:26:00,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +10: [2023-05-08 11:26:00,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:26:00,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +13: [2023-05-08 11:26:00,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +10: [2023-05-08 11:26:00,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:26:00,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +24: [2023-05-08 11:26:00,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +27: [2023-05-08 11:26:00,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:26:00,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:26:00,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:26:00,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +28: [2023-05-08 11:26:00,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +28: [2023-05-08 11:26:00,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +13: [2023-05-08 11:26:00,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +24: [2023-05-08 11:26:00,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:26:00,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:26:00,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +24: [2023-05-08 11:26:00,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +24: [2023-05-08 11:26:00,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +21: [2023-05-08 11:26:00,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:26:00,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:26:00,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +22: [2023-05-08 11:26:00,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:26:00,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +22: [2023-05-08 11:26:00,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +22: [2023-05-08 11:26:00,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +22: [2023-05-08 11:26:00,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +10: [2023-05-08 11:26:00,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +27: [2023-05-08 11:26:00,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +27: [2023-05-08 11:26:00,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +19: [2023-05-08 11:26:00,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +28: [2023-05-08 11:26:00,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +19: [2023-05-08 11:26:00,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +19: [2023-05-08 11:26:00,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +19: [2023-05-08 11:26:00,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 2: [2023-05-08 11:26:00,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:26:00,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:26:00,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:26:00,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +18: [2023-05-08 11:26:00,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt... +31: [2023-05-08 11:26:00,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:26:00,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +31: [2023-05-08 11:26:00,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +27: [2023-05-08 11:26:00,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +25: [2023-05-08 11:26:00,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:26:00,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +25: [2023-05-08 11:26:00,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:26:00,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:26:00,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:26:00,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +25: [2023-05-08 11:26:00,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:26:00,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +25: [2023-05-08 11:26:00,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:26:00,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +13: [2023-05-08 11:26:00,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:26:00,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +21: [2023-05-08 11:26:00,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +10: [2023-05-08 11:26:00,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:26:00,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:26:00,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +10: [2023-05-08 11:26:00,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +10: [2023-05-08 11:26:00,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +10: [2023-05-08 11:26:00,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 2: [2023-05-08 11:26:00,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +13: [2023-05-08 11:26:00,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:26:00,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 1: [2023-05-08 11:26:00,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +28: [2023-05-08 11:26:00,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:26:00,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:26:00,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 6: [2023-05-08 11:26:00,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 1: [2023-05-08 11:26:00,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +28: [2023-05-08 11:26:00,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 1: [2023-05-08 11:26:00,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +21: [2023-05-08 11:26:00,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +21: [2023-05-08 11:26:00,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +19: [2023-05-08 11:26:00,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +19: [2023-05-08 11:26:00,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 5: [2023-05-08 11:26:00,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +31: [2023-05-08 11:26:00,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +31: [2023-05-08 11:26:00,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +13: [2023-05-08 11:26:00,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +13: [2023-05-08 11:26:00,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 7: [2023-05-08 11:26:00,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +18: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 0: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 2: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:26:00,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:26:00,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 5: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +18: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 1: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 7: [2023-05-08 11:26:00,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 2: [2023-05-08 11:26:00,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +18: [2023-05-08 11:26:00,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +28: [2023-05-08 11:26:00,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:26:00,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +18: [2023-05-08 11:26:00,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +28: [2023-05-08 11:26:00,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:26:00,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +28: [2023-05-08 11:26:00,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +28: [2023-05-08 11:26:00,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +24: [2023-05-08 11:26:00,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:26:00,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:26:00,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +24: [2023-05-08 11:26:00,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +24: [2023-05-08 11:26:00,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +24: [2023-05-08 11:26:00,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 5: [2023-05-08 11:26:00,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +21: [2023-05-08 11:26:00,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +25: [2023-05-08 11:26:00,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +21: [2023-05-08 11:26:00,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +21: [2023-05-08 11:26:00,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +18: [2023-05-08 11:26:00,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +18: [2023-05-08 11:26:00,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +27: [2023-05-08 11:26:00,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +27: [2023-05-08 11:26:00,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +31: [2023-05-08 11:26:00,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +10: [2023-05-08 11:26:00,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +31: [2023-05-08 11:26:00,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +21: [2023-05-08 11:26:00,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +10: [2023-05-08 11:26:00,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +19: [2023-05-08 11:26:00,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +19: [2023-05-08 11:26:00,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +19: [2023-05-08 11:26:00,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +19: [2023-05-08 11:26:00,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +19: [2023-05-08 11:26:00,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +19: [2023-05-08 11:26:00,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +10: [2023-05-08 11:26:00,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +10: [2023-05-08 11:26:00,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 7: [2023-05-08 11:26:00,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:26:00,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:26:00,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:26:00,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:26:00,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. +28: [2023-05-08 11:26:00,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 2: [2023-05-08 11:26:00,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:26:00,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... +21: [2023-05-08 11:26:00,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +18: [2023-05-08 11:26:00,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +18: [2023-05-08 11:26:00,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt... + 7: [2023-05-08 11:26:00,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +25: [2023-05-08 11:26:00,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +25: [2023-05-08 11:26:00,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +25: [2023-05-08 11:26:00,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:26:00,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:26:00,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +17: [2023-05-08 11:26:00,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:26:00,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:26:00,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +17: [2023-05-08 11:26:00,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:26:00,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:26:00,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:26:00,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:26:00,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:26:00,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:26:00,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:26:00,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:26:00,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:26:00,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:26:00,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:26:00,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:26:00,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:26:00,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:26:00,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:26:00,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:26:00,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:26:00,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:26:00,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:26:00,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +13: [2023-05-08 11:26:00,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:26:00,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +13: [2023-05-08 11:26:00,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:26:00,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +13: [2023-05-08 11:26:00,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:26:00,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:26:00,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +13: [2023-05-08 11:26:00,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +13: [2023-05-08 11:26:00,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:26:00,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +14: [2023-05-08 11:26:00,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +14: [2023-05-08 11:26:00,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +13: [2023-05-08 11:26:00,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:26:00,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +13: [2023-05-08 11:26:00,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:26:00,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:26:00,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:26:00,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +24: [2023-05-08 11:26:00,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +24: [2023-05-08 11:26:00,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:26:00,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:26:00,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +24: [2023-05-08 11:26:00,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 7: [2023-05-08 11:26:00,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 7: [2023-05-08 11:26:00,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 7: [2023-05-08 11:26:00,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +23: [2023-05-08 11:26:00,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +23: [2023-05-08 11:26:00,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +23: [2023-05-08 11:26:00,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +29: [2023-05-08 11:26:00,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +29: [2023-05-08 11:26:00,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +29: [2023-05-08 11:26:00,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 3: [2023-05-08 11:26:00,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +23: [2023-05-08 11:26:00,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 3: [2023-05-08 11:26:00,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 3: [2023-05-08 11:26:00,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +13: [2023-05-08 11:26:00,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +21: [2023-05-08 11:26:00,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +15: [2023-05-08 11:26:00,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +15: [2023-05-08 11:26:00,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +15: [2023-05-08 11:26:00,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 0: [2023-05-08 11:26:00,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:26:00,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 0: [2023-05-08 11:26:00,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +21: [2023-05-08 11:26:00,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +21: [2023-05-08 11:26:00,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +15: [2023-05-08 11:26:00,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +21: [2023-05-08 11:26:00,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +14: [2023-05-08 11:26:00,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:26:00,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +11: [2023-05-08 11:26:00,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +11: [2023-05-08 11:26:00,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +11: [2023-05-08 11:26:00,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +13: [2023-05-08 11:26:00,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +11: [2023-05-08 11:26:00,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +19: [2023-05-08 11:26:00,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +19: [2023-05-08 11:26:00,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:26:00,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +19: [2023-05-08 11:26:00,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +19: [2023-05-08 11:26:00,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +19: [2023-05-08 11:26:00,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:26:00,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:26:00,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +14: [2023-05-08 11:26:00,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +14: [2023-05-08 11:26:00,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +14: [2023-05-08 11:26:00,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 4: [2023-05-08 11:26:00,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 4: [2023-05-08 11:26:00,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +18: [2023-05-08 11:26:00,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +18: [2023-05-08 11:26:00,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +18: [2023-05-08 11:26:00,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +18: [2023-05-08 11:26:00,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +18: [2023-05-08 11:26:00,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:26:00,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:26:00,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +18: [2023-05-08 11:26:00,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:26:00,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +20: [2023-05-08 11:26:00,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +18: [2023-05-08 11:26:00,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:26:00,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +20: [2023-05-08 11:26:00,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:26:00,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:26:00,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:26:00,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +20: [2023-05-08 11:26:00,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +20: [2023-05-08 11:26:00,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:26:00,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:26:00,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +21: [2023-05-08 11:26:00,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +21: [2023-05-08 11:26:00,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:26:00,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:26:00,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +11: [2023-05-08 11:26:00,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:26:00,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:26:00,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:26:00,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +21: [2023-05-08 11:26:00,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +21: [2023-05-08 11:26:00,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:26:00,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +23: [2023-05-08 11:26:00,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:26:00,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +11: [2023-05-08 11:26:00,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +11: [2023-05-08 11:26:00,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:26:00,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +11: [2023-05-08 11:26:00,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:26:00,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:26:00,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:26:00,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:26:00,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +14: [2023-05-08 11:26:00,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +19: [2023-05-08 11:26:00,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:26:00,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:26:00,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:26:00,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:26:00,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +18: [2023-05-08 11:26:00,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:26:00,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +23: [2023-05-08 11:26:00,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +19: [2023-05-08 11:26:00,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +18: [2023-05-08 11:26:00,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:26:00,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +24: [2023-05-08 11:26:00,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:26:00,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:26:00,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +15: [2023-05-08 11:26:00,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +18: [2023-05-08 11:26:00,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:26:00,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:26:00,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:26:00,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +11: [2023-05-08 11:26:00,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +11: [2023-05-08 11:26:00,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 7: [2023-05-08 11:26:00,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +19: [2023-05-08 11:26:00,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +14: [2023-05-08 11:26:00,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +14: [2023-05-08 11:26:00,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +22: [2023-05-08 11:26:00,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +22: [2023-05-08 11:26:00,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +22: [2023-05-08 11:26:00,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +22: [2023-05-08 11:26:00,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:26:00,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:26:00,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:26:00,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +22: [2023-05-08 11:26:00,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +15: [2023-05-08 11:26:00,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +15: [2023-05-08 11:26:00,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 6: [2023-05-08 11:26:00,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +12: [2023-05-08 11:26:00,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:26:00,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:26:00,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:26:00,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +12: [2023-05-08 11:26:00,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +12: [2023-05-08 11:26:00,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +12: [2023-05-08 11:26:00,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +12: [2023-05-08 11:26:00,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +12: [2023-05-08 11:26:00,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +11: [2023-05-08 11:26:00,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:26:00,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:26:00,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +15: [2023-05-08 11:26:00,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +18: [2023-05-08 11:26:00,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +21: [2023-05-08 11:26:00,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +23: [2023-05-08 11:26:00,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +21: [2023-05-08 11:26:00,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +11: [2023-05-08 11:26:00,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +29: [2023-05-08 11:26:00,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +11: [2023-05-08 11:26:00,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +24: [2023-05-08 11:26:00,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:26:00,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:26:00,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 3: [2023-05-08 11:26:00,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +19: [2023-05-08 11:26:00,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +11: [2023-05-08 11:26:00,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +18: [2023-05-08 11:26:00,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:26:00,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +20: [2023-05-08 11:26:00,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +20: [2023-05-08 11:26:00,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +20: [2023-05-08 11:26:00,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:26:00,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:26:00,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +21: [2023-05-08 11:26:00,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:26:00,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:26:00,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:26:00,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +21: [2023-05-08 11:26:00,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 7: [2023-05-08 11:26:00,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +23: [2023-05-08 11:26:00,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +23: [2023-05-08 11:26:00,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 4: [2023-05-08 11:26:00,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 4: [2023-05-08 11:26:00,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 4: [2023-05-08 11:26:00,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +18: [2023-05-08 11:26:00,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +18: [2023-05-08 11:26:00,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +21: [2023-05-08 11:26:00,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:26:00,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:26:00,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:26:00,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:26:00,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +15: [2023-05-08 11:26:00,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +21: [2023-05-08 11:26:00,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 6: [2023-05-08 11:26:00,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +21: [2023-05-08 11:26:00,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +23: [2023-05-08 11:26:00,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:26:00,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:26:00,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +29: [2023-05-08 11:26:00,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +29: [2023-05-08 11:26:00,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +29: [2023-05-08 11:26:00,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +15: [2023-05-08 11:26:00,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +29: [2023-05-08 11:26:00,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:26:00,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:26:00,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +11: [2023-05-08 11:26:00,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:26:00,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +29: [2023-05-08 11:26:00,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:26:00,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +21: [2023-05-08 11:26:00,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +23: [2023-05-08 11:26:00,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 3: [2023-05-08 11:26:00,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +11: [2023-05-08 11:26:00,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 3: [2023-05-08 11:26:00,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +24: [2023-05-08 11:26:00,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:26:00,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:26:00,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +15: [2023-05-08 11:26:00,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:26:00,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:26:00,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +15: [2023-05-08 11:26:00,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +15: [2023-05-08 11:26:00,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:26:00,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +29: [2023-05-08 11:26:00,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:26:00,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:26:00,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:26:00,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:26:00,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +29: [2023-05-08 11:26:00,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +15: [2023-05-08 11:26:00,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:26:00,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +22: [2023-05-08 11:26:00,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +27: [2023-05-08 11:26:00,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:26:00,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:26:00,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:26:00,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:26:00,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +12: [2023-05-08 11:26:00,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +12: [2023-05-08 11:26:00,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +30: [2023-05-08 11:26:00,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +12: [2023-05-08 11:26:00,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:26:00,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +27: [2023-05-08 11:26:00,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:26:00,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:26:00,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:26:00,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +30: [2023-05-08 11:26:00,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +22: [2023-05-08 11:26:00,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +22: [2023-05-08 11:26:00,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +22: [2023-05-08 11:26:00,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:26:00,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +15: [2023-05-08 11:26:00,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +22: [2023-05-08 11:26:00,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:26:00,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +30: [2023-05-08 11:26:00,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:26:00,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:26:00,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:26:00,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:26:00,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +17: [2023-05-08 11:26:00,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +17: [2023-05-08 11:26:00,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +29: [2023-05-08 11:26:00,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +29: [2023-05-08 11:26:00,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +29: [2023-05-08 11:26:00,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:26:00,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +27: [2023-05-08 11:26:00,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:26:00,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +30: [2023-05-08 11:26:00,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +14: [2023-05-08 11:26:00,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:26:00,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:26:00,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:26:00,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +14: [2023-05-08 11:26:00,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +14: [2023-05-08 11:26:00,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +14: [2023-05-08 11:26:00,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:26:00,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +15: [2023-05-08 11:26:00,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:26:00,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +20: [2023-05-08 11:26:00,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +30: [2023-05-08 11:26:00,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +20: [2023-05-08 11:26:00,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 9: [2023-05-08 11:26:00,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +30: [2023-05-08 11:26:00,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +30: [2023-05-08 11:26:00,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 9: [2023-05-08 11:26:00,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +20: [2023-05-08 11:26:00,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +30: [2023-05-08 11:26:00,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +20: [2023-05-08 11:26:00,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +25: [2023-05-08 11:26:00,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +25: [2023-05-08 11:26:00,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +25: [2023-05-08 11:26:00,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +25: [2023-05-08 11:26:00,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 9: [2023-05-08 11:26:00,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 9: [2023-05-08 11:26:00,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +29: [2023-05-08 11:26:00,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 9: [2023-05-08 11:26:00,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:26:00,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +22: [2023-05-08 11:26:00,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +22: [2023-05-08 11:26:00,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:26:00,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:26:00,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +27: [2023-05-08 11:26:00,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:26:00,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +27: [2023-05-08 11:26:00,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +17: [2023-05-08 11:26:00,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:26:00,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +29: [2023-05-08 11:26:00,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +17: [2023-05-08 11:26:00,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +25: [2023-05-08 11:26:00,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +20: [2023-05-08 11:26:00,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +25: [2023-05-08 11:26:00,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +25: [2023-05-08 11:26:00,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +30: [2023-05-08 11:26:00,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +20: [2023-05-08 11:26:00,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:26:00,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +30: [2023-05-08 11:26:00,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:26:00,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:26:00,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +15: [2023-05-08 11:26:00,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +25: [2023-05-08 11:26:00,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +14: [2023-05-08 11:26:00,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:26:00,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:26:00,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +30: [2023-05-08 11:26:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +30: [2023-05-08 11:26:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +20: [2023-05-08 11:26:00,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +26: [2023-05-08 11:26:00,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:26:00,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +14: [2023-05-08 11:26:00,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:26:00,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:26:00,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:26:00,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:26:00,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:26:00,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:26:00,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:26:00,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:26:00,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:26:00,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:26:00,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +11: [2023-05-08 11:26:00,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:26:00,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +11: [2023-05-08 11:26:00,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:26:00,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +11: [2023-05-08 11:26:00,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +11: [2023-05-08 11:26:00,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:26:00,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +16: [2023-05-08 11:26:00,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +11: [2023-05-08 11:26:00,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +15: [2023-05-08 11:26:00,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:26:00,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 6: [2023-05-08 11:26:00,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 6: [2023-05-08 11:26:00,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +16: [2023-05-08 11:26:00,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 6: [2023-05-08 11:26:00,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +16: [2023-05-08 11:26:00,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +14: [2023-05-08 11:26:00,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:26:00,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:26:00,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:26:00,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:26:00,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:26:00,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:26:00,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:26:00,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:26:00,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:26:00,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:26:00,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:26:00,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +17: [2023-05-08 11:26:00,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +25: [2023-05-08 11:26:00,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:26:00,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:26:00,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:26:00,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +17: [2023-05-08 11:26:00,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +25: [2023-05-08 11:26:00,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +14: [2023-05-08 11:26:00,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +25: [2023-05-08 11:26:00,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +14: [2023-05-08 11:26:00,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +14: [2023-05-08 11:26:00,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:26:00,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +20: [2023-05-08 11:26:00,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:26:00,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +11: [2023-05-08 11:26:00,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:26:00,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:26:00,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:26:00,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:26:00,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +17: [2023-05-08 11:26:00,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +30: [2023-05-08 11:26:00,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +11: [2023-05-08 11:26:00,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:26:00,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:26:00,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +11: [2023-05-08 11:26:00,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:26:00,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +26: [2023-05-08 11:26:00,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:26:00,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:26:00,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +26: [2023-05-08 11:26:00,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:26:00,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +16: [2023-05-08 11:26:00,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 6: [2023-05-08 11:26:00,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 9: [2023-05-08 11:26:00,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +16: [2023-05-08 11:26:00,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +20: [2023-05-08 11:26:00,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:26:00,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +20: [2023-05-08 11:26:00,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +20: [2023-05-08 11:26:00,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +30: [2023-05-08 11:26:00,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:26:00,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +16: [2023-05-08 11:26:00,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +25: [2023-05-08 11:26:00,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +30: [2023-05-08 11:26:00,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:26:00,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +30: [2023-05-08 11:26:00,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +11: [2023-05-08 11:26:00,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +11: [2023-05-08 11:26:00,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +11: [2023-05-08 11:26:00,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:26:00,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +26: [2023-05-08 11:26:00,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:26:00,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +26: [2023-05-08 11:26:00,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:26:00,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:26:00,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 4: [2023-05-08 11:26:00,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:26:00,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:26:00,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +10: [2023-05-08 11:26:00,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:26:00,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +10: [2023-05-08 11:26:00,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:26:00,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +16: [2023-05-08 11:26:00,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:26:00,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:26:00,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:26:00,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 6: [2023-05-08 11:26:00,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +11: [2023-05-08 11:26:00,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:26:00,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:26:00,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:26:00,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:26:00,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +16: [2023-05-08 11:26:00,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +16: [2023-05-08 11:26:00,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 4: [2023-05-08 11:26:00,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:26:00,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:26:00,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +26: [2023-05-08 11:26:00,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +26: [2023-05-08 11:26:00,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +26: [2023-05-08 11:26:00,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 8: [2023-05-08 11:26:00,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 8: [2023-05-08 11:26:00,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 8: [2023-05-08 11:26:00,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +12: [2023-05-08 11:26:00,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 8: [2023-05-08 11:26:00,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +12: [2023-05-08 11:26:00,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +12: [2023-05-08 11:26:00,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +12: [2023-05-08 11:26:00,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +11: [2023-05-08 11:26:00,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:26:00,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:26:00,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +16: [2023-05-08 11:26:00,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:26:00,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +16: [2023-05-08 11:26:00,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:26:00,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +27: [2023-05-08 11:26:00,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +27: [2023-05-08 11:26:00,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +16: [2023-05-08 11:26:00,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:26:00,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:26:00,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:26:00,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +12: [2023-05-08 11:26:00,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +16: [2023-05-08 11:26:00,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +26: [2023-05-08 11:26:00,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +26: [2023-05-08 11:26:00,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +16: [2023-05-08 11:26:00,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:26:00,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 4: [2023-05-08 11:26:00,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:26:00,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +26: [2023-05-08 11:26:00,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +28: [2023-05-08 11:26:00,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:26:00,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:26:00,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:26:00,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:26:00,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:26:00,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:26:00,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +10: [2023-05-08 11:26:00,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +10: [2023-05-08 11:26:00,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +28: [2023-05-08 11:26:00,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:26:00,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +16: [2023-05-08 11:26:00,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +27: [2023-05-08 11:26:00,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:26:00,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:26:00,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +27: [2023-05-08 11:26:00,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:26:00,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +31: [2023-05-08 11:26:00,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +31: [2023-05-08 11:26:00,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +10: [2023-05-08 11:26:00,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:26:00,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +10: [2023-05-08 11:26:00,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +26: [2023-05-08 11:26:00,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:26:00,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +28: [2023-05-08 11:26:00,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:26:00,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:26:00,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +12: [2023-05-08 11:26:00,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:26:00,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:26:00,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +12: [2023-05-08 11:26:00,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 5: [2023-05-08 11:26:00,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:26:00,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:26:00,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +31: [2023-05-08 11:26:00,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +26: [2023-05-08 11:26:00,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +31: [2023-05-08 11:26:00,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:26:00,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:26:00,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +26: [2023-05-08 11:26:00,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +27: [2023-05-08 11:26:00,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +10: [2023-05-08 11:26:00,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +10: [2023-05-08 11:26:00,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +27: [2023-05-08 11:26:00,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +10: [2023-05-08 11:26:00,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +12: [2023-05-08 11:26:00,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +12: [2023-05-08 11:26:00,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +12: [2023-05-08 11:26:00,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:26:00,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +26: [2023-05-08 11:26:00,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +12: [2023-05-08 11:26:00,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +31: [2023-05-08 11:26:00,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +10: [2023-05-08 11:26:00,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +27: [2023-05-08 11:26:00,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +28: [2023-05-08 11:26:00,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:26:00,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 8: [2023-05-08 11:26:00,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:26:00,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 8: [2023-05-08 11:26:00,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:26:00,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:26:00,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +28: [2023-05-08 11:26:00,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +27: [2023-05-08 11:26:00,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +10: [2023-05-08 11:26:00,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +10: [2023-05-08 11:26:00,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:26:00,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +10: [2023-05-08 11:26:00,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +31: [2023-05-08 11:26:00,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +10: [2023-05-08 11:26:00,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:26:00,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:26:00,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +27: [2023-05-08 11:26:00,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:26:00,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +31: [2023-05-08 11:26:00,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +28: [2023-05-08 11:26:00,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +27: [2023-05-08 11:26:00,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 8: [2023-05-08 11:26:00,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +21: [2023-05-08 11:26:00,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:26:00,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:26:00,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +21: [2023-05-08 11:26:00,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +21: [2023-05-08 11:26:00,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:26:00,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:26:00,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:26:00,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:26:00,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:26:00,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:26:00,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +17: [2023-05-08 11:26:00,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:26:00,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +21: [2023-05-08 11:26:00,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:26:00,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:26:00,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +31: [2023-05-08 11:26:00,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +10: [2023-05-08 11:26:00,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +10: [2023-05-08 11:26:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +10: [2023-05-08 11:26:00,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 5: [2023-05-08 11:26:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:26:00,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 2: [2023-05-08 11:26:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 2: [2023-05-08 11:26:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 2: [2023-05-08 11:26:00,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +31: [2023-05-08 11:26:00,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 2: [2023-05-08 11:26:00,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 5: [2023-05-08 11:26:00,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:26:00,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 1: [2023-05-08 11:26:00,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 1: [2023-05-08 11:26:00,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +10: [2023-05-08 11:26:00,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 1: [2023-05-08 11:26:00,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 0: [2023-05-08 11:26:00,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 0: [2023-05-08 11:26:00,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 0: [2023-05-08 11:26:00,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +17: [2023-05-08 11:26:00,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +18: [2023-05-08 11:26:00,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +18: [2023-05-08 11:26:00,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +18: [2023-05-08 11:26:00,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +18: [2023-05-08 11:26:00,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +24: [2023-05-08 11:26:00,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +19: [2023-05-08 11:26:00,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +21: [2023-05-08 11:26:00,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:26:00,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +19: [2023-05-08 11:26:00,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +19: [2023-05-08 11:26:00,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +19: [2023-05-08 11:26:00,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +21: [2023-05-08 11:26:00,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:26:00,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +24: [2023-05-08 11:26:00,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +17: [2023-05-08 11:26:00,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +24: [2023-05-08 11:26:00,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 2: [2023-05-08 11:26:00,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +10: [2023-05-08 11:26:00,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:26:00,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +13: [2023-05-08 11:26:00,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +13: [2023-05-08 11:26:00,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +13: [2023-05-08 11:26:00,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 2: [2023-05-08 11:26:00,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:26:00,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +28: [2023-05-08 11:26:00,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +28: [2023-05-08 11:26:00,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +28: [2023-05-08 11:26:00,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 1: [2023-05-08 11:26:00,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +10: [2023-05-08 11:26:00,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 5: [2023-05-08 11:26:00,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +22: [2023-05-08 11:26:00,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 2: [2023-05-08 11:26:00,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:26:00,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. +22: [2023-05-08 11:26:00,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 2: [2023-05-08 11:26:00,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:26:00,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 2: [2023-05-08 11:26:00,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 2: [2023-05-08 11:26:00,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 5: [2023-05-08 11:26:00,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 5: [2023-05-08 11:26:00,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt. + 2: [2023-05-08 11:26:00,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:26:00,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:26:00,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +21: [2023-05-08 11:26:00,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:26:00,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:26:00,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:26:00,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:26:00,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:26:00,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:26:00,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:26:00,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +21: [2023-05-08 11:26:00,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:26:00,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:26:00,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:26:00,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:26:00,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:26:00,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +18: [2023-05-08 11:26:00,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +19: [2023-05-08 11:26:00,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:26:00,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +21: [2023-05-08 11:26:00,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:26:00,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:26:00,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:26:00,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:26:00,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:26:00,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:26:00,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +28: [2023-05-08 11:26:00,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +13: [2023-05-08 11:26:00,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:26:00,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +24: [2023-05-08 11:26:00,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:26:00,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +22: [2023-05-08 11:26:00,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt... +17: [2023-05-08 11:26:00,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +13: [2023-05-08 11:26:00,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:26:00,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:26:00,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:26:00,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +13: [2023-05-08 11:26:00,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +13: [2023-05-08 11:26:00,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +18: [2023-05-08 11:26:00,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:26:00,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +17: [2023-05-08 11:26:00,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +13: [2023-05-08 11:26:00,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +18: [2023-05-08 11:26:00,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 1: [2023-05-08 11:26:00,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:26:00,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 2: [2023-05-08 11:26:00,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:26:00,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:26:00,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +18: [2023-05-08 11:26:00,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:26:00,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 1: [2023-05-08 11:26:00,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +19: [2023-05-08 11:26:00,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 0: [2023-05-08 11:26:00,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +13: [2023-05-08 11:26:00,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 2: [2023-05-08 11:26:00,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:26:00,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +22: [2023-05-08 11:26:00,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 1: [2023-05-08 11:26:00,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:26:00,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:26:00,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 1: [2023-05-08 11:26:00,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 0: [2023-05-08 11:26:00,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 5: [2023-05-08 11:26:00,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:26:00,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +24: [2023-05-08 11:26:00,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 5: [2023-05-08 11:26:00,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 5: [2023-05-08 11:26:00,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +13: [2023-05-08 11:26:00,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:26:00,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +18: [2023-05-08 11:26:00,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +18: [2023-05-08 11:26:00,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +18: [2023-05-08 11:26:00,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 5: [2023-05-08 11:26:00,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +13: [2023-05-08 11:26:00,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:26:00,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 2: [2023-05-08 11:26:00,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +28: [2023-05-08 11:26:00,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +24: [2023-05-08 11:26:00,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +24: [2023-05-08 11:26:00,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +24: [2023-05-08 11:26:00,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +28: [2023-05-08 11:26:00,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +28: [2023-05-08 11:26:00,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +28: [2023-05-08 11:26:00,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 2: [2023-05-08 11:26:00,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +22: [2023-05-08 11:26:00,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +22: [2023-05-08 11:26:00,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +13: [2023-05-08 11:26:00,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:26:00,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +13: [2023-05-08 11:26:00,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:26:00,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +25: [2023-05-08 11:26:00,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:26:00,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:26:00,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:26:00,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +25: [2023-05-08 11:26:00,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +25: [2023-05-08 11:26:00,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +24: [2023-05-08 11:26:00,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +28: [2023-05-08 11:26:00,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +25: [2023-05-08 11:26:00,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +28: [2023-05-08 11:26:00,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +25: [2023-05-08 11:26:00,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +22: [2023-05-08 11:26:00,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +22: [2023-05-08 11:26:00,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 2: [2023-05-08 11:26:00,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +22: [2023-05-08 11:26:00,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +22: [2023-05-08 11:26:00,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +19: [2023-05-08 11:26:00,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:26:00,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +19: [2023-05-08 11:26:00,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +19: [2023-05-08 11:26:00,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 5: [2023-05-08 11:26:00,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:26:00,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:26:00,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +28: [2023-05-08 11:26:00,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +13: [2023-05-08 11:26:00,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. +28: [2023-05-08 11:26:00,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... + 2: [2023-05-08 11:26:00,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:26:00,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +24: [2023-05-08 11:26:00,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt... +13: [2023-05-08 11:26:00,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:26:00,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +13: [2023-05-08 11:26:00,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:26:00,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:26:00,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:26:00,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:26:00,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:26:00,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:26:00,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:26:00,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:26:00,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:26:00,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +23: [2023-05-08 11:26:00,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:26:00,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:26:00,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +12: [2023-05-08 11:26:00,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +12: [2023-05-08 11:26:00,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:26:00,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:26:00,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +23: [2023-05-08 11:26:00,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +23: [2023-05-08 11:26:00,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:26:00,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:26:00,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:26:00,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:26:00,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:26:00,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:26:00,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:26:00,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:26:00,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:26:00,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:26:00,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:26:00,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:26:00,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +22: [2023-05-08 11:26:00,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:26:00,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:26:00,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:26:00,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:26:00,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +22: [2023-05-08 11:26:00,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +22: [2023-05-08 11:26:00,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +22: [2023-05-08 11:26:00,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +23: [2023-05-08 11:26:00,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:26:00,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:26:00,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 6: [2023-05-08 11:26:00,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:26:00,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:26:00,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:26:00,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:26:00,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:26:00,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:26:00,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:26:00,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:26:00,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +18: [2023-05-08 11:26:00,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +18: [2023-05-08 11:26:00,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:26:00,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +29: [2023-05-08 11:26:00,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +18: [2023-05-08 11:26:00,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:26:00,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +24: [2023-05-08 11:26:00,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:26:00,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +24: [2023-05-08 11:26:00,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +24: [2023-05-08 11:26:00,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:26:00,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:26:00,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +24: [2023-05-08 11:26:00,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:26:00,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:26:00,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:26:00,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:26:00,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:26:00,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:26:00,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +19: [2023-05-08 11:26:00,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:26:00,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +19: [2023-05-08 11:26:00,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +19: [2023-05-08 11:26:00,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:26:00,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:26:00,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +30: [2023-05-08 11:26:00,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:26:00,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:26:00,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +19: [2023-05-08 11:26:00,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:26:00,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +20: [2023-05-08 11:26:00,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +20: [2023-05-08 11:26:00,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:26:00,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:26:00,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +20: [2023-05-08 11:26:00,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:26:00,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:26:00,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:26:00,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +19: [2023-05-08 11:26:00,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:26:00,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:26:00,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:26:00,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:26:00,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:26:00,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:26:00,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +15: [2023-05-08 11:26:00,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +15: [2023-05-08 11:26:00,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +15: [2023-05-08 11:26:00,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:26:00,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +23: [2023-05-08 11:26:00,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +23: [2023-05-08 11:26:00,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +15: [2023-05-08 11:26:00,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:26:00,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +15: [2023-05-08 11:26:00,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:26:00,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +15: [2023-05-08 11:26:00,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +22: [2023-05-08 11:26:00,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:26:00,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:26:00,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:26:00,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:26:00,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +24: [2023-05-08 11:26:00,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +29: [2023-05-08 11:26:00,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +29: [2023-05-08 11:26:00,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +29: [2023-05-08 11:26:00,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +18: [2023-05-08 11:26:00,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +29: [2023-05-08 11:26:00,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +19: [2023-05-08 11:26:00,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:26:00,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +30: [2023-05-08 11:26:00,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:26:00,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:26:00,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:26:00,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +24: [2023-05-08 11:26:00,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +12: [2023-05-08 11:26:00,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:26:00,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +12: [2023-05-08 11:26:00,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:26:00,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:26:00,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +12: [2023-05-08 11:26:00,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +12: [2023-05-08 11:26:00,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:26:00,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:26:00,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +12: [2023-05-08 11:26:00,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:26:00,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +19: [2023-05-08 11:26:00,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +23: [2023-05-08 11:26:00,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +22: [2023-05-08 11:26:00,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +15: [2023-05-08 11:26:00,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +29: [2023-05-08 11:26:00,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +22: [2023-05-08 11:26:00,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:26:00,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:26:00,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +22: [2023-05-08 11:26:00,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:26:00,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:26:00,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +24: [2023-05-08 11:26:00,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:26:00,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:26:00,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +23: [2023-05-08 11:26:00,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +23: [2023-05-08 11:26:00,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +22: [2023-05-08 11:26:00,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:26:00,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:26:00,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:26:00,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +20: [2023-05-08 11:26:00,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +29: [2023-05-08 11:26:00,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:26:00,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:26:00,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:26:00,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 3: [2023-05-08 11:26:00,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 3: [2023-05-08 11:26:00,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +19: [2023-05-08 11:26:00,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +18: [2023-05-08 11:26:00,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:26:00,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:26:00,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:26:00,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +15: [2023-05-08 11:26:00,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:26:00,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:26:00,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +27: [2023-05-08 11:26:00,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +27: [2023-05-08 11:26:00,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +27: [2023-05-08 11:26:00,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +27: [2023-05-08 11:26:00,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +27: [2023-05-08 11:26:00,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +27: [2023-05-08 11:26:00,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +27: [2023-05-08 11:26:00,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:26:00,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +19: [2023-05-08 11:26:00,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:26:00,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:26:00,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:26:00,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:26:00,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +29: [2023-05-08 11:26:00,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:26:00,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +19: [2023-05-08 11:26:00,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:26:00,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:26:00,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +21: [2023-05-08 11:26:00,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +23: [2023-05-08 11:26:00,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:26:00,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +21: [2023-05-08 11:26:00,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +23: [2023-05-08 11:26:00,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:26:00,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:26:00,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +20: [2023-05-08 11:26:00,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +24: [2023-05-08 11:26:00,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +20: [2023-05-08 11:26:00,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:26:00,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:26:00,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:26:00,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:26:00,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:26:00,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:26:00,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:26:00,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:26:00,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +18: [2023-05-08 11:26:00,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +15: [2023-05-08 11:26:00,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:26:00,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:26:00,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:26:00,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +28: [2023-05-08 11:26:00,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +28: [2023-05-08 11:26:00,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +28: [2023-05-08 11:26:00,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +28: [2023-05-08 11:26:00,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:26:00,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:26:00,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:26:00,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:26:00,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +28: [2023-05-08 11:26:00,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +28: [2023-05-08 11:26:00,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:26:00,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:26:00,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:26:00,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:26:00,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +27: [2023-05-08 11:26:00,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:26:00,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 0: [2023-05-08 11:26:00,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:26:00,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:26:00,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +23: [2023-05-08 11:26:00,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:26:00,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:26:00,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:26:00,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:26:00,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +26: [2023-05-08 11:26:00,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:26:00,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:26:00,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:26:00,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +10: [2023-05-08 11:26:00,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +10: [2023-05-08 11:26:00,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +10: [2023-05-08 11:26:00,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +10: [2023-05-08 11:26:00,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +10: [2023-05-08 11:26:00,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +10: [2023-05-08 11:26:00,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +20: [2023-05-08 11:26:00,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:26:00,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +24: [2023-05-08 11:26:00,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:26:00,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:26:00,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +26: [2023-05-08 11:26:00,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:26:00,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:26:00,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:26:00,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +29: [2023-05-08 11:26:00,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:26:00,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +15: [2023-05-08 11:26:00,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:26:00,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:26:00,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:26:00,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +23: [2023-05-08 11:26:00,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:26:00,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:26:00,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:26:00,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +23: [2023-05-08 11:26:00,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 0: [2023-05-08 11:26:00,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:26:00,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +27: [2023-05-08 11:26:00,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +27: [2023-05-08 11:26:00,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:26:00,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +28: [2023-05-08 11:26:00,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +28: [2023-05-08 11:26:00,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:26:00,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +28: [2023-05-08 11:26:00,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +23: [2023-05-08 11:26:00,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:26:00,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +27: [2023-05-08 11:26:00,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +21: [2023-05-08 11:26:00,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:26:00,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:26:00,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +12: [2023-05-08 11:26:00,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:26:00,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +10: [2023-05-08 11:26:00,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +10: [2023-05-08 11:26:00,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 3: [2023-05-08 11:26:00,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 3: [2023-05-08 11:26:00,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +10: [2023-05-08 11:26:00,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +10: [2023-05-08 11:26:00,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:26:00,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +21: [2023-05-08 11:26:00,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +26: [2023-05-08 11:26:00,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:26:00,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +21: [2023-05-08 11:26:00,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +28: [2023-05-08 11:26:00,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +28: [2023-05-08 11:26:00,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +28: [2023-05-08 11:26:00,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:26:00,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +11: [2023-05-08 11:26:00,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:26:00,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:26:00,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +10: [2023-05-08 11:26:00,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:26:00,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +10: [2023-05-08 11:26:00,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:26:00,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:26:00,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:26:00,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +11: [2023-05-08 11:26:00,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:26:00,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:26:00,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +11: [2023-05-08 11:26:00,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +28: [2023-05-08 11:26:00,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:26:00,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:26:00,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +11: [2023-05-08 11:26:00,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:26:00,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +26: [2023-05-08 11:26:00,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +28: [2023-05-08 11:26:00,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:26:00,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +16: [2023-05-08 11:26:00,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +16: [2023-05-08 11:26:00,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +16: [2023-05-08 11:26:00,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +11: [2023-05-08 11:26:00,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:26:00,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:26:00,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +16: [2023-05-08 11:26:00,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +16: [2023-05-08 11:26:00,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +16: [2023-05-08 11:26:00,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +14: [2023-05-08 11:26:00,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +14: [2023-05-08 11:26:00,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +14: [2023-05-08 11:26:00,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +14: [2023-05-08 11:26:00,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +11: [2023-05-08 11:26:00,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +11: [2023-05-08 11:26:00,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +14: [2023-05-08 11:26:00,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +14: [2023-05-08 11:26:00,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +14: [2023-05-08 11:26:00,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +14: [2023-05-08 11:26:00,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +11: [2023-05-08 11:26:00,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:26:00,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:26:00,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +16: [2023-05-08 11:26:00,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +14: [2023-05-08 11:26:00,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +14: [2023-05-08 11:26:00,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +14: [2023-05-08 11:26:00,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +14: [2023-05-08 11:26:00,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +11: [2023-05-08 11:26:00,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +14: [2023-05-08 11:26:00,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +11: [2023-05-08 11:26:00,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +11: [2023-05-08 11:26:00,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +11: [2023-05-08 11:26:00,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +11: [2023-05-08 11:26:00,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 4: [2023-05-08 11:26:00,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 4: [2023-05-08 11:26:00,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 4: [2023-05-08 11:26:00,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +14: [2023-05-08 11:26:00,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +14: [2023-05-08 11:26:00,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:26:00,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +17: [2023-05-08 11:26:00,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +17: [2023-05-08 11:26:00,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +17: [2023-05-08 11:26:00,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 3: [2023-05-08 11:26:00,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:26:00,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:26:00,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:26:00,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:26:00,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:26:00,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:26:00,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:26:00,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:26:00,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:26:00,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:26:00,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:26:00,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:26:00,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +16: [2023-05-08 11:26:00,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +13: [2023-05-08 11:26:00,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:26:00,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +13: [2023-05-08 11:26:00,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +13: [2023-05-08 11:26:00,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:26:00,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +13: [2023-05-08 11:26:00,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:26:00,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:26:00,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +11: [2023-05-08 11:26:00,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:26:00,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +14: [2023-05-08 11:26:00,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:26:00,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 4: [2023-05-08 11:26:00,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 4: [2023-05-08 11:26:00,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +14: [2023-05-08 11:26:00,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:26:00,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +17: [2023-05-08 11:26:00,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +13: [2023-05-08 11:26:00,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:26:00,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:26:00,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:26:00,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:26:00,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +17: [2023-05-08 11:26:00,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +17: [2023-05-08 11:26:00,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +17: [2023-05-08 11:26:00,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:26:00,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:26:00,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:26:00,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +17: [2023-05-08 11:26:00,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +17: [2023-05-08 11:26:00,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +17: [2023-05-08 11:26:00,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 3: [2023-05-08 11:26:00,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +30: [2023-05-08 11:26:00,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +30: [2023-05-08 11:26:00,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +30: [2023-05-08 11:26:00,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +30: [2023-05-08 11:26:00,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 8: [2023-05-08 11:26:00,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:26:00,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:26:00,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:26:00,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +14: [2023-05-08 11:26:00,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 8: [2023-05-08 11:26:00,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:26:00,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:26:00,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:26:00,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:26:00,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:26:00,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:26:00,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +16: [2023-05-08 11:26:00,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:26:00,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:26:00,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:26:00,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:26:00,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 4: [2023-05-08 11:26:00,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:26:00,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +14: [2023-05-08 11:26:00,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:26:00,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 5: [2023-05-08 11:26:00,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +25: [2023-05-08 11:26:00,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +25: [2023-05-08 11:26:00,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +16: [2023-05-08 11:26:00,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:26:00,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 3: [2023-05-08 11:26:00,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +11: [2023-05-08 11:26:00,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +14: [2023-05-08 11:26:00,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:26:00,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +11: [2023-05-08 11:26:00,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +11: [2023-05-08 11:26:00,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +30: [2023-05-08 11:26:00,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:26:00,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:26:00,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:26:00,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:26:00,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:26:00,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:26:00,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:26:00,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:26:00,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:26:00,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:26:00,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 3: [2023-05-08 11:26:00,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +13: [2023-05-08 11:26:00,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:26:00,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:26:00,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:26:00,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:26:00,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:26:00,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +25: [2023-05-08 11:26:00,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +25: [2023-05-08 11:26:00,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:26:00,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 3: [2023-05-08 11:26:00,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:26:00,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 4: [2023-05-08 11:26:00,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 4: [2023-05-08 11:26:00,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +17: [2023-05-08 11:26:00,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 2: [2023-05-08 11:26:00,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:26:00,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +14: [2023-05-08 11:26:00,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:26:00,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:26:00,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 2: [2023-05-08 11:26:00,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:26:00,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +17: [2023-05-08 11:26:00,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:26:00,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +17: [2023-05-08 11:26:00,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:26:00,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +20: [2023-05-08 11:26:00,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +17: [2023-05-08 11:26:00,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:26:00,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +20: [2023-05-08 11:26:00,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +17: [2023-05-08 11:26:00,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:26:00,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 2: [2023-05-08 11:26:00,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:26:00,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:26:00,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +30: [2023-05-08 11:26:00,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:26:00,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +30: [2023-05-08 11:26:00,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 2: [2023-05-08 11:26:00,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +14: [2023-05-08 11:26:00,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +17: [2023-05-08 11:26:00,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:26:00,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +14: [2023-05-08 11:26:00,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:26:00,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +15: [2023-05-08 11:26:00,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +15: [2023-05-08 11:26:00,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +15: [2023-05-08 11:26:00,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +20: [2023-05-08 11:26:00,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +17: [2023-05-08 11:26:00,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:26:00,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:26:00,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:26:00,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 7: [2023-05-08 11:26:00,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +20: [2023-05-08 11:26:00,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:26:00,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:26:00,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 2: [2023-05-08 11:26:00,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 7: [2023-05-08 11:26:00,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +30: [2023-05-08 11:26:00,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +30: [2023-05-08 11:26:00,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:26:00,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:26:00,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +30: [2023-05-08 11:26:00,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +25: [2023-05-08 11:26:00,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:26:00,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:26:00,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:26:00,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +25: [2023-05-08 11:26:00,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +25: [2023-05-08 11:26:00,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +26: [2023-05-08 11:26:00,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +26: [2023-05-08 11:26:00,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +26: [2023-05-08 11:26:00,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +26: [2023-05-08 11:26:00,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +15: [2023-05-08 11:26:00,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +15: [2023-05-08 11:26:00,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +15: [2023-05-08 11:26:00,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +25: [2023-05-08 11:26:00,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:26:00,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +20: [2023-05-08 11:26:00,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:26:00,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:26:00,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +15: [2023-05-08 11:26:00,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:26:00,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:26:00,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +20: [2023-05-08 11:26:00,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 6: [2023-05-08 11:26:00,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 6: [2023-05-08 11:26:00,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 6: [2023-05-08 11:26:00,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 6: [2023-05-08 11:26:00,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +26: [2023-05-08 11:26:00,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 9: [2023-05-08 11:26:00,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 9: [2023-05-08 11:26:00,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 9: [2023-05-08 11:26:00,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +27: [2023-05-08 11:26:00,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 9: [2023-05-08 11:26:00,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +27: [2023-05-08 11:26:00,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +27: [2023-05-08 11:26:00,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +27: [2023-05-08 11:26:00,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +16: [2023-05-08 11:26:00,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +16: [2023-05-08 11:26:00,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +16: [2023-05-08 11:26:00,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +16: [2023-05-08 11:26:00,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +26: [2023-05-08 11:26:00,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:26:00,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:26:00,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:26:00,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +20: [2023-05-08 11:26:00,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +15: [2023-05-08 11:26:01,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:26:01,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 6: [2023-05-08 11:26:01,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +15: [2023-05-08 11:26:01,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:26:01,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 7: [2023-05-08 11:26:01,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 9: [2023-05-08 11:26:01,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 7: [2023-05-08 11:26:01,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +27: [2023-05-08 11:26:01,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:26:01,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:26:01,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:26:01,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +27: [2023-05-08 11:26:01,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +15: [2023-05-08 11:26:01,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:26:01,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +15: [2023-05-08 11:26:01,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +16: [2023-05-08 11:26:01,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +20: [2023-05-08 11:26:01,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:26:01,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +15: [2023-05-08 11:26:01,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:26:01,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 7: [2023-05-08 11:26:01,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +20: [2023-05-08 11:26:01,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +23: [2023-05-08 11:26:01,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +31: [2023-05-08 11:26:01,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +26: [2023-05-08 11:26:01,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:26:01,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +31: [2023-05-08 11:26:01,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +23: [2023-05-08 11:26:01,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:26:01,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:26:01,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +31: [2023-05-08 11:26:01,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +26: [2023-05-08 11:26:01,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +23: [2023-05-08 11:26:01,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:26:01,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:26:01,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:26:01,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:26:01,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:26:01,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +26: [2023-05-08 11:26:01,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:26:01,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +26: [2023-05-08 11:26:01,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +26: [2023-05-08 11:26:01,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +25: [2023-05-08 11:26:01,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:26:01,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:26:01,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:26:01,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:26:01,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:26:01,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:26:01,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:26:01,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +27: [2023-05-08 11:26:01,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +27: [2023-05-08 11:26:01,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 6: [2023-05-08 11:26:01,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 6: [2023-05-08 11:26:01,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 9: [2023-05-08 11:26:01,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +16: [2023-05-08 11:26:01,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:26:01,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +16: [2023-05-08 11:26:01,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +31: [2023-05-08 11:26:01,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:26:01,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +31: [2023-05-08 11:26:01,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:26:01,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 9: [2023-05-08 11:26:01,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +16: [2023-05-08 11:26:01,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:26:01,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +16: [2023-05-08 11:26:01,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:26:01,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +27: [2023-05-08 11:26:01,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +27: [2023-05-08 11:26:01,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +27: [2023-05-08 11:26:01,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +27: [2023-05-08 11:26:01,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +27: [2023-05-08 11:26:01,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +27: [2023-05-08 11:26:01,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +23: [2023-05-08 11:26:01,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:26:01,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:26:01,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +16: [2023-05-08 11:26:01,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +16: [2023-05-08 11:26:01,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +25: [2023-05-08 11:26:01,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:26:01,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +31: [2023-05-08 11:26:01,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:26:01,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:26:01,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +31: [2023-05-08 11:26:01,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +31: [2023-05-08 11:26:01,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:26:01,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 9: [2023-05-08 11:26:01,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +25: [2023-05-08 11:26:01,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:26:01,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:26:01,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +25: [2023-05-08 11:26:01,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +31: [2023-05-08 11:26:01,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +31: [2023-05-08 11:26:01,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +23: [2023-05-08 11:26:01,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:26:01,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:26:01,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +10: [2023-05-08 11:26:01,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +10: [2023-05-08 11:26:01,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +10: [2023-05-08 11:26:01,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +25: [2023-05-08 11:26:01,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +10: [2023-05-08 11:26:01,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +25: [2023-05-08 11:26:01,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:26:01,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +10: [2023-05-08 11:26:01,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +10: [2023-05-08 11:26:01,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +10: [2023-05-08 11:26:01,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +10: [2023-05-08 11:26:01,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:26:01,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +28: [2023-05-08 11:26:01,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +28: [2023-05-08 11:26:01,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +28: [2023-05-08 11:26:01,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +28: [2023-05-08 11:26:01,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:26:01,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:26:01,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +10: [2023-05-08 11:26:01,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:26:01,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:26:01,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +10: [2023-05-08 11:26:01,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +10: [2023-05-08 11:26:01,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:26:01,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +28: [2023-05-08 11:26:01,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +24: [2023-05-08 11:26:01,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +24: [2023-05-08 11:26:01,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +24: [2023-05-08 11:26:01,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +24: [2023-05-08 11:26:01,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +10: [2023-05-08 11:26:01,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:26:01,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +24: [2023-05-08 11:26:01,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +10: [2023-05-08 11:26:01,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +24: [2023-05-08 11:26:01,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +24: [2023-05-08 11:26:01,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +28: [2023-05-08 11:26:01,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:26:01,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 0: [2023-05-08 11:26:01,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +28: [2023-05-08 11:26:01,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:26:01,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:26:01,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +28: [2023-05-08 11:26:01,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 0: [2023-05-08 11:26:01,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 0: [2023-05-08 11:26:01,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 0: [2023-05-08 11:26:01,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +28: [2023-05-08 11:26:01,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +28: [2023-05-08 11:26:01,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 7: [2023-05-08 11:26:01,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:26:01,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +18: [2023-05-08 11:26:01,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +18: [2023-05-08 11:26:01,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +18: [2023-05-08 11:26:01,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +19: [2023-05-08 11:26:01,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 0: [2023-05-08 11:26:01,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:26:01,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +19: [2023-05-08 11:26:01,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +24: [2023-05-08 11:26:01,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +19: [2023-05-08 11:26:01,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +24: [2023-05-08 11:26:01,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 1: [2023-05-08 11:26:01,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 1: [2023-05-08 11:26:01,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 1: [2023-05-08 11:26:01,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 1: [2023-05-08 11:26:01,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 0: [2023-05-08 11:26:01,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +12: [2023-05-08 11:26:01,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +12: [2023-05-08 11:26:01,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +12: [2023-05-08 11:26:01,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +12: [2023-05-08 11:26:01,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 8: [2023-05-08 11:26:01,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 8: [2023-05-08 11:26:01,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 8: [2023-05-08 11:26:01,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 8: [2023-05-08 11:26:01,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 0: [2023-05-08 11:26:01,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +24: [2023-05-08 11:26:01,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:26:01,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +10: [2023-05-08 11:26:01,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:26:01,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:26:01,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:26:01,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +10: [2023-05-08 11:26:01,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:26:01,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +10: [2023-05-08 11:26:01,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +10: [2023-05-08 11:26:01,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +18: [2023-05-08 11:26:01,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +18: [2023-05-08 11:26:01,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +18: [2023-05-08 11:26:01,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:26:01,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +24: [2023-05-08 11:26:01,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:26:01,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:26:01,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +24: [2023-05-08 11:26:01,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +19: [2023-05-08 11:26:01,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +19: [2023-05-08 11:26:01,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +12: [2023-05-08 11:26:01,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:26:01,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +19: [2023-05-08 11:26:01,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:26:01,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +29: [2023-05-08 11:26:01,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:26:01,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:26:01,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +29: [2023-05-08 11:26:01,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +29: [2023-05-08 11:26:01,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +29: [2023-05-08 11:26:01,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +12: [2023-05-08 11:26:01,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:26:01,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +12: [2023-05-08 11:26:01,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +23: [2023-05-08 11:26:01,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +23: [2023-05-08 11:26:01,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +23: [2023-05-08 11:26:01,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +23: [2023-05-08 11:26:01,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +12: [2023-05-08 11:26:01,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +10: [2023-05-08 11:26:01,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:26:01,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +24: [2023-05-08 11:26:01,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +24: [2023-05-08 11:26:01,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +24: [2023-05-08 11:26:01,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:26:01,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:26:01,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:26:01,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +24: [2023-05-08 11:26:01,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:26:01,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:26:01,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +29: [2023-05-08 11:26:01,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +29: [2023-05-08 11:26:01,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 0: [2023-05-08 11:26:01,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 7: [2023-05-08 11:26:01,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:26:01,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +24: [2023-05-08 11:26:01,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +24: [2023-05-08 11:26:01,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:26:01,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +18: [2023-05-08 11:26:01,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +19: [2023-05-08 11:26:01,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +19: [2023-05-08 11:26:01,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:26:01,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:26:01,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:26:01,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +12: [2023-05-08 11:26:01,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +19: [2023-05-08 11:26:01,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +19: [2023-05-08 11:26:01,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:26:01,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +19: [2023-05-08 11:26:01,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +19: [2023-05-08 11:26:01,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +18: [2023-05-08 11:26:01,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +18: [2023-05-08 11:26:01,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +18: [2023-05-08 11:26:01,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +18: [2023-05-08 11:26:01,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +19: [2023-05-08 11:26:01,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:26:01,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +12: [2023-05-08 11:26:01,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +23: [2023-05-08 11:26:01,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +23: [2023-05-08 11:26:01,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 0: [2023-05-08 11:26:01,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 1: [2023-05-08 11:26:01,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 1: [2023-05-08 11:26:01,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +10: [2023-05-08 11:26:01,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +27: [2023-05-08 11:26:01,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +27: [2023-05-08 11:26:01,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +27: [2023-05-08 11:26:01,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +27: [2023-05-08 11:26:01,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +27: [2023-05-08 11:26:01,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +27: [2023-05-08 11:26:01,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 8: [2023-05-08 11:26:01,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 8: [2023-05-08 11:26:01,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 8: [2023-05-08 11:26:01,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +27: [2023-05-08 11:26:01,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 7: [2023-05-08 11:26:01,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +12: [2023-05-08 11:26:01,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +12: [2023-05-08 11:26:01,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +26: [2023-05-08 11:26:01,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +26: [2023-05-08 11:26:01,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +26: [2023-05-08 11:26:01,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:26:01,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +26: [2023-05-08 11:26:01,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:26:01,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:26:01,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +26: [2023-05-08 11:26:01,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:26:01,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +27: [2023-05-08 11:26:01,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +12: [2023-05-08 11:26:01,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +29: [2023-05-08 11:26:01,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:26:01,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:26:01,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +29: [2023-05-08 11:26:01,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +29: [2023-05-08 11:26:01,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:26:01,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:26:01,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:26:01,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +19: [2023-05-08 11:26:01,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +29: [2023-05-08 11:26:01,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +29: [2023-05-08 11:26:01,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +19: [2023-05-08 11:26:01,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +19: [2023-05-08 11:26:01,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +10: [2023-05-08 11:26:01,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:26:01,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:26:01,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +24: [2023-05-08 11:26:01,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +19: [2023-05-08 11:26:01,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +27: [2023-05-08 11:26:01,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +27: [2023-05-08 11:26:01,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +19: [2023-05-08 11:26:01,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +19: [2023-05-08 11:26:01,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +27: [2023-05-08 11:26:01,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +19: [2023-05-08 11:26:01,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:26:01,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:26:01,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:26:01,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +24: [2023-05-08 11:26:01,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:26:01,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:26:01,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:26:01,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +22: [2023-05-08 11:26:01,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +22: [2023-05-08 11:26:01,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +24: [2023-05-08 11:26:01,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +22: [2023-05-08 11:26:01,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +23: [2023-05-08 11:26:01,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +23: [2023-05-08 11:26:01,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +10: [2023-05-08 11:26:01,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +29: [2023-05-08 11:26:01,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +24: [2023-05-08 11:26:01,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +19: [2023-05-08 11:26:01,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:26:01,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:26:01,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:26:01,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:26:01,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +19: [2023-05-08 11:26:01,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +15: [2023-05-08 11:26:01,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +15: [2023-05-08 11:26:01,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +15: [2023-05-08 11:26:01,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:26:01,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:26:01,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:26:01,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:26:01,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:26:01,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +15: [2023-05-08 11:26:01,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:26:01,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 5: [2023-05-08 11:26:01,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +30: [2023-05-08 11:26:01,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +30: [2023-05-08 11:26:01,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:26:01,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +30: [2023-05-08 11:26:01,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +30: [2023-05-08 11:26:01,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:26:01,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:26:01,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +12: [2023-05-08 11:26:01,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:26:01,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +12: [2023-05-08 11:26:01,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:26:01,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:26:01,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:26:01,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:26:01,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:26:01,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:26:01,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +27: [2023-05-08 11:26:01,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:26:01,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:26:01,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:26:01,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +27: [2023-05-08 11:26:01,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +13: [2023-05-08 11:26:01,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +13: [2023-05-08 11:26:01,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 2: [2023-05-08 11:26:01,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. + 2: [2023-05-08 11:26:01,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +13: [2023-05-08 11:26:01,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +13: [2023-05-08 11:26:01,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +23: [2023-05-08 11:26:01,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +26: [2023-05-08 11:26:01,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +22: [2023-05-08 11:26:01,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:26:01,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt. +27: [2023-05-08 11:26:01,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:26:01,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:26:01,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:26:01,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +29: [2023-05-08 11:26:01,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +23: [2023-05-08 11:26:01,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +19: [2023-05-08 11:26:01,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +29: [2023-05-08 11:26:01,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 9: [2023-05-08 11:26:01,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +24: [2023-05-08 11:26:01,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,162] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:26:01,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:26:01,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:26:01,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +26: [2023-05-08 11:26:01,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:26:01,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +29: [2023-05-08 11:26:01,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +29: [2023-05-08 11:26:01,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +29: [2023-05-08 11:26:01,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +15: [2023-05-08 11:26:01,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:26:01,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +30: [2023-05-08 11:26:01,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +27: [2023-05-08 11:26:01,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +13: [2023-05-08 11:26:01,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +13: [2023-05-08 11:26:01,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +22: [2023-05-08 11:26:01,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +24: [2023-05-08 11:26:01,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:26:01,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:26:01,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +21: [2023-05-08 11:26:01,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +21: [2023-05-08 11:26:01,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +21: [2023-05-08 11:26:01,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:26:01,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:26:01,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:26:01,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +29: [2023-05-08 11:26:01,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:26:01,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +13: [2023-05-08 11:26:01,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt... +21: [2023-05-08 11:26:01,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:26:01,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:26:01,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +15: [2023-05-08 11:26:01,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +29: [2023-05-08 11:26:01,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +12: [2023-05-08 11:26:01,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:26:01,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:26:01,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +12: [2023-05-08 11:26:01,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:26:01,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +15: [2023-05-08 11:26:01,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:26:01,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +30: [2023-05-08 11:26:01,176] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +29: [2023-05-08 11:26:01,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +17: [2023-05-08 11:26:01,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +17: [2023-05-08 11:26:01,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +17: [2023-05-08 11:26:01,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:26:01,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:26:01,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +17: [2023-05-08 11:26:01,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +27: [2023-05-08 11:26:01,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:26:01,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:26:01,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:26:01,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +17: [2023-05-08 11:26:01,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +22: [2023-05-08 11:26:01,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +22: [2023-05-08 11:26:01,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +22: [2023-05-08 11:26:01,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +22: [2023-05-08 11:26:01,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +22: [2023-05-08 11:26:01,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 1: [2023-05-08 11:26:01,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:26:01,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:26:01,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:26:01,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +28: [2023-05-08 11:26:01,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +28: [2023-05-08 11:26:01,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:26:01,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:26:01,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +20: [2023-05-08 11:26:01,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:26:01,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:26:01,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:26:01,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +20: [2023-05-08 11:26:01,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +20: [2023-05-08 11:26:01,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:26:01,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:26:01,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:26:01,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:26:01,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:26:01,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +20: [2023-05-08 11:26:01,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:26:01,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 9: [2023-05-08 11:26:01,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +12: [2023-05-08 11:26:01,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +15: [2023-05-08 11:26:01,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:26:01,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +30: [2023-05-08 11:26:01,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:26:01,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 2: [2023-05-08 11:26:01,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +13: [2023-05-08 11:26:01,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:26:01,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +15: [2023-05-08 11:26:01,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +21: [2023-05-08 11:26:01,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 5: [2023-05-08 11:26:01,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +21: [2023-05-08 11:26:01,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:26:01,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +22: [2023-05-08 11:26:01,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +22: [2023-05-08 11:26:01,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +22: [2023-05-08 11:26:01,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +22: [2023-05-08 11:26:01,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +22: [2023-05-08 11:26:01,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +12: [2023-05-08 11:26:01,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +12: [2023-05-08 11:26:01,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 9: [2023-05-08 11:26:01,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:26:01,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +22: [2023-05-08 11:26:01,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:26:01,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 1: [2023-05-08 11:26:01,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:26:01,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +14: [2023-05-08 11:26:01,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:26:01,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +14: [2023-05-08 11:26:01,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +14: [2023-05-08 11:26:01,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:26:01,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:26:01,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +14: [2023-05-08 11:26:01,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... +17: [2023-05-08 11:26:01,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:26:01,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:26:01,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. +13: [2023-05-08 11:26:01,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 9: [2023-05-08 11:26:01,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt... + 3: [2023-05-08 11:26:01,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +17: [2023-05-08 11:26:01,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +17: [2023-05-08 11:26:01,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +28: [2023-05-08 11:26:01,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:26:01,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +21: [2023-05-08 11:26:01,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:26:01,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:26:01,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:26:01,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:26:01,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +21: [2023-05-08 11:26:01,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +14: [2023-05-08 11:26:01,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +14: [2023-05-08 11:26:01,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:26:01,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:26:01,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +14: [2023-05-08 11:26:01,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +22: [2023-05-08 11:26:01,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:26:01,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:26:01,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +20: [2023-05-08 11:26:01,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:26:01,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:26:01,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:26:01,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:26:01,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:26:01,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:26:01,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:26:01,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +22: [2023-05-08 11:26:01,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +22: [2023-05-08 11:26:01,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +22: [2023-05-08 11:26:01,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:26:01,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +21: [2023-05-08 11:26:01,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:26:01,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:26:01,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +28: [2023-05-08 11:26:01,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:26:01,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:26:01,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:26:01,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:26:01,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:26:01,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:26:01,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +22: [2023-05-08 11:26:01,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +22: [2023-05-08 11:26:01,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:26:01,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +22: [2023-05-08 11:26:01,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +28: [2023-05-08 11:26:01,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:26:01,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +23: [2023-05-08 11:26:01,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +23: [2023-05-08 11:26:01,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +23: [2023-05-08 11:26:01,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:26:01,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:26:01,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:26:01,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:26:01,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:26:01,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +17: [2023-05-08 11:26:01,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +17: [2023-05-08 11:26:01,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +17: [2023-05-08 11:26:01,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +30: [2023-05-08 11:26:01,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +30: [2023-05-08 11:26:01,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +30: [2023-05-08 11:26:01,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +30: [2023-05-08 11:26:01,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 2: [2023-05-08 11:26:01,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:26:01,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:26:01,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:26:01,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:26:01,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:26:01,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:26:01,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:26:01,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:26:01,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +17: [2023-05-08 11:26:01,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:26:01,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:26:01,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +30: [2023-05-08 11:26:01,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:26:01,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +30: [2023-05-08 11:26:01,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:26:01,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +30: [2023-05-08 11:26:01,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:26:01,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +31: [2023-05-08 11:26:01,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +31: [2023-05-08 11:26:01,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +31: [2023-05-08 11:26:01,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:26:01,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:26:01,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:26:01,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +31: [2023-05-08 11:26:01,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:26:01,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +18: [2023-05-08 11:26:01,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:26:01,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +18: [2023-05-08 11:26:01,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:26:01,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:26:01,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:26:01,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +11: [2023-05-08 11:26:01,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +18: [2023-05-08 11:26:01,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:26:01,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +14: [2023-05-08 11:26:01,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +18: [2023-05-08 11:26:01,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:26:01,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +14: [2023-05-08 11:26:01,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +18: [2023-05-08 11:26:01,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:26:01,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 2: [2023-05-08 11:26:01,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +13: [2023-05-08 11:26:01,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +13: [2023-05-08 11:26:01,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +13: [2023-05-08 11:26:01,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +13: [2023-05-08 11:26:01,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:26:01,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:26:01,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:26:01,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:26:01,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:26:01,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:26:01,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +16: [2023-05-08 11:26:01,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +16: [2023-05-08 11:26:01,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +16: [2023-05-08 11:26:01,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:26:01,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:26:01,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:26:01,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +16: [2023-05-08 11:26:01,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:26:01,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:26:01,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,252] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:26:01,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:26:01,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +23: [2023-05-08 11:26:01,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +17: [2023-05-08 11:26:01,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +23: [2023-05-08 11:26:01,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:26:01,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +17: [2023-05-08 11:26:01,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:26:01,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +23: [2023-05-08 11:26:01,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 3: [2023-05-08 11:26:01,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 3: [2023-05-08 11:26:01,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 6: [2023-05-08 11:26:01,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:26:01,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +14: [2023-05-08 11:26:01,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +18: [2023-05-08 11:26:01,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 6: [2023-05-08 11:26:01,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:26:01,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 7: [2023-05-08 11:26:01,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +18: [2023-05-08 11:26:01,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +11: [2023-05-08 11:26:01,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 4: [2023-05-08 11:26:01,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +30: [2023-05-08 11:26:01,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +11: [2023-05-08 11:26:01,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +23: [2023-05-08 11:26:01,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:26:01,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +17: [2023-05-08 11:26:01,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +17: [2023-05-08 11:26:01,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +31: [2023-05-08 11:26:01,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +18: [2023-05-08 11:26:01,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:26:01,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:26:01,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +31: [2023-05-08 11:26:01,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:26:01,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:26:01,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:26:01,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +14: [2023-05-08 11:26:01,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +14: [2023-05-08 11:26:01,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +11: [2023-05-08 11:26:01,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:26:01,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:26:01,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 5: [2023-05-08 11:26:01,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:26:01,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:26:01,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:26:01,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +25: [2023-05-08 11:26:01,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +25: [2023-05-08 11:26:01,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +25: [2023-05-08 11:26:01,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +15: [2023-05-08 11:26:01,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +15: [2023-05-08 11:26:01,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +15: [2023-05-08 11:26:01,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +15: [2023-05-08 11:26:01,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +13: [2023-05-08 11:26:01,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +13: [2023-05-08 11:26:01,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +16: [2023-05-08 11:26:01,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +16: [2023-05-08 11:26:01,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 9: [2023-05-08 11:26:01,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +30: [2023-05-08 11:26:01,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +30: [2023-05-08 11:26:01,267] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 9: [2023-05-08 11:26:01,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +11: [2023-05-08 11:26:01,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +31: [2023-05-08 11:26:01,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:26:01,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +18: [2023-05-08 11:26:01,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +31: [2023-05-08 11:26:01,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +30: [2023-05-08 11:26:01,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +18: [2023-05-08 11:26:01,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:26:01,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +23: [2023-05-08 11:26:01,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +18: [2023-05-08 11:26:01,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:26:01,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:26:01,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 3: [2023-05-08 11:26:01,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:26:01,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +16: [2023-05-08 11:26:01,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:26:01,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +11: [2023-05-08 11:26:01,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +14: [2023-05-08 11:26:01,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +18: [2023-05-08 11:26:01,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:26:01,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:26:01,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +25: [2023-05-08 11:26:01,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +25: [2023-05-08 11:26:01,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:26:01,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:26:01,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:26:01,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +25: [2023-05-08 11:26:01,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +16: [2023-05-08 11:26:01,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:26:01,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:26:01,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:26:01,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:26:01,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +14: [2023-05-08 11:26:01,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +15: [2023-05-08 11:26:01,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +11: [2023-05-08 11:26:01,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +16: [2023-05-08 11:26:01,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:26:01,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:26:01,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +16: [2023-05-08 11:26:01,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:26:01,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +21: [2023-05-08 11:26:01,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +21: [2023-05-08 11:26:01,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +21: [2023-05-08 11:26:01,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 6: [2023-05-08 11:26:01,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:26:01,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +14: [2023-05-08 11:26:01,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +14: [2023-05-08 11:26:01,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 7: [2023-05-08 11:26:01,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 6: [2023-05-08 11:26:01,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:26:01,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +11: [2023-05-08 11:26:01,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:26:01,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:26:01,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +11: [2023-05-08 11:26:01,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 7: [2023-05-08 11:26:01,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +14: [2023-05-08 11:26:01,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:26:01,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:26:01,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:26:01,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +25: [2023-05-08 11:26:01,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +20: [2023-05-08 11:26:01,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +20: [2023-05-08 11:26:01,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +20: [2023-05-08 11:26:01,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 4: [2023-05-08 11:26:01,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +11: [2023-05-08 11:26:01,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +11: [2023-05-08 11:26:01,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 4: [2023-05-08 11:26:01,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 3: [2023-05-08 11:26:01,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 7: [2023-05-08 11:26:01,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:26:01,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +13: [2023-05-08 11:26:01,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +31: [2023-05-08 11:26:01,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +31: [2023-05-08 11:26:01,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +31: [2023-05-08 11:26:01,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +14: [2023-05-08 11:26:01,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +15: [2023-05-08 11:26:01,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +31: [2023-05-08 11:26:01,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +21: [2023-05-08 11:26:01,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 9: [2023-05-08 11:26:01,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +15: [2023-05-08 11:26:01,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 9: [2023-05-08 11:26:01,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +16: [2023-05-08 11:26:01,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +16: [2023-05-08 11:26:01,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 6: [2023-05-08 11:26:01,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +16: [2023-05-08 11:26:01,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 6: [2023-05-08 11:26:01,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 6: [2023-05-08 11:26:01,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 9: [2023-05-08 11:26:01,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +16: [2023-05-08 11:26:01,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +11: [2023-05-08 11:26:01,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +11: [2023-05-08 11:26:01,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +20: [2023-05-08 11:26:01,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 9: [2023-05-08 11:26:01,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +21: [2023-05-08 11:26:01,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:26:01,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +15: [2023-05-08 11:26:01,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +21: [2023-05-08 11:26:01,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 7: [2023-05-08 11:26:01,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 4: [2023-05-08 11:26:01,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +25: [2023-05-08 11:26:01,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 8: [2023-05-08 11:26:01,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 8: [2023-05-08 11:26:01,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 4: [2023-05-08 11:26:01,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 3: [2023-05-08 11:26:01,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +20: [2023-05-08 11:26:01,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,302] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +25: [2023-05-08 11:26:01,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 4: [2023-05-08 11:26:01,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:26:01,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:26:01,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +20: [2023-05-08 11:26:01,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:26:01,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +28: [2023-05-08 11:26:01,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +31: [2023-05-08 11:26:01,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +16: [2023-05-08 11:26:01,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:26:01,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +28: [2023-05-08 11:26:01,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +28: [2023-05-08 11:26:01,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +28: [2023-05-08 11:26:01,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +16: [2023-05-08 11:26:01,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 6: [2023-05-08 11:26:01,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:26:01,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:26:01,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +16: [2023-05-08 11:26:01,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:26:01,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:26:01,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +21: [2023-05-08 11:26:01,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 9: [2023-05-08 11:26:01,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +16: [2023-05-08 11:26:01,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:26:01,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +27: [2023-05-08 11:26:01,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +27: [2023-05-08 11:26:01,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +27: [2023-05-08 11:26:01,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 7: [2023-05-08 11:26:01,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +15: [2023-05-08 11:26:01,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +10: [2023-05-08 11:26:01,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +10: [2023-05-08 11:26:01,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +10: [2023-05-08 11:26:01,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +10: [2023-05-08 11:26:01,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +15: [2023-05-08 11:26:01,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +21: [2023-05-08 11:26:01,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:26:01,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +26: [2023-05-08 11:26:01,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +24: [2023-05-08 11:26:01,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +26: [2023-05-08 11:26:01,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +26: [2023-05-08 11:26:01,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +26: [2023-05-08 11:26:01,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +24: [2023-05-08 11:26:01,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +24: [2023-05-08 11:26:01,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +24: [2023-05-08 11:26:01,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 8: [2023-05-08 11:26:01,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +28: [2023-05-08 11:26:01,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +28: [2023-05-08 11:26:01,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +28: [2023-05-08 11:26:01,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +28: [2023-05-08 11:26:01,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +27: [2023-05-08 11:26:01,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:26:01,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:26:01,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:26:01,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:26:01,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +31: [2023-05-08 11:26:01,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:26:01,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:26:01,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 7: [2023-05-08 11:26:01,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:26:01,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:26:01,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:26:01,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +16: [2023-05-08 11:26:01,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +16: [2023-05-08 11:26:01,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +20: [2023-05-08 11:26:01,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:26:01,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +26: [2023-05-08 11:26:01,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +24: [2023-05-08 11:26:01,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:26:01,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +10: [2023-05-08 11:26:01,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +20: [2023-05-08 11:26:01,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 0: [2023-05-08 11:26:01,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +16: [2023-05-08 11:26:01,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +16: [2023-05-08 11:26:01,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 0: [2023-05-08 11:26:01,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 0: [2023-05-08 11:26:01,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 0: [2023-05-08 11:26:01,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 6: [2023-05-08 11:26:01,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:26:01,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +26: [2023-05-08 11:26:01,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +24: [2023-05-08 11:26:01,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +24: [2023-05-08 11:26:01,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:26:01,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +26: [2023-05-08 11:26:01,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +21: [2023-05-08 11:26:01,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +24: [2023-05-08 11:26:01,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +24: [2023-05-08 11:26:01,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +24: [2023-05-08 11:26:01,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +21: [2023-05-08 11:26:01,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:26:01,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +24: [2023-05-08 11:26:01,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:26:01,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 1: [2023-05-08 11:26:01,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +26: [2023-05-08 11:26:01,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +31: [2023-05-08 11:26:01,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:26:01,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +18: [2023-05-08 11:26:01,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +18: [2023-05-08 11:26:01,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +18: [2023-05-08 11:26:01,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +20: [2023-05-08 11:26:01,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +21: [2023-05-08 11:26:01,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 7: [2023-05-08 11:26:01,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +18: [2023-05-08 11:26:01,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 6: [2023-05-08 11:26:01,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +20: [2023-05-08 11:26:01,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 1: [2023-05-08 11:26:01,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 1: [2023-05-08 11:26:01,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 6: [2023-05-08 11:26:01,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +16: [2023-05-08 11:26:01,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 6: [2023-05-08 11:26:01,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +16: [2023-05-08 11:26:01,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +10: [2023-05-08 11:26:01,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:26:01,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:26:01,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:26:01,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +27: [2023-05-08 11:26:01,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +28: [2023-05-08 11:26:01,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +31: [2023-05-08 11:26:01,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 0: [2023-05-08 11:26:01,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +19: [2023-05-08 11:26:01,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 0: [2023-05-08 11:26:01,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:26:01,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +19: [2023-05-08 11:26:01,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +19: [2023-05-08 11:26:01,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +20: [2023-05-08 11:26:01,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +28: [2023-05-08 11:26:01,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +19: [2023-05-08 11:26:01,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +16: [2023-05-08 11:26:01,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +28: [2023-05-08 11:26:01,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +20: [2023-05-08 11:26:01,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 0: [2023-05-08 11:26:01,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +24: [2023-05-08 11:26:01,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +16: [2023-05-08 11:26:01,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +28: [2023-05-08 11:26:01,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 1: [2023-05-08 11:26:01,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:26:01,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +26: [2023-05-08 11:26:01,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:26:01,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +27: [2023-05-08 11:26:01,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:26:01,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:26:01,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +18: [2023-05-08 11:26:01,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:26:01,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +12: [2023-05-08 11:26:01,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +12: [2023-05-08 11:26:01,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +28: [2023-05-08 11:26:01,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +12: [2023-05-08 11:26:01,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +28: [2023-05-08 11:26:01,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +18: [2023-05-08 11:26:01,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +18: [2023-05-08 11:26:01,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +26: [2023-05-08 11:26:01,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 1: [2023-05-08 11:26:01,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 1: [2023-05-08 11:26:01,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +24: [2023-05-08 11:26:01,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +13: [2023-05-08 11:26:01,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +13: [2023-05-08 11:26:01,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +13: [2023-05-08 11:26:01,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +13: [2023-05-08 11:26:01,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 8: [2023-05-08 11:26:01,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +28: [2023-05-08 11:26:01,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 5: [2023-05-08 11:26:01,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 5: [2023-05-08 11:26:01,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 5: [2023-05-08 11:26:01,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 5: [2023-05-08 11:26:01,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +19: [2023-05-08 11:26:01,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +24: [2023-05-08 11:26:01,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:26:01,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +28: [2023-05-08 11:26:01,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 8: [2023-05-08 11:26:01,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +25: [2023-05-08 11:26:01,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:26:01,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:26:01,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:26:01,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:26:01,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:26:01,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +25: [2023-05-08 11:26:01,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +19: [2023-05-08 11:26:01,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 2: [2023-05-08 11:26:01,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. + 2: [2023-05-08 11:26:01,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +10: [2023-05-08 11:26:01,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 2: [2023-05-08 11:26:01,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +25: [2023-05-08 11:26:01,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +24: [2023-05-08 11:26:01,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +25: [2023-05-08 11:26:01,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +22: [2023-05-08 11:26:01,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +22: [2023-05-08 11:26:01,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +22: [2023-05-08 11:26:01,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +22: [2023-05-08 11:26:01,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt. +19: [2023-05-08 11:26:01,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +24: [2023-05-08 11:26:01,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:26:01,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:26:01,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:26:01,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:26:01,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:26:01,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +12: [2023-05-08 11:26:01,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +18: [2023-05-08 11:26:01,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +10: [2023-05-08 11:26:01,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +12: [2023-05-08 11:26:01,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +13: [2023-05-08 11:26:01,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:26:01,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +26: [2023-05-08 11:26:01,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +18: [2023-05-08 11:26:01,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +13: [2023-05-08 11:26:01,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +13: [2023-05-08 11:26:01,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +13: [2023-05-08 11:26:01,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +26: [2023-05-08 11:26:01,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +12: [2023-05-08 11:26:01,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:26:01,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 0: [2023-05-08 11:26:01,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 1: [2023-05-08 11:26:01,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +26: [2023-05-08 11:26:01,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 1: [2023-05-08 11:26:01,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 5: [2023-05-08 11:26:01,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +27: [2023-05-08 11:26:01,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 2: [2023-05-08 11:26:01,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +22: [2023-05-08 11:26:01,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +22: [2023-05-08 11:26:01,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +22: [2023-05-08 11:26:01,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +26: [2023-05-08 11:26:01,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:26:01,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:26:01,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +22: [2023-05-08 11:26:01,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 2: [2023-05-08 11:26:01,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... +26: [2023-05-08 11:26:01,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +24: [2023-05-08 11:26:01,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 2: [2023-05-08 11:26:01,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:26:01,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:26:01,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:26:01,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +24: [2023-05-08 11:26:01,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +25: [2023-05-08 11:26:01,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:26:01,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:26:01,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:26:01,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +19: [2023-05-08 11:26:01,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +19: [2023-05-08 11:26:01,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +18: [2023-05-08 11:26:01,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:26:01,363] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:26:01,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +13: [2023-05-08 11:26:01,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:26:01,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +18: [2023-05-08 11:26:01,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +18: [2023-05-08 11:26:01,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +24: [2023-05-08 11:26:01,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +18: [2023-05-08 11:26:01,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +22: [2023-05-08 11:26:01,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:26:01,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +13: [2023-05-08 11:26:01,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 5: [2023-05-08 11:26:01,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 5: [2023-05-08 11:26:01,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +25: [2023-05-08 11:26:01,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +13: [2023-05-08 11:26:01,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 5: [2023-05-08 11:26:01,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +12: [2023-05-08 11:26:01,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +12: [2023-05-08 11:26:01,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +12: [2023-05-08 11:26:01,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +12: [2023-05-08 11:26:01,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +19: [2023-05-08 11:26:01,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:26:01,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:26:01,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +25: [2023-05-08 11:26:01,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +13: [2023-05-08 11:26:01,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +13: [2023-05-08 11:26:01,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... + 2: [2023-05-08 11:26:01,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +19: [2023-05-08 11:26:01,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +12: [2023-05-08 11:26:01,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +12: [2023-05-08 11:26:01,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +12: [2023-05-08 11:26:01,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +22: [2023-05-08 11:26:01,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:26:01,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +22: [2023-05-08 11:26:01,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +22: [2023-05-08 11:26:01,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +24: [2023-05-08 11:26:01,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +19: [2023-05-08 11:26:01,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +19: [2023-05-08 11:26:01,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +19: [2023-05-08 11:26:01,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +19: [2023-05-08 11:26:01,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +25: [2023-05-08 11:26:01,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:26:01,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:26:01,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:26:01,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:26:01,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:26:01,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:26:01,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +24: [2023-05-08 11:26:01,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:26:01,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:26:01,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:26:01,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:26:01,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:26:01,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:26:01,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +15: [2023-05-08 11:26:01,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:26:01,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:26:01,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:26:01,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:26:01,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +22: [2023-05-08 11:26:01,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. +24: [2023-05-08 11:26:01,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:26:01,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +25: [2023-05-08 11:26:01,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt... +26: [2023-05-08 11:26:01,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:26:01,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +27: [2023-05-08 11:26:01,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +27: [2023-05-08 11:26:01,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +27: [2023-05-08 11:26:01,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +27: [2023-05-08 11:26:01,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:26:01,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +27: [2023-05-08 11:26:01,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:26:01,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:26:01,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:26:01,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:26:01,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:26:01,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:26:01,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +23: [2023-05-08 11:26:01,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +23: [2023-05-08 11:26:01,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +23: [2023-05-08 11:26:01,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +23: [2023-05-08 11:26:01,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +26: [2023-05-08 11:26:01,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:26:01,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:26:01,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +27: [2023-05-08 11:26:01,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +15: [2023-05-08 11:26:01,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:26:01,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +12: [2023-05-08 11:26:01,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +12: [2023-05-08 11:26:01,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +12: [2023-05-08 11:26:01,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +29: [2023-05-08 11:26:01,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:26:01,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:26:01,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +29: [2023-05-08 11:26:01,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +29: [2023-05-08 11:26:01,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +29: [2023-05-08 11:26:01,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +29: [2023-05-08 11:26:01,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +29: [2023-05-08 11:26:01,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +26: [2023-05-08 11:26:01,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +23: [2023-05-08 11:26:01,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:26:01,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:26:01,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +27: [2023-05-08 11:26:01,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:26:01,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +12: [2023-05-08 11:26:01,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:26:01,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +23: [2023-05-08 11:26:01,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:26:01,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:26:01,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:26:01,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +29: [2023-05-08 11:26:01,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +29: [2023-05-08 11:26:01,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:26:01,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:26:01,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +12: [2023-05-08 11:26:01,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:26:01,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:26:01,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:26:01,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:26:01,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:26:01,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:26:01,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:26:01,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:26:01,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:26:01,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:26:01,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:26:01,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:26:01,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:26:01,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +21: [2023-05-08 11:26:01,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:26:01,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:26:01,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:26:01,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:26:01,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +21: [2023-05-08 11:26:01,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +29: [2023-05-08 11:26:01,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:26:01,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:26:01,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +27: [2023-05-08 11:26:01,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:26:01,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:26:01,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:26:01,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +29: [2023-05-08 11:26:01,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:26:01,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:26:01,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 9: [2023-05-08 11:26:01,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:26:01,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:26:01,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +17: [2023-05-08 11:26:01,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:26:01,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:26:01,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:26:01,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +27: [2023-05-08 11:26:01,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:26:01,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +23: [2023-05-08 11:26:01,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 2: [2023-05-08 11:26:01,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:26:01,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +21: [2023-05-08 11:26:01,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:26:01,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:26:01,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:26:01,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:26:01,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:26:01,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:26:01,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:26:01,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:26:01,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:26:01,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:26:01,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +23: [2023-05-08 11:26:01,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +23: [2023-05-08 11:26:01,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 2: [2023-05-08 11:26:01,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +21: [2023-05-08 11:26:01,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:26:01,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:26:01,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:26:01,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:26:01,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +21: [2023-05-08 11:26:01,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:26:01,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +17: [2023-05-08 11:26:01,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +17: [2023-05-08 11:26:01,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:26:01,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +17: [2023-05-08 11:26:01,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:26:01,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +29: [2023-05-08 11:26:01,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:26:01,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:26:01,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:26:01,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +29: [2023-05-08 11:26:01,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:26:01,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +30: [2023-05-08 11:26:01,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:26:01,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +30: [2023-05-08 11:26:01,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +21: [2023-05-08 11:26:01,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +29: [2023-05-08 11:26:01,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +29: [2023-05-08 11:26:01,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +29: [2023-05-08 11:26:01,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +30: [2023-05-08 11:26:01,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 2: [2023-05-08 11:26:01,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:26:01,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:26:01,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:26:01,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +29: [2023-05-08 11:26:01,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:26:01,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 9: [2023-05-08 11:26:01,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:26:01,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +12: [2023-05-08 11:26:01,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +29: [2023-05-08 11:26:01,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:26:01,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +29: [2023-05-08 11:26:01,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +29: [2023-05-08 11:26:01,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:26:01,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:26:01,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:26:01,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:26:01,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:26:01,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +19: [2023-05-08 11:26:01,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:26:01,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +19: [2023-05-08 11:26:01,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +19: [2023-05-08 11:26:01,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:26:01,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:26:01,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +19: [2023-05-08 11:26:01,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:26:01,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:26:01,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:26:01,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +29: [2023-05-08 11:26:01,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:26:01,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +29: [2023-05-08 11:26:01,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:26:01,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +10: [2023-05-08 11:26:01,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +10: [2023-05-08 11:26:01,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +10: [2023-05-08 11:26:01,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +10: [2023-05-08 11:26:01,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +10: [2023-05-08 11:26:01,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +10: [2023-05-08 11:26:01,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +10: [2023-05-08 11:26:01,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +10: [2023-05-08 11:26:01,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:26:01,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:26:01,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +19: [2023-05-08 11:26:01,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:26:01,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:26:01,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +30: [2023-05-08 11:26:01,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +30: [2023-05-08 11:26:01,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +30: [2023-05-08 11:26:01,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +19: [2023-05-08 11:26:01,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:26:01,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +10: [2023-05-08 11:26:01,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:26:01,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +19: [2023-05-08 11:26:01,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +10: [2023-05-08 11:26:01,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:26:01,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:26:01,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +13: [2023-05-08 11:26:01,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:26:01,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:26:01,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +13: [2023-05-08 11:26:01,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +13: [2023-05-08 11:26:01,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:26:01,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +19: [2023-05-08 11:26:01,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +19: [2023-05-08 11:26:01,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:26:01,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +10: [2023-05-08 11:26:01,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:26:01,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +10: [2023-05-08 11:26:01,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +19: [2023-05-08 11:26:01,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +10: [2023-05-08 11:26:01,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +10: [2023-05-08 11:26:01,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +13: [2023-05-08 11:26:01,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:26:01,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:26:01,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +28: [2023-05-08 11:26:01,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +28: [2023-05-08 11:26:01,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +28: [2023-05-08 11:26:01,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:26:01,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:26:01,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +10: [2023-05-08 11:26:01,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:26:01,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:26:01,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:26:01,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:26:01,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +23: [2023-05-08 11:26:01,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +23: [2023-05-08 11:26:01,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +23: [2023-05-08 11:26:01,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:26:01,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:26:01,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:26:01,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +28: [2023-05-08 11:26:01,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:26:01,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:26:01,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +14: [2023-05-08 11:26:01,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +14: [2023-05-08 11:26:01,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +14: [2023-05-08 11:26:01,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +13: [2023-05-08 11:26:01,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:26:01,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:26:01,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:26:01,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:26:01,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:26:01,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:26:01,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +13: [2023-05-08 11:26:01,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +23: [2023-05-08 11:26:01,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 9: [2023-05-08 11:26:01,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 9: [2023-05-08 11:26:01,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +15: [2023-05-08 11:26:01,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +15: [2023-05-08 11:26:01,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:26:01,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +28: [2023-05-08 11:26:01,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:26:01,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +15: [2023-05-08 11:26:01,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +15: [2023-05-08 11:26:01,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +15: [2023-05-08 11:26:01,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:26:01,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:26:01,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:26:01,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:26:01,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +25: [2023-05-08 11:26:01,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +25: [2023-05-08 11:26:01,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +25: [2023-05-08 11:26:01,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +25: [2023-05-08 11:26:01,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +25: [2023-05-08 11:26:01,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +25: [2023-05-08 11:26:01,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +24: [2023-05-08 11:26:01,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +25: [2023-05-08 11:26:01,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +24: [2023-05-08 11:26:01,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +23: [2023-05-08 11:26:01,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +24: [2023-05-08 11:26:01,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +24: [2023-05-08 11:26:01,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +24: [2023-05-08 11:26:01,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +24: [2023-05-08 11:26:01,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +24: [2023-05-08 11:26:01,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +23: [2023-05-08 11:26:01,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:26:01,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:26:01,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +23: [2023-05-08 11:26:01,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:26:01,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 8: [2023-05-08 11:26:01,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +31: [2023-05-08 11:26:01,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +31: [2023-05-08 11:26:01,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +28: [2023-05-08 11:26:01,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:26:01,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:26:01,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +18: [2023-05-08 11:26:01,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:26:01,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:26:01,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:26:01,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:26:01,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:26:01,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +22: [2023-05-08 11:26:01,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:26:01,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:26:01,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:26:01,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:26:01,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:26:01,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +28: [2023-05-08 11:26:01,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:26:01,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:26:01,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:26:01,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:26:01,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +22: [2023-05-08 11:26:01,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:26:01,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:26:01,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:26:01,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:26:01,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +16: [2023-05-08 11:26:01,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:26:01,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:26:01,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +16: [2023-05-08 11:26:01,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:26:01,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +24: [2023-05-08 11:26:01,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:26:01,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:26:01,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:26:01,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:26:01,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:26:01,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +22: [2023-05-08 11:26:01,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +22: [2023-05-08 11:26:01,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:26:01,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:26:01,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:26:01,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:26:01,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +20: [2023-05-08 11:26:01,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +20: [2023-05-08 11:26:01,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +20: [2023-05-08 11:26:01,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:26:01,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:26:01,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:26:01,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +14: [2023-05-08 11:26:01,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:26:01,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:26:01,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:26:01,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:26:01,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:26:01,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +20: [2023-05-08 11:26:01,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +31: [2023-05-08 11:26:01,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:26:01,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:26:01,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +15: [2023-05-08 11:26:01,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +15: [2023-05-08 11:26:01,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:26:01,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:26:01,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +24: [2023-05-08 11:26:01,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:26:01,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +25: [2023-05-08 11:26:01,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:26:01,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +18: [2023-05-08 11:26:01,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +14: [2023-05-08 11:26:01,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:26:01,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 9: [2023-05-08 11:26:01,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 8: [2023-05-08 11:26:01,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +14: [2023-05-08 11:26:01,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:26:01,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 9: [2023-05-08 11:26:01,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +25: [2023-05-08 11:26:01,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +25: [2023-05-08 11:26:01,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +25: [2023-05-08 11:26:01,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +18: [2023-05-08 11:26:01,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:26:01,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:26:01,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:26:01,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:26:01,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:26:01,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:26:01,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:26:01,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +14: [2023-05-08 11:26:01,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:26:01,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:26:01,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:26:01,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +26: [2023-05-08 11:26:01,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:26:01,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:26:01,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:26:01,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +26: [2023-05-08 11:26:01,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:26:01,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +26: [2023-05-08 11:26:01,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:26:01,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:26:01,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:26:01,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:26:01,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:26:01,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:26:01,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:26:01,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +31: [2023-05-08 11:26:01,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:26:01,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:26:01,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:26:01,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +24: [2023-05-08 11:26:01,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +16: [2023-05-08 11:26:01,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:26:01,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:26:01,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +24: [2023-05-08 11:26:01,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:26:01,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:26:01,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:26:01,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:26:01,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:26:01,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:26:01,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +18: [2023-05-08 11:26:01,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:26:01,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +14: [2023-05-08 11:26:01,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:26:01,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +31: [2023-05-08 11:26:01,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 8: [2023-05-08 11:26:01,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +18: [2023-05-08 11:26:01,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:26:01,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:26:01,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:26:01,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +25: [2023-05-08 11:26:01,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:26:01,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:26:01,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:26:01,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:26:01,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +25: [2023-05-08 11:26:01,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:26:01,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +31: [2023-05-08 11:26:01,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +24: [2023-05-08 11:26:01,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:26:01,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:26:01,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:26:01,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +20: [2023-05-08 11:26:01,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +20: [2023-05-08 11:26:01,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +20: [2023-05-08 11:26:01,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:26:01,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +15: [2023-05-08 11:26:01,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +15: [2023-05-08 11:26:01,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +15: [2023-05-08 11:26:01,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +16: [2023-05-08 11:26:01,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +25: [2023-05-08 11:26:01,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +31: [2023-05-08 11:26:01,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:26:01,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:26:01,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:26:01,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:26:01,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:26:01,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:26:01,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +26: [2023-05-08 11:26:01,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:26:01,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:26:01,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:26:01,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:26:01,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:26:01,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +24: [2023-05-08 11:26:01,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +18: [2023-05-08 11:26:01,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +20: [2023-05-08 11:26:01,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:26:01,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:26:01,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:26:01,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:26:01,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:26:01,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:26:01,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:26:01,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:26:01,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:26:01,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:26:01,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:26:01,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:26:01,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:26:01,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:26:01,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:26:01,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:26:01,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +19: [2023-05-08 11:26:01,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +19: [2023-05-08 11:26:01,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +19: [2023-05-08 11:26:01,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +19: [2023-05-08 11:26:01,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +19: [2023-05-08 11:26:01,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +20: [2023-05-08 11:26:01,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:26:01,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +19: [2023-05-08 11:26:01,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +19: [2023-05-08 11:26:01,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:26:01,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:26:01,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:26:01,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:26:01,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:26:01,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:26:01,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:26:01,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:26:01,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:26:01,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:26:01,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +25: [2023-05-08 11:26:01,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:26:01,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +26: [2023-05-08 11:26:01,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:26:01,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +25: [2023-05-08 11:26:01,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +11: [2023-05-08 11:26:01,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:26:01,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:26:01,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:26:01,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +31: [2023-05-08 11:26:01,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:26:01,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +20: [2023-05-08 11:26:01,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:26:01,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:26:01,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:26:01,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:26:01,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:26:01,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +22: [2023-05-08 11:26:01,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:26:01,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:26:01,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:26:01,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +15: [2023-05-08 11:26:01,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +15: [2023-05-08 11:26:01,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +30: [2023-05-08 11:26:01,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:26:01,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +15: [2023-05-08 11:26:01,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +15: [2023-05-08 11:26:01,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +19: [2023-05-08 11:26:01,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +19: [2023-05-08 11:26:01,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:26:01,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:26:01,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +15: [2023-05-08 11:26:01,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +25: [2023-05-08 11:26:01,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +25: [2023-05-08 11:26:01,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +20: [2023-05-08 11:26:01,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:26:01,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +15: [2023-05-08 11:26:01,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 1: [2023-05-08 11:26:01,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:26:01,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +31: [2023-05-08 11:26:01,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +19: [2023-05-08 11:26:01,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:26:01,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:26:01,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:26:01,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:26:01,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:26:01,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:26:01,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +11: [2023-05-08 11:26:01,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +11: [2023-05-08 11:26:01,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +11: [2023-05-08 11:26:01,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +20: [2023-05-08 11:26:01,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +11: [2023-05-08 11:26:01,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +20: [2023-05-08 11:26:01,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +20: [2023-05-08 11:26:01,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +20: [2023-05-08 11:26:01,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +31: [2023-05-08 11:26:01,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +19: [2023-05-08 11:26:01,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:26:01,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:26:01,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:26:01,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:26:01,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:26:01,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +11: [2023-05-08 11:26:01,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +20: [2023-05-08 11:26:01,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +20: [2023-05-08 11:26:01,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +20: [2023-05-08 11:26:01,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:26:01,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:26:01,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:26:01,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 0: [2023-05-08 11:26:01,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 0: [2023-05-08 11:26:01,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 0: [2023-05-08 11:26:01,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 0: [2023-05-08 11:26:01,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 6: [2023-05-08 11:26:01,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 6: [2023-05-08 11:26:01,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 6: [2023-05-08 11:26:01,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 6: [2023-05-08 11:26:01,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +20: [2023-05-08 11:26:01,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:26:01,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +11: [2023-05-08 11:26:01,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:26:01,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 1: [2023-05-08 11:26:01,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 1: [2023-05-08 11:26:01,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 1: [2023-05-08 11:26:01,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +11: [2023-05-08 11:26:01,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +11: [2023-05-08 11:26:01,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +11: [2023-05-08 11:26:01,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +20: [2023-05-08 11:26:01,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:26:01,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 6: [2023-05-08 11:26:01,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:26:01,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +14: [2023-05-08 11:26:01,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +14: [2023-05-08 11:26:01,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 4: [2023-05-08 11:26:01,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:26:01,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 6: [2023-05-08 11:26:01,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 4: [2023-05-08 11:26:01,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 4: [2023-05-08 11:26:01,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 4: [2023-05-08 11:26:01,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +19: [2023-05-08 11:26:01,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +19: [2023-05-08 11:26:01,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +19: [2023-05-08 11:26:01,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +19: [2023-05-08 11:26:01,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +11: [2023-05-08 11:26:01,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:26:01,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 1: [2023-05-08 11:26:01,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:26:01,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +18: [2023-05-08 11:26:01,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 1: [2023-05-08 11:26:01,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:26:01,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +18: [2023-05-08 11:26:01,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +21: [2023-05-08 11:26:01,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +18: [2023-05-08 11:26:01,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +21: [2023-05-08 11:26:01,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +11: [2023-05-08 11:26:01,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 1: [2023-05-08 11:26:01,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +28: [2023-05-08 11:26:01,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:26:01,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:26:01,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:26:01,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:26:01,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:26:01,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:26:01,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:26:01,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 5: [2023-05-08 11:26:01,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +28: [2023-05-08 11:26:01,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 5: [2023-05-08 11:26:01,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +14: [2023-05-08 11:26:01,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:26:01,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:26:01,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +20: [2023-05-08 11:26:01,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +20: [2023-05-08 11:26:01,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +17: [2023-05-08 11:26:01,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +17: [2023-05-08 11:26:01,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +17: [2023-05-08 11:26:01,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +17: [2023-05-08 11:26:01,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +18: [2023-05-08 11:26:01,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +21: [2023-05-08 11:26:01,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +18: [2023-05-08 11:26:01,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:26:01,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:26:01,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +19: [2023-05-08 11:26:01,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:26:01,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +19: [2023-05-08 11:26:01,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:26:01,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:26:01,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +21: [2023-05-08 11:26:01,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:26:01,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 0: [2023-05-08 11:26:01,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 1: [2023-05-08 11:26:01,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 5: [2023-05-08 11:26:01,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 0: [2023-05-08 11:26:01,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 5: [2023-05-08 11:26:01,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 5: [2023-05-08 11:26:01,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +28: [2023-05-08 11:26:01,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:26:01,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:26:01,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:26:01,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +14: [2023-05-08 11:26:01,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +14: [2023-05-08 11:26:01,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 4: [2023-05-08 11:26:01,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:26:01,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +17: [2023-05-08 11:26:01,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +21: [2023-05-08 11:26:01,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:26:01,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:26:01,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 4: [2023-05-08 11:26:01,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +19: [2023-05-08 11:26:01,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +19: [2023-05-08 11:26:01,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +18: [2023-05-08 11:26:01,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +18: [2023-05-08 11:26:01,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +14: [2023-05-08 11:26:01,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +14: [2023-05-08 11:26:01,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +11: [2023-05-08 11:26:01,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +11: [2023-05-08 11:26:01,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:26:01,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +11: [2023-05-08 11:26:01,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +11: [2023-05-08 11:26:01,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:26:01,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:26:01,665] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +11: [2023-05-08 11:26:01,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +21: [2023-05-08 11:26:01,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +18: [2023-05-08 11:26:01,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +18: [2023-05-08 11:26:01,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +18: [2023-05-08 11:26:01,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +18: [2023-05-08 11:26:01,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 1: [2023-05-08 11:26:01,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +21: [2023-05-08 11:26:01,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 2: [2023-05-08 11:26:01,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +21: [2023-05-08 11:26:01,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 2: [2023-05-08 11:26:01,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 1: [2023-05-08 11:26:01,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +19: [2023-05-08 11:26:01,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +11: [2023-05-08 11:26:01,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +19: [2023-05-08 11:26:01,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +14: [2023-05-08 11:26:01,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +14: [2023-05-08 11:26:01,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +18: [2023-05-08 11:26:01,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +14: [2023-05-08 11:26:01,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +14: [2023-05-08 11:26:01,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +18: [2023-05-08 11:26:01,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +28: [2023-05-08 11:26:01,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:26:01,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 5: [2023-05-08 11:26:01,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 5: [2023-05-08 11:26:01,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +28: [2023-05-08 11:26:01,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:26:01,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:26:01,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:26:01,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:26:01,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +17: [2023-05-08 11:26:01,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +19: [2023-05-08 11:26:01,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +19: [2023-05-08 11:26:01,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:26:01,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +19: [2023-05-08 11:26:01,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 2: [2023-05-08 11:26:01,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +17: [2023-05-08 11:26:01,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:26:01,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +17: [2023-05-08 11:26:01,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +17: [2023-05-08 11:26:01,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +14: [2023-05-08 11:26:01,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:26:01,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:26:01,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:26:01,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:26:01,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:26:01,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +14: [2023-05-08 11:26:01,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:26:01,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +18: [2023-05-08 11:26:01,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:26:01,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:26:01,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +18: [2023-05-08 11:26:01,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:26:01,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:26:01,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:26:01,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:26:01,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:26:01,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:26:01,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:26:01,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:26:01,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:26:01,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +23: [2023-05-08 11:26:01,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +23: [2023-05-08 11:26:01,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +23: [2023-05-08 11:26:01,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 4: [2023-05-08 11:26:01,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 4: [2023-05-08 11:26:01,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:26:01,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +11: [2023-05-08 11:26:01,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 4: [2023-05-08 11:26:01,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 4: [2023-05-08 11:26:01,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 2: [2023-05-08 11:26:01,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +14: [2023-05-08 11:26:01,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:26:01,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +11: [2023-05-08 11:26:01,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:26:01,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +11: [2023-05-08 11:26:01,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:26:01,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +22: [2023-05-08 11:26:01,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:26:01,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:26:01,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:26:01,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:26:01,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:26:01,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +23: [2023-05-08 11:26:01,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:26:01,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:26:01,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:26:01,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:26:01,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +22: [2023-05-08 11:26:01,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:26:01,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:26:01,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:26:01,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:26:01,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 2: [2023-05-08 11:26:01,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 2: [2023-05-08 11:26:01,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +14: [2023-05-08 11:26:01,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +18: [2023-05-08 11:26:01,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:26:01,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +18: [2023-05-08 11:26:01,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:26:01,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +18: [2023-05-08 11:26:01,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:26:01,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:26:01,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:26:01,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +27: [2023-05-08 11:26:01,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +27: [2023-05-08 11:26:01,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +27: [2023-05-08 11:26:01,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +23: [2023-05-08 11:26:01,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +27: [2023-05-08 11:26:01,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 8: [2023-05-08 11:26:01,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +16: [2023-05-08 11:26:01,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +16: [2023-05-08 11:26:01,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +16: [2023-05-08 11:26:01,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:26:01,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:26:01,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +16: [2023-05-08 11:26:01,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +16: [2023-05-08 11:26:01,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:26:01,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:26:01,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:26:01,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +18: [2023-05-08 11:26:01,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:26:01,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +29: [2023-05-08 11:26:01,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +29: [2023-05-08 11:26:01,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +29: [2023-05-08 11:26:01,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +22: [2023-05-08 11:26:01,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:26:01,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:26:01,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 9: [2023-05-08 11:26:01,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 9: [2023-05-08 11:26:01,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 9: [2023-05-08 11:26:01,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +23: [2023-05-08 11:26:01,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +23: [2023-05-08 11:26:01,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +27: [2023-05-08 11:26:01,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +27: [2023-05-08 11:26:01,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +27: [2023-05-08 11:26:01,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +27: [2023-05-08 11:26:01,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +18: [2023-05-08 11:26:01,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:26:01,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +23: [2023-05-08 11:26:01,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:26:01,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:26:01,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +24: [2023-05-08 11:26:01,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +22: [2023-05-08 11:26:01,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:26:01,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +29: [2023-05-08 11:26:01,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +16: [2023-05-08 11:26:01,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:26:01,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +23: [2023-05-08 11:26:01,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +23: [2023-05-08 11:26:01,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 9: [2023-05-08 11:26:01,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +16: [2023-05-08 11:26:01,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +24: [2023-05-08 11:26:01,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +24: [2023-05-08 11:26:01,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +22: [2023-05-08 11:26:01,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:26:01,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +27: [2023-05-08 11:26:01,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +27: [2023-05-08 11:26:01,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +30: [2023-05-08 11:26:01,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +30: [2023-05-08 11:26:01,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +30: [2023-05-08 11:26:01,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +30: [2023-05-08 11:26:01,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +29: [2023-05-08 11:26:01,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +16: [2023-05-08 11:26:01,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +26: [2023-05-08 11:26:01,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +26: [2023-05-08 11:26:01,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +26: [2023-05-08 11:26:01,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +26: [2023-05-08 11:26:01,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +24: [2023-05-08 11:26:01,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:26:01,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:26:01,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:26:01,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:26:01,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:26:01,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:26:01,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +31: [2023-05-08 11:26:01,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:26:01,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +24: [2023-05-08 11:26:01,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +24: [2023-05-08 11:26:01,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +27: [2023-05-08 11:26:01,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +27: [2023-05-08 11:26:01,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +29: [2023-05-08 11:26:01,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:26:01,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +31: [2023-05-08 11:26:01,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +29: [2023-05-08 11:26:01,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +16: [2023-05-08 11:26:01,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +16: [2023-05-08 11:26:01,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:26:01,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:26:01,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 9: [2023-05-08 11:26:01,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +16: [2023-05-08 11:26:01,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:26:01,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:26:01,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:26:01,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:26:01,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +27: [2023-05-08 11:26:01,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +27: [2023-05-08 11:26:01,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 8: [2023-05-08 11:26:01,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +29: [2023-05-08 11:26:01,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +29: [2023-05-08 11:26:01,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +27: [2023-05-08 11:26:01,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +27: [2023-05-08 11:26:01,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +26: [2023-05-08 11:26:01,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:26:01,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 7: [2023-05-08 11:26:01,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 3: [2023-05-08 11:26:01,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 7: [2023-05-08 11:26:01,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 3: [2023-05-08 11:26:01,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 3: [2023-05-08 11:26:01,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:26:01,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +28: [2023-05-08 11:26:01,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +28: [2023-05-08 11:26:01,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +28: [2023-05-08 11:26:01,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +16: [2023-05-08 11:26:01,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +16: [2023-05-08 11:26:01,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +22: [2023-05-08 11:26:01,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 3: [2023-05-08 11:26:01,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +24: [2023-05-08 11:26:01,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 3: [2023-05-08 11:26:01,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +16: [2023-05-08 11:26:01,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +22: [2023-05-08 11:26:01,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +22: [2023-05-08 11:26:01,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +24: [2023-05-08 11:26:01,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +22: [2023-05-08 11:26:01,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +13: [2023-05-08 11:26:01,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +13: [2023-05-08 11:26:01,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 3: [2023-05-08 11:26:01,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +13: [2023-05-08 11:26:01,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +16: [2023-05-08 11:26:01,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +13: [2023-05-08 11:26:01,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +16: [2023-05-08 11:26:01,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +12: [2023-05-08 11:26:01,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +12: [2023-05-08 11:26:01,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +12: [2023-05-08 11:26:01,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +10: [2023-05-08 11:26:01,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +10: [2023-05-08 11:26:01,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +10: [2023-05-08 11:26:01,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. + 3: [2023-05-08 11:26:01,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:26:01,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +10: [2023-05-08 11:26:01,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt. +31: [2023-05-08 11:26:01,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:26:01,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:26:01,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:26:01,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 8: [2023-05-08 11:26:01,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:26:01,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +15: [2023-05-08 11:26:01,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +15: [2023-05-08 11:26:01,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +15: [2023-05-08 11:26:01,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 7: [2023-05-08 11:26:01,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:26:01,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 9: [2023-05-08 11:26:01,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 7: [2023-05-08 11:26:01,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +24: [2023-05-08 11:26:01,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +24: [2023-05-08 11:26:01,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +24: [2023-05-08 11:26:01,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +24: [2023-05-08 11:26:01,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +24: [2023-05-08 11:26:01,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +24: [2023-05-08 11:26:01,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +30: [2023-05-08 11:26:01,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +29: [2023-05-08 11:26:01,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +28: [2023-05-08 11:26:01,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:26:01,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +13: [2023-05-08 11:26:01,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +28: [2023-05-08 11:26:01,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +28: [2023-05-08 11:26:01,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +13: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +28: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:26:01,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +22: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 0: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +22: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +22: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +20: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +20: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:26:01,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:26:01,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:26:01,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:26:01,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:26:01,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +26: [2023-05-08 11:26:01,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +26: [2023-05-08 11:26:01,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +16: [2023-05-08 11:26:01,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +26: [2023-05-08 11:26:01,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +13: [2023-05-08 11:26:01,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +16: [2023-05-08 11:26:01,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +20: [2023-05-08 11:26:01,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:26:01,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:26:01,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +10: [2023-05-08 11:26:01,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +10: [2023-05-08 11:26:01,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +10: [2023-05-08 11:26:01,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +10: [2023-05-08 11:26:01,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +30: [2023-05-08 11:26:01,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:26:01,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +12: [2023-05-08 11:26:01,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +12: [2023-05-08 11:26:01,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +29: [2023-05-08 11:26:01,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:26:01,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +31: [2023-05-08 11:26:01,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:26:01,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +30: [2023-05-08 11:26:01,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +29: [2023-05-08 11:26:01,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +26: [2023-05-08 11:26:01,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +26: [2023-05-08 11:26:01,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +15: [2023-05-08 11:26:01,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +12: [2023-05-08 11:26:01,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt... +15: [2023-05-08 11:26:01,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:26:01,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +15: [2023-05-08 11:26:01,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +30: [2023-05-08 11:26:01,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:26:01,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +30: [2023-05-08 11:26:01,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +13: [2023-05-08 11:26:01,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +28: [2023-05-08 11:26:01,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:26:01,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +13: [2023-05-08 11:26:01,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 7: [2023-05-08 11:26:01,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 7: [2023-05-08 11:26:01,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 7: [2023-05-08 11:26:01,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +16: [2023-05-08 11:26:01,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:26:01,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:26:01,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 8: [2023-05-08 11:26:01,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 8: [2023-05-08 11:26:01,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +16: [2023-05-08 11:26:01,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 8: [2023-05-08 11:26:01,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +10: [2023-05-08 11:26:01,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:26:01,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:26:01,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +13: [2023-05-08 11:26:01,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +13: [2023-05-08 11:26:01,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +10: [2023-05-08 11:26:01,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 7: [2023-05-08 11:26:01,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +10: [2023-05-08 11:26:01,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +10: [2023-05-08 11:26:01,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 8: [2023-05-08 11:26:01,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 6: [2023-05-08 11:26:01,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:26:01,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:26:01,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:26:01,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:26:01,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +28: [2023-05-08 11:26:01,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +22: [2023-05-08 11:26:01,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +22: [2023-05-08 11:26:01,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:26:01,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +16: [2023-05-08 11:26:01,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +16: [2023-05-08 11:26:01,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +16: [2023-05-08 11:26:01,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +16: [2023-05-08 11:26:01,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +22: [2023-05-08 11:26:01,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +22: [2023-05-08 11:26:01,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +15: [2023-05-08 11:26:01,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:26:01,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +15: [2023-05-08 11:26:01,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +15: [2023-05-08 11:26:01,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 3: [2023-05-08 11:26:01,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +10: [2023-05-08 11:26:01,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +10: [2023-05-08 11:26:01,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 3: [2023-05-08 11:26:01,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +10: [2023-05-08 11:26:01,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +10: [2023-05-08 11:26:01,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 3: [2023-05-08 11:26:01,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 3: [2023-05-08 11:26:01,778] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +28: [2023-05-08 11:26:01,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +28: [2023-05-08 11:26:01,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +22: [2023-05-08 11:26:01,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +22: [2023-05-08 11:26:01,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +22: [2023-05-08 11:26:01,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:26:01,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:26:01,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 3: [2023-05-08 11:26:01,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +22: [2023-05-08 11:26:01,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +16: [2023-05-08 11:26:01,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +16: [2023-05-08 11:26:01,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... + 6: [2023-05-08 11:26:01,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:26:01,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +12: [2023-05-08 11:26:01,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +15: [2023-05-08 11:26:01,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:26:01,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +25: [2023-05-08 11:26:01,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +15: [2023-05-08 11:26:01,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:26:01,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +20: [2023-05-08 11:26:01,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +25: [2023-05-08 11:26:01,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +12: [2023-05-08 11:26:01,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. +15: [2023-05-08 11:26:01,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +12: [2023-05-08 11:26:01,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +15: [2023-05-08 11:26:01,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 6: [2023-05-08 11:26:01,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +31: [2023-05-08 11:26:01,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +31: [2023-05-08 11:26:01,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +31: [2023-05-08 11:26:01,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +31: [2023-05-08 11:26:01,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +20: [2023-05-08 11:26:01,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:26:01,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:26:01,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:26:01,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:26:01,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt... +20: [2023-05-08 11:26:01,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:26:01,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:26:01,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:26:01,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:26:01,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +31: [2023-05-08 11:26:01,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:26:01,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +31: [2023-05-08 11:26:01,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +31: [2023-05-08 11:26:01,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +31: [2023-05-08 11:26:01,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:26:01,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:26:01,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 6: [2023-05-08 11:26:01,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 6: [2023-05-08 11:26:01,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 6: [2023-05-08 11:26:01,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 6: [2023-05-08 11:26:01,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +20: [2023-05-08 11:26:01,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +20: [2023-05-08 11:26:01,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +20: [2023-05-08 11:26:01,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +11: [2023-05-08 11:26:01,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +11: [2023-05-08 11:26:01,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +20: [2023-05-08 11:26:01,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +11: [2023-05-08 11:26:01,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +11: [2023-05-08 11:26:01,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +31: [2023-05-08 11:26:01,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:26:01,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:26:01,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +31: [2023-05-08 11:26:01,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +31: [2023-05-08 11:26:01,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:26:01,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:26:01,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +25: [2023-05-08 11:26:01,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +25: [2023-05-08 11:26:01,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +31: [2023-05-08 11:26:01,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 6: [2023-05-08 11:26:01,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:26:01,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:26:01,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:26:01,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:26:01,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +20: [2023-05-08 11:26:01,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:26:01,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:26:01,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +11: [2023-05-08 11:26:01,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 6: [2023-05-08 11:26:01,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +25: [2023-05-08 11:26:01,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +20: [2023-05-08 11:26:01,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +31: [2023-05-08 11:26:01,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +31: [2023-05-08 11:26:01,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 6: [2023-05-08 11:26:01,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +20: [2023-05-08 11:26:01,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +11: [2023-05-08 11:26:01,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +20: [2023-05-08 11:26:01,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +11: [2023-05-08 11:26:01,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 6: [2023-05-08 11:26:01,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +11: [2023-05-08 11:26:01,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:26:01,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 6: [2023-05-08 11:26:01,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +20: [2023-05-08 11:26:01,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 6: [2023-05-08 11:26:01,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +20: [2023-05-08 11:26:01,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 6: [2023-05-08 11:26:01,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +20: [2023-05-08 11:26:01,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +20: [2023-05-08 11:26:01,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:26:01,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +20: [2023-05-08 11:26:01,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 1: [2023-05-08 11:26:01,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 1: [2023-05-08 11:26:01,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 1: [2023-05-08 11:26:01,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 5: [2023-05-08 11:26:01,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +19: [2023-05-08 11:26:01,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 0: [2023-05-08 11:26:01,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +19: [2023-05-08 11:26:01,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +19: [2023-05-08 11:26:01,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 5: [2023-05-08 11:26:01,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 5: [2023-05-08 11:26:01,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 0: [2023-05-08 11:26:01,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 0: [2023-05-08 11:26:01,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 5: [2023-05-08 11:26:01,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +19: [2023-05-08 11:26:01,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 0: [2023-05-08 11:26:01,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +20: [2023-05-08 11:26:01,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 1: [2023-05-08 11:26:01,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +11: [2023-05-08 11:26:01,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +11: [2023-05-08 11:26:01,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +11: [2023-05-08 11:26:01,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 4: [2023-05-08 11:26:01,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 4: [2023-05-08 11:26:01,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 4: [2023-05-08 11:26:01,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 4: [2023-05-08 11:26:01,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +21: [2023-05-08 11:26:01,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +21: [2023-05-08 11:26:01,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +21: [2023-05-08 11:26:01,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +21: [2023-05-08 11:26:01,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 1: [2023-05-08 11:26:01,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:26:01,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:26:01,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:26:01,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:26:01,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 1: [2023-05-08 11:26:01,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 4: [2023-05-08 11:26:01,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:26:01,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:26:01,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:26:01,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:26:01,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:26:01,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +18: [2023-05-08 11:26:01,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +18: [2023-05-08 11:26:01,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +18: [2023-05-08 11:26:01,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 1: [2023-05-08 11:26:01,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +19: [2023-05-08 11:26:01,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +19: [2023-05-08 11:26:01,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 5: [2023-05-08 11:26:01,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 5: [2023-05-08 11:26:01,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 5: [2023-05-08 11:26:01,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 4: [2023-05-08 11:26:01,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 1: [2023-05-08 11:26:01,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +21: [2023-05-08 11:26:01,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 2: [2023-05-08 11:26:01,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 2: [2023-05-08 11:26:01,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 4: [2023-05-08 11:26:01,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 2: [2023-05-08 11:26:01,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +18: [2023-05-08 11:26:01,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:26:01,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +21: [2023-05-08 11:26:01,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +18: [2023-05-08 11:26:01,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 5: [2023-05-08 11:26:01,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 0: [2023-05-08 11:26:01,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 0: [2023-05-08 11:26:01,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 4: [2023-05-08 11:26:01,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 4: [2023-05-08 11:26:01,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 4: [2023-05-08 11:26:01,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 4: [2023-05-08 11:26:01,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +19: [2023-05-08 11:26:01,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +19: [2023-05-08 11:26:01,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +18: [2023-05-08 11:26:01,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +21: [2023-05-08 11:26:01,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +21: [2023-05-08 11:26:01,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 2: [2023-05-08 11:26:01,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:26:01,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +19: [2023-05-08 11:26:01,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 5: [2023-05-08 11:26:01,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 0: [2023-05-08 11:26:01,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 0: [2023-05-08 11:26:01,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 5: [2023-05-08 11:26:01,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 0: [2023-05-08 11:26:01,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 1: [2023-05-08 11:26:01,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:26:01,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +19: [2023-05-08 11:26:01,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:26:01,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 0: [2023-05-08 11:26:01,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 2: [2023-05-08 11:26:01,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +18: [2023-05-08 11:26:01,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:26:01,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +19: [2023-05-08 11:26:01,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 1: [2023-05-08 11:26:01,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +21: [2023-05-08 11:26:01,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:26:01,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:26:01,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +21: [2023-05-08 11:26:01,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +21: [2023-05-08 11:26:01,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +21: [2023-05-08 11:26:01,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 2: [2023-05-08 11:26:01,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 2: [2023-05-08 11:26:01,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +18: [2023-05-08 11:26:01,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +18: [2023-05-08 11:26:01,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 2: [2023-05-08 11:26:01,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +14: [2023-05-08 11:26:01,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +14: [2023-05-08 11:26:01,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +14: [2023-05-08 11:26:01,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +14: [2023-05-08 11:26:01,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 2: [2023-05-08 11:26:01,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 2: [2023-05-08 11:26:01,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +14: [2023-05-08 11:26:01,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:26:01,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:26:01,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:26:01,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:26:01,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +26: [2023-05-08 11:26:01,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +26: [2023-05-08 11:26:01,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +26: [2023-05-08 11:26:01,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +17: [2023-05-08 11:26:01,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +17: [2023-05-08 11:26:01,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +17: [2023-05-08 11:26:01,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +17: [2023-05-08 11:26:01,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +24: [2023-05-08 11:26:01,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +24: [2023-05-08 11:26:01,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +24: [2023-05-08 11:26:01,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +24: [2023-05-08 11:26:01,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +27: [2023-05-08 11:26:01,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +27: [2023-05-08 11:26:01,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +27: [2023-05-08 11:26:01,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +27: [2023-05-08 11:26:01,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +28: [2023-05-08 11:26:01,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +28: [2023-05-08 11:26:01,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +28: [2023-05-08 11:26:01,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +28: [2023-05-08 11:26:01,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +13: [2023-05-08 11:26:01,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +13: [2023-05-08 11:26:01,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +13: [2023-05-08 11:26:01,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +13: [2023-05-08 11:26:01,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +10: [2023-05-08 11:26:01,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +10: [2023-05-08 11:26:01,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +10: [2023-05-08 11:26:01,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +10: [2023-05-08 11:26:01,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 8: [2023-05-08 11:26:01,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 8: [2023-05-08 11:26:01,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 8: [2023-05-08 11:26:01,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 8: [2023-05-08 11:26:01,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +17: [2023-05-08 11:26:01,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:26:01,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:26:01,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +14: [2023-05-08 11:26:01,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +24: [2023-05-08 11:26:01,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:26:01,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:26:01,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +24: [2023-05-08 11:26:01,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:26:01,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +27: [2023-05-08 11:26:01,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +16: [2023-05-08 11:26:01,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +27: [2023-05-08 11:26:01,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +16: [2023-05-08 11:26:01,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +27: [2023-05-08 11:26:01,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +16: [2023-05-08 11:26:01,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +16: [2023-05-08 11:26:01,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +17: [2023-05-08 11:26:01,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:26:01,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +14: [2023-05-08 11:26:01,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:26:01,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:26:01,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +14: [2023-05-08 11:26:01,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +26: [2023-05-08 11:26:01,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +27: [2023-05-08 11:26:01,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +12: [2023-05-08 11:26:01,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +24: [2023-05-08 11:26:01,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +12: [2023-05-08 11:26:01,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +12: [2023-05-08 11:26:01,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +28: [2023-05-08 11:26:01,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +12: [2023-05-08 11:26:01,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +24: [2023-05-08 11:26:01,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +14: [2023-05-08 11:26:01,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +14: [2023-05-08 11:26:01,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +10: [2023-05-08 11:26:01,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +10: [2023-05-08 11:26:01,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +10: [2023-05-08 11:26:01,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +28: [2023-05-08 11:26:01,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +28: [2023-05-08 11:26:01,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +28: [2023-05-08 11:26:01,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +10: [2023-05-08 11:26:01,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:26:01,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 8: [2023-05-08 11:26:01,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:26:01,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +17: [2023-05-08 11:26:01,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 8: [2023-05-08 11:26:01,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:26:01,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:26:01,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +12: [2023-05-08 11:26:01,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +16: [2023-05-08 11:26:01,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 7: [2023-05-08 11:26:01,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 7: [2023-05-08 11:26:01,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +16: [2023-05-08 11:26:01,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +12: [2023-05-08 11:26:01,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +16: [2023-05-08 11:26:01,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +16: [2023-05-08 11:26:01,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +12: [2023-05-08 11:26:01,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +13: [2023-05-08 11:26:01,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +12: [2023-05-08 11:26:01,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +24: [2023-05-08 11:26:01,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +17: [2023-05-08 11:26:01,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +17: [2023-05-08 11:26:01,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +24: [2023-05-08 11:26:01,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +27: [2023-05-08 11:26:01,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +27: [2023-05-08 11:26:01,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +10: [2023-05-08 11:26:01,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:26:01,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +10: [2023-05-08 11:26:01,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:26:01,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +22: [2023-05-08 11:26:01,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +22: [2023-05-08 11:26:01,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +22: [2023-05-08 11:26:01,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 8: [2023-05-08 11:26:01,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +27: [2023-05-08 11:26:01,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +28: [2023-05-08 11:26:01,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +13: [2023-05-08 11:26:01,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:26:01,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +28: [2023-05-08 11:26:01,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +13: [2023-05-08 11:26:01,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +27: [2023-05-08 11:26:01,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +24: [2023-05-08 11:26:01,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:26:01,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +10: [2023-05-08 11:26:01,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +22: [2023-05-08 11:26:01,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. +27: [2023-05-08 11:26:01,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +27: [2023-05-08 11:26:01,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +17: [2023-05-08 11:26:01,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 7: [2023-05-08 11:26:01,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:26:01,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:26:01,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +12: [2023-05-08 11:26:01,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +12: [2023-05-08 11:26:01,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +17: [2023-05-08 11:26:01,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +16: [2023-05-08 11:26:01,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +26: [2023-05-08 11:26:01,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +26: [2023-05-08 11:26:01,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +16: [2023-05-08 11:26:01,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +17: [2023-05-08 11:26:01,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:26:01,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +17: [2023-05-08 11:26:01,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +13: [2023-05-08 11:26:01,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:26:01,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:26:01,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +13: [2023-05-08 11:26:01,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +26: [2023-05-08 11:26:01,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +22: [2023-05-08 11:26:01,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +22: [2023-05-08 11:26:01,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:26:01,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +28: [2023-05-08 11:26:01,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:26:01,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +28: [2023-05-08 11:26:01,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +22: [2023-05-08 11:26:01,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... +26: [2023-05-08 11:26:01,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +12: [2023-05-08 11:26:01,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +12: [2023-05-08 11:26:01,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +24: [2023-05-08 11:26:01,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +27: [2023-05-08 11:26:01,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +10: [2023-05-08 11:26:01,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +24: [2023-05-08 11:26:01,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 8: [2023-05-08 11:26:01,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +13: [2023-05-08 11:26:01,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +27: [2023-05-08 11:26:01,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +10: [2023-05-08 11:26:01,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +28: [2023-05-08 11:26:01,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +28: [2023-05-08 11:26:01,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 7: [2023-05-08 11:26:01,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +13: [2023-05-08 11:26:01,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 8: [2023-05-08 11:26:01,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +12: [2023-05-08 11:26:01,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +12: [2023-05-08 11:26:01,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +28: [2023-05-08 11:26:01,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +28: [2023-05-08 11:26:01,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 7: [2023-05-08 11:26:01,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +12: [2023-05-08 11:26:01,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +12: [2023-05-08 11:26:01,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +16: [2023-05-08 11:26:01,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:26:01,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 7: [2023-05-08 11:26:01,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 8: [2023-05-08 11:26:01,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 7: [2023-05-08 11:26:01,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +13: [2023-05-08 11:26:01,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:26:01,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +16: [2023-05-08 11:26:01,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 8: [2023-05-08 11:26:01,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +16: [2023-05-08 11:26:01,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +16: [2023-05-08 11:26:01,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +13: [2023-05-08 11:26:01,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +22: [2023-05-08 11:26:01,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:26:01,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +22: [2023-05-08 11:26:01,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:26:01,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:26:01,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +22: [2023-05-08 11:26:01,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 3: [2023-05-08 11:26:01,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 3: [2023-05-08 11:26:01,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 3: [2023-05-08 11:26:01,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 3: [2023-05-08 11:26:01,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt. + 7: [2023-05-08 11:26:01,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 7: [2023-05-08 11:26:01,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... +22: [2023-05-08 11:26:01,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. +22: [2023-05-08 11:26:02,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 3: [2023-05-08 11:26:02,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:26:02,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:26:02,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:26:02,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt... + 3: [2023-05-08 11:26:02,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:26:02,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 3: [2023-05-08 11:26:02,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:26:02,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 3: [2023-05-08 11:26:02,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:26:02,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 3: [2023-05-08 11:26:02,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt. + 3: [2023-05-08 11:26:02,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt... + 9: [2023-05-08 11:26:02,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 9: [2023-05-08 11:26:02,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 9: [2023-05-08 11:26:02,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 9: [2023-05-08 11:26:02,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 9: [2023-05-08 11:26:02,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 9: [2023-05-08 11:26:02,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 9: [2023-05-08 11:26:02,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 9: [2023-05-08 11:26:02,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 9: [2023-05-08 11:26:02,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 9: [2023-05-08 11:26:02,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 9: [2023-05-08 11:26:02,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 9: [2023-05-08 11:26:02,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +31: [2023-05-08 11:26:02,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +31: [2023-05-08 11:26:02,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +31: [2023-05-08 11:26:02,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +31: [2023-05-08 11:26:02,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +25: [2023-05-08 11:26:02,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +25: [2023-05-08 11:26:02,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +25: [2023-05-08 11:26:02,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +25: [2023-05-08 11:26:02,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +31: [2023-05-08 11:26:02,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:26:02,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:26:02,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +25: [2023-05-08 11:26:02,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +25: [2023-05-08 11:26:02,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:26:02,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +25: [2023-05-08 11:26:02,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +25: [2023-05-08 11:26:02,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +31: [2023-05-08 11:26:02,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:26:02,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +25: [2023-05-08 11:26:02,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:26:02,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:26:02,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +25: [2023-05-08 11:26:02,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +31: [2023-05-08 11:26:02,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:26:02,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +25: [2023-05-08 11:26:02,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +25: [2023-05-08 11:26:02,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +31: [2023-05-08 11:26:02,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +25: [2023-05-08 11:26:02,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:26:02,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +25: [2023-05-08 11:26:02,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +25: [2023-05-08 11:26:02,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +25: [2023-05-08 11:26:02,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +23: [2023-05-08 11:26:02,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +23: [2023-05-08 11:26:02,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +23: [2023-05-08 11:26:02,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +23: [2023-05-08 11:26:02,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +23: [2023-05-08 11:26:02,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +23: [2023-05-08 11:26:02,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +23: [2023-05-08 11:26:02,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +23: [2023-05-08 11:26:02,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:26:02,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +15: [2023-05-08 11:26:02,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +15: [2023-05-08 11:26:02,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +15: [2023-05-08 11:26:02,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +23: [2023-05-08 11:26:02,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:26:02,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +15: [2023-05-08 11:26:02,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:26:02,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:26:02,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:26:02,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +19: [2023-05-08 11:26:02,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +11: [2023-05-08 11:26:02,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +11: [2023-05-08 11:26:02,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +11: [2023-05-08 11:26:02,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +19: [2023-05-08 11:26:02,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +19: [2023-05-08 11:26:02,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +19: [2023-05-08 11:26:02,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +11: [2023-05-08 11:26:02,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +23: [2023-05-08 11:26:02,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:26:02,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:26:02,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +23: [2023-05-08 11:26:02,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +23: [2023-05-08 11:26:02,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +23: [2023-05-08 11:26:02,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 1: [2023-05-08 11:26:02,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 1: [2023-05-08 11:26:02,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 1: [2023-05-08 11:26:02,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 1: [2023-05-08 11:26:02,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +11: [2023-05-08 11:26:02,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:26:02,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +19: [2023-05-08 11:26:02,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +19: [2023-05-08 11:26:02,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:26:02,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +19: [2023-05-08 11:26:02,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:26:02,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +15: [2023-05-08 11:26:02,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +15: [2023-05-08 11:26:02,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +11: [2023-05-08 11:26:02,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:26:02,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:26:02,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +15: [2023-05-08 11:26:02,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +19: [2023-05-08 11:26:02,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:26:02,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 6: [2023-05-08 11:26:02,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 6: [2023-05-08 11:26:02,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 6: [2023-05-08 11:26:02,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +15: [2023-05-08 11:26:02,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +15: [2023-05-08 11:26:02,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 1: [2023-05-08 11:26:02,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +19: [2023-05-08 11:26:02,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +19: [2023-05-08 11:26:02,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +11: [2023-05-08 11:26:02,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +11: [2023-05-08 11:26:02,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +11: [2023-05-08 11:26:02,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +11: [2023-05-08 11:26:02,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +11: [2023-05-08 11:26:02,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +11: [2023-05-08 11:26:02,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 6: [2023-05-08 11:26:02,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:26:02,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:26:02,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 6: [2023-05-08 11:26:02,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +19: [2023-05-08 11:26:02,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +19: [2023-05-08 11:26:02,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +19: [2023-05-08 11:26:02,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:26:02,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:26:02,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:26:02,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:26:02,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +23: [2023-05-08 11:26:02,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:26:02,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:26:02,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:26:02,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:26:02,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:26:02,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:26:02,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +17: [2023-05-08 11:26:02,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:26:02,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +17: [2023-05-08 11:26:02,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +17: [2023-05-08 11:26:02,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:26:02,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:26:02,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:26:02,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:26:02,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +23: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 5: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +26: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +26: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +23: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +26: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +15: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 9: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 9: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +25: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +25: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +15: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +15: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +29: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +29: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +15: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +25: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +29: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +29: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +29: [2023-05-08 11:26:02,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 5: [2023-05-08 11:26:02,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +29: [2023-05-08 11:26:02,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +23: [2023-05-08 11:26:02,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:26:02,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 5: [2023-05-08 11:26:02,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:26:02,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:26:02,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +19: [2023-05-08 11:26:02,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 2: [2023-05-08 11:26:02,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 9: [2023-05-08 11:26:02,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:26:02,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:26:02,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:26:02,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:26:02,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:26:02,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +29: [2023-05-08 11:26:02,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:26:02,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:26:02,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:26:02,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 6: [2023-05-08 11:26:02,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 6: [2023-05-08 11:26:02,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 6: [2023-05-08 11:26:02,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 6: [2023-05-08 11:26:02,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 1: [2023-05-08 11:26:02,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 1: [2023-05-08 11:26:02,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 1: [2023-05-08 11:26:02,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +19: [2023-05-08 11:26:02,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:26:02,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:26:02,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:26:02,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 6: [2023-05-08 11:26:02,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +17: [2023-05-08 11:26:02,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:26:02,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:26:02,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 6: [2023-05-08 11:26:02,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +29: [2023-05-08 11:26:02,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:26:02,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:26:02,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:26:02,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +15: [2023-05-08 11:26:02,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +26: [2023-05-08 11:26:02,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:26:02,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:26:02,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:26:02,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:26:02,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:26:02,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:26:02,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +25: [2023-05-08 11:26:02,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +25: [2023-05-08 11:26:02,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:26:02,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:26:02,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 1: [2023-05-08 11:26:02,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:26:02,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +15: [2023-05-08 11:26:02,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +15: [2023-05-08 11:26:02,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:26:02,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:26:02,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:26:02,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:26:02,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +15: [2023-05-08 11:26:02,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +11: [2023-05-08 11:26:02,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:26:02,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:26:02,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +17: [2023-05-08 11:26:02,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +29: [2023-05-08 11:26:02,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +17: [2023-05-08 11:26:02,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +17: [2023-05-08 11:26:02,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:26:02,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +11: [2023-05-08 11:26:02,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +25: [2023-05-08 11:26:02,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:26:02,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +29: [2023-05-08 11:26:02,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:26:02,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:26:02,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:26:02,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +25: [2023-05-08 11:26:02,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:26:02,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:26:02,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +25: [2023-05-08 11:26:02,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:26:02,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +26: [2023-05-08 11:26:02,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 9: [2023-05-08 11:26:02,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 9: [2023-05-08 11:26:02,290] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +15: [2023-05-08 11:26:02,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:26:02,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:26:02,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +15: [2023-05-08 11:26:02,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:26:02,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +24: [2023-05-08 11:26:02,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:26:02,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +30: [2023-05-08 11:26:02,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +30: [2023-05-08 11:26:02,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +23: [2023-05-08 11:26:02,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +23: [2023-05-08 11:26:02,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +20: [2023-05-08 11:26:02,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +20: [2023-05-08 11:26:02,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +20: [2023-05-08 11:26:02,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 1: [2023-05-08 11:26:02,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:26:02,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +11: [2023-05-08 11:26:02,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +29: [2023-05-08 11:26:02,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +29: [2023-05-08 11:26:02,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:26:02,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:26:02,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 9: [2023-05-08 11:26:02,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +15: [2023-05-08 11:26:02,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +25: [2023-05-08 11:26:02,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:26:02,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:26:02,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:26:02,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +25: [2023-05-08 11:26:02,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +26: [2023-05-08 11:26:02,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:26:02,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:26:02,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:26:02,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +11: [2023-05-08 11:26:02,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:26:02,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +29: [2023-05-08 11:26:02,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +29: [2023-05-08 11:26:02,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:26:02,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 9: [2023-05-08 11:26:02,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:26:02,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +20: [2023-05-08 11:26:02,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +23: [2023-05-08 11:26:02,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:26:02,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:26:02,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +15: [2023-05-08 11:26:02,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:26:02,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 7: [2023-05-08 11:26:02,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 7: [2023-05-08 11:26:02,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 7: [2023-05-08 11:26:02,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +25: [2023-05-08 11:26:02,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:26:02,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:26:02,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:26:02,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +20: [2023-05-08 11:26:02,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +30: [2023-05-08 11:26:02,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:26:02,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 8: [2023-05-08 11:26:02,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +20: [2023-05-08 11:26:02,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 7: [2023-05-08 11:26:02,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 8: [2023-05-08 11:26:02,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 7: [2023-05-08 11:26:02,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 8: [2023-05-08 11:26:02,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 8: [2023-05-08 11:26:02,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +30: [2023-05-08 11:26:02,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:26:02,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +29: [2023-05-08 11:26:02,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +20: [2023-05-08 11:26:02,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:26:02,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +20: [2023-05-08 11:26:02,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +29: [2023-05-08 11:26:02,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +29: [2023-05-08 11:26:02,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +20: [2023-05-08 11:26:02,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +20: [2023-05-08 11:26:02,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +30: [2023-05-08 11:26:02,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +30: [2023-05-08 11:26:02,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 8: [2023-05-08 11:26:02,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 8: [2023-05-08 11:26:02,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:26:02,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 7: [2023-05-08 11:26:02,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 8: [2023-05-08 11:26:02,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +29: [2023-05-08 11:26:02,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +20: [2023-05-08 11:26:02,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:26:02,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +29: [2023-05-08 11:26:02,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +20: [2023-05-08 11:26:02,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +29: [2023-05-08 11:26:02,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 7: [2023-05-08 11:26:02,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 8: [2023-05-08 11:26:02,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:26:02,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +29: [2023-05-08 11:26:02,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:26:02,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +29: [2023-05-08 11:26:02,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:26:02,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 8: [2023-05-08 11:26:02,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:26:02,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:26:02,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 8: [2023-05-08 11:26:02,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 8: [2023-05-08 11:26:02,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 8: [2023-05-08 11:26:02,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +29: [2023-05-08 11:26:02,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:26:02,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +29: [2023-05-08 11:26:02,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +29: [2023-05-08 11:26:02,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 3: [2023-05-08 11:26:02,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 3: [2023-05-08 11:26:02,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +10: [2023-05-08 11:26:02,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +10: [2023-05-08 11:26:02,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 5: [2023-05-08 11:26:02,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 5: [2023-05-08 11:26:02,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 5: [2023-05-08 11:26:02,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 5: [2023-05-08 11:26:02,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 3: [2023-05-08 11:26:02,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +12: [2023-05-08 11:26:02,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +17: [2023-05-08 11:26:02,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +17: [2023-05-08 11:26:02,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 3: [2023-05-08 11:26:02,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +12: [2023-05-08 11:26:02,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +12: [2023-05-08 11:26:02,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +16: [2023-05-08 11:26:02,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +26: [2023-05-08 11:26:02,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +26: [2023-05-08 11:26:02,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +10: [2023-05-08 11:26:02,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +26: [2023-05-08 11:26:02,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +27: [2023-05-08 11:26:02,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +12: [2023-05-08 11:26:02,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +26: [2023-05-08 11:26:02,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +27: [2023-05-08 11:26:02,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +10: [2023-05-08 11:26:02,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +14: [2023-05-08 11:26:02,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +14: [2023-05-08 11:26:02,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +17: [2023-05-08 11:26:02,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +28: [2023-05-08 11:26:02,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +14: [2023-05-08 11:26:02,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +21: [2023-05-08 11:26:02,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +17: [2023-05-08 11:26:02,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +14: [2023-05-08 11:26:02,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +28: [2023-05-08 11:26:02,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +16: [2023-05-08 11:26:02,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 0: [2023-05-08 11:26:02,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 0: [2023-05-08 11:26:02,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 0: [2023-05-08 11:26:02,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 4: [2023-05-08 11:26:02,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 4: [2023-05-08 11:26:02,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 4: [2023-05-08 11:26:02,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +27: [2023-05-08 11:26:02,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +21: [2023-05-08 11:26:02,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 4: [2023-05-08 11:26:02,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +27: [2023-05-08 11:26:02,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +13: [2023-05-08 11:26:02,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +13: [2023-05-08 11:26:02,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +16: [2023-05-08 11:26:02,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +16: [2023-05-08 11:26:02,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +28: [2023-05-08 11:26:02,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +28: [2023-05-08 11:26:02,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 0: [2023-05-08 11:26:02,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 3: [2023-05-08 11:26:02,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:26:02,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 5: [2023-05-08 11:26:02,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 5: [2023-05-08 11:26:02,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +21: [2023-05-08 11:26:02,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +13: [2023-05-08 11:26:02,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +21: [2023-05-08 11:26:02,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 5: [2023-05-08 11:26:02,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +13: [2023-05-08 11:26:02,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +10: [2023-05-08 11:26:02,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:26:02,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +10: [2023-05-08 11:26:02,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:26:02,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:26:02,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +10: [2023-05-08 11:26:02,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:26:02,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:26:02,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:26:02,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:26:02,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:26:02,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:26:02,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +21: [2023-05-08 11:26:02,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:26:02,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 3: [2023-05-08 11:26:02,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +12: [2023-05-08 11:26:02,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +10: [2023-05-08 11:26:02,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:26:02,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 5: [2023-05-08 11:26:02,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:26:02,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:26:02,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:26:02,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:26:02,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:26:02,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +26: [2023-05-08 11:26:02,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:26:02,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:26:02,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:26:02,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:26:02,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:26:02,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:26:02,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:26:02,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 0: [2023-05-08 11:26:02,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:26:02,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 0: [2023-05-08 11:26:02,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:26:02,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 0: [2023-05-08 11:26:02,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:26:02,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:26:02,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +27: [2023-05-08 11:26:02,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:26:02,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:26:02,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +16: [2023-05-08 11:26:02,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +13: [2023-05-08 11:26:02,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 0: [2023-05-08 11:26:02,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +21: [2023-05-08 11:26:02,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +13: [2023-05-08 11:26:02,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +10: [2023-05-08 11:26:02,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:26:02,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +17: [2023-05-08 11:26:02,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +10: [2023-05-08 11:26:02,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +17: [2023-05-08 11:26:02,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +10: [2023-05-08 11:26:02,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +10: [2023-05-08 11:26:02,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +13: [2023-05-08 11:26:02,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +21: [2023-05-08 11:26:02,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 4: [2023-05-08 11:26:02,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +27: [2023-05-08 11:26:02,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +27: [2023-05-08 11:26:02,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 4: [2023-05-08 11:26:02,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 5: [2023-05-08 11:26:02,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +16: [2023-05-08 11:26:02,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 2: [2023-05-08 11:26:02,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 5: [2023-05-08 11:26:02,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +13: [2023-05-08 11:26:02,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +10: [2023-05-08 11:26:02,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:26:02,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +10: [2023-05-08 11:26:02,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +16: [2023-05-08 11:26:02,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +17: [2023-05-08 11:26:02,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:26:02,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +28: [2023-05-08 11:26:02,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 2: [2023-05-08 11:26:02,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 2: [2023-05-08 11:26:02,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +26: [2023-05-08 11:26:02,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +26: [2023-05-08 11:26:02,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:26:02,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +26: [2023-05-08 11:26:02,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +26: [2023-05-08 11:26:02,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 3: [2023-05-08 11:26:02,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:26:02,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +10: [2023-05-08 11:26:02,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +10: [2023-05-08 11:26:02,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +21: [2023-05-08 11:26:02,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 3: [2023-05-08 11:26:02,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +14: [2023-05-08 11:26:02,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +14: [2023-05-08 11:26:02,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 5: [2023-05-08 11:26:02,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +17: [2023-05-08 11:26:02,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +17: [2023-05-08 11:26:02,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +21: [2023-05-08 11:26:02,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 3: [2023-05-08 11:26:02,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 3: [2023-05-08 11:26:02,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +12: [2023-05-08 11:26:02,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:26:02,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +12: [2023-05-08 11:26:02,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:26:02,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:26:02,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +12: [2023-05-08 11:26:02,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +12: [2023-05-08 11:26:02,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +12: [2023-05-08 11:26:02,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +26: [2023-05-08 11:26:02,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +26: [2023-05-08 11:26:02,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +26: [2023-05-08 11:26:02,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +26: [2023-05-08 11:26:02,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 4: [2023-05-08 11:26:02,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 4: [2023-05-08 11:26:02,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +17: [2023-05-08 11:26:02,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +17: [2023-05-08 11:26:02,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 0: [2023-05-08 11:26:02,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +16: [2023-05-08 11:26:02,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +16: [2023-05-08 11:26:02,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +16: [2023-05-08 11:26:02,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +16: [2023-05-08 11:26:02,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +12: [2023-05-08 11:26:02,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:26:02,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 3: [2023-05-08 11:26:02,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 3: [2023-05-08 11:26:02,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 3: [2023-05-08 11:26:02,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 3: [2023-05-08 11:26:02,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +13: [2023-05-08 11:26:02,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:26:02,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +14: [2023-05-08 11:26:02,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +28: [2023-05-08 11:26:02,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +28: [2023-05-08 11:26:02,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +27: [2023-05-08 11:26:02,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +27: [2023-05-08 11:26:02,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +27: [2023-05-08 11:26:02,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:26:02,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +27: [2023-05-08 11:26:02,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +28: [2023-05-08 11:26:02,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +13: [2023-05-08 11:26:02,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +12: [2023-05-08 11:26:02,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +13: [2023-05-08 11:26:02,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:26:02,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +14: [2023-05-08 11:26:02,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +13: [2023-05-08 11:26:02,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:26:02,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 2: [2023-05-08 11:26:02,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +14: [2023-05-08 11:26:02,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 0: [2023-05-08 11:26:02,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:26:02,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 2: [2023-05-08 11:26:02,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +21: [2023-05-08 11:26:02,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:26:02,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +21: [2023-05-08 11:26:02,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +21: [2023-05-08 11:26:02,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +16: [2023-05-08 11:26:02,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:26:02,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:26:02,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 5: [2023-05-08 11:26:02,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +16: [2023-05-08 11:26:02,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +28: [2023-05-08 11:26:02,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +28: [2023-05-08 11:26:02,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +27: [2023-05-08 11:26:02,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +27: [2023-05-08 11:26:02,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 5: [2023-05-08 11:26:02,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +13: [2023-05-08 11:26:02,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +13: [2023-05-08 11:26:02,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 4: [2023-05-08 11:26:02,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 0: [2023-05-08 11:26:02,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 0: [2023-05-08 11:26:02,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 2: [2023-05-08 11:26:02,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 2: [2023-05-08 11:26:02,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +25: [2023-05-08 11:26:02,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +25: [2023-05-08 11:26:02,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +25: [2023-05-08 11:26:02,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +25: [2023-05-08 11:26:02,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +24: [2023-05-08 11:26:02,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 2: [2023-05-08 11:26:02,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:26:02,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +18: [2023-05-08 11:26:02,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 2: [2023-05-08 11:26:02,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +18: [2023-05-08 11:26:02,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +24: [2023-05-08 11:26:02,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +24: [2023-05-08 11:26:02,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +18: [2023-05-08 11:26:02,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +24: [2023-05-08 11:26:02,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. + 2: [2023-05-08 11:26:02,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +25: [2023-05-08 11:26:02,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:26:02,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:26:02,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:26:02,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:26:02,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +18: [2023-05-08 11:26:02,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +24: [2023-05-08 11:26:02,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +24: [2023-05-08 11:26:02,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +18: [2023-05-08 11:26:02,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +18: [2023-05-08 11:26:02,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +24: [2023-05-08 11:26:02,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +18: [2023-05-08 11:26:02,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:26:02,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +22: [2023-05-08 11:26:02,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +22: [2023-05-08 11:26:02,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +22: [2023-05-08 11:26:02,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt. +24: [2023-05-08 11:26:02,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +24: [2023-05-08 11:26:02,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +25: [2023-05-08 11:26:02,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +25: [2023-05-08 11:26:02,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +25: [2023-05-08 11:26:02,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +25: [2023-05-08 11:26:02,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +25: [2023-05-08 11:26:02,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +25: [2023-05-08 11:26:02,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +25: [2023-05-08 11:26:02,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +25: [2023-05-08 11:26:02,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +18: [2023-05-08 11:26:02,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:26:02,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:26:02,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:26:02,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +18: [2023-05-08 11:26:02,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +18: [2023-05-08 11:26:02,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +22: [2023-05-08 11:26:02,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +18: [2023-05-08 11:26:02,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +18: [2023-05-08 11:26:02,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +24: [2023-05-08 11:26:02,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +24: [2023-05-08 11:26:02,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +24: [2023-05-08 11:26:02,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +24: [2023-05-08 11:26:02,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +22: [2023-05-08 11:26:02,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:26:02,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... +22: [2023-05-08 11:26:02,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 1: [2023-05-08 11:26:02,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 1: [2023-05-08 11:26:02,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 1: [2023-05-08 11:26:02,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +24: [2023-05-08 11:26:02,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +24: [2023-05-08 11:26:02,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 1: [2023-05-08 11:26:02,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:26:02,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:26:02,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +31: [2023-05-08 11:26:02,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +31: [2023-05-08 11:26:02,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +31: [2023-05-08 11:26:02,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +31: [2023-05-08 11:26:02,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +22: [2023-05-08 11:26:02,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +22: [2023-05-08 11:26:02,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... + 1: [2023-05-08 11:26:02,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +22: [2023-05-08 11:26:02,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +22: [2023-05-08 11:26:02,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt. +31: [2023-05-08 11:26:02,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:26:02,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +23: [2023-05-08 11:26:02,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +23: [2023-05-08 11:26:02,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +23: [2023-05-08 11:26:02,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +22: [2023-05-08 11:26:02,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +22: [2023-05-08 11:26:02,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt... +31: [2023-05-08 11:26:02,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +31: [2023-05-08 11:26:02,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +31: [2023-05-08 11:26:02,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 1: [2023-05-08 11:26:02,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 1: [2023-05-08 11:26:02,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +23: [2023-05-08 11:26:02,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:26:02,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:26:02,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:26:02,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 6: [2023-05-08 11:26:02,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 6: [2023-05-08 11:26:02,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 6: [2023-05-08 11:26:02,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 6: [2023-05-08 11:26:02,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +23: [2023-05-08 11:26:02,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +23: [2023-05-08 11:26:02,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 6: [2023-05-08 11:26:02,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:26:02,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:26:02,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 6: [2023-05-08 11:26:02,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 6: [2023-05-08 11:26:02,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:26:02,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +20: [2023-05-08 11:26:02,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +20: [2023-05-08 11:26:02,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +20: [2023-05-08 11:26:02,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +15: [2023-05-08 11:26:02,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +15: [2023-05-08 11:26:02,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +15: [2023-05-08 11:26:02,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +15: [2023-05-08 11:26:02,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +20: [2023-05-08 11:26:02,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +11: [2023-05-08 11:26:02,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +11: [2023-05-08 11:26:02,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +11: [2023-05-08 11:26:02,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +23: [2023-05-08 11:26:02,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +23: [2023-05-08 11:26:02,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 9: [2023-05-08 11:26:02,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +30: [2023-05-08 11:26:02,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +30: [2023-05-08 11:26:02,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +30: [2023-05-08 11:26:02,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 9: [2023-05-08 11:26:02,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +11: [2023-05-08 11:26:02,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 9: [2023-05-08 11:26:02,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +23: [2023-05-08 11:26:02,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +23: [2023-05-08 11:26:02,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +19: [2023-05-08 11:26:02,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +19: [2023-05-08 11:26:02,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +19: [2023-05-08 11:26:02,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +30: [2023-05-08 11:26:02,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 8: [2023-05-08 11:26:02,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 8: [2023-05-08 11:26:02,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 8: [2023-05-08 11:26:02,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 8: [2023-05-08 11:26:02,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +29: [2023-05-08 11:26:02,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +19: [2023-05-08 11:26:02,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +20: [2023-05-08 11:26:02,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:26:02,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:26:02,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +29: [2023-05-08 11:26:02,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +29: [2023-05-08 11:26:02,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +29: [2023-05-08 11:26:02,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 7: [2023-05-08 11:26:02,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 7: [2023-05-08 11:26:02,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 7: [2023-05-08 11:26:02,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 7: [2023-05-08 11:26:02,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +15: [2023-05-08 11:26:02,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:26:02,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:26:02,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:26:02,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:26:02,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:26:02,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:26:02,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:26:02,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:26:02,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 9: [2023-05-08 11:26:02,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 9: [2023-05-08 11:26:02,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:26:02,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:26:02,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:26:02,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +10: [2023-05-08 11:26:02,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +10: [2023-05-08 11:26:02,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +10: [2023-05-08 11:26:02,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 9: [2023-05-08 11:26:02,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 9: [2023-05-08 11:26:02,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +10: [2023-05-08 11:26:02,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +19: [2023-05-08 11:26:02,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:26:02,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:26:02,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:26:02,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:26:02,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +29: [2023-05-08 11:26:02,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:26:02,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:26:02,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:26:02,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:26:02,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +29: [2023-05-08 11:26:02,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +29: [2023-05-08 11:26:02,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:26:02,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +29: [2023-05-08 11:26:02,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:26:02,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 7: [2023-05-08 11:26:02,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:26:02,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +15: [2023-05-08 11:26:02,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +11: [2023-05-08 11:26:02,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +10: [2023-05-08 11:26:02,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +10: [2023-05-08 11:26:02,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:26:02,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +15: [2023-05-08 11:26:02,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +15: [2023-05-08 11:26:02,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +10: [2023-05-08 11:26:02,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +10: [2023-05-08 11:26:02,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:26:02,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:26:02,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 9: [2023-05-08 11:26:02,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 9: [2023-05-08 11:26:02,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +29: [2023-05-08 11:26:02,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 9: [2023-05-08 11:26:02,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +15: [2023-05-08 11:26:02,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +15: [2023-05-08 11:26:02,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +29: [2023-05-08 11:26:02,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 7: [2023-05-08 11:26:02,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 9: [2023-05-08 11:26:02,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +19: [2023-05-08 11:26:02,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +19: [2023-05-08 11:26:02,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +19: [2023-05-08 11:26:02,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:26:02,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 7: [2023-05-08 11:26:02,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +29: [2023-05-08 11:26:02,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +29: [2023-05-08 11:26:02,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +19: [2023-05-08 11:26:02,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:26:02,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +19: [2023-05-08 11:26:02,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:26:02,575] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +11: [2023-05-08 11:26:02,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:26:02,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:26:02,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +29: [2023-05-08 11:26:02,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:26:02,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +29: [2023-05-08 11:26:02,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +29: [2023-05-08 11:26:02,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +29: [2023-05-08 11:26:02,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 7: [2023-05-08 11:26:02,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +17: [2023-05-08 11:26:02,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:26:02,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +17: [2023-05-08 11:26:02,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:26:02,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +17: [2023-05-08 11:26:02,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:26:02,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:26:02,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +17: [2023-05-08 11:26:02,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +17: [2023-05-08 11:26:02,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:26:02,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:26:02,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +16: [2023-05-08 11:26:02,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +16: [2023-05-08 11:26:02,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +16: [2023-05-08 11:26:02,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +16: [2023-05-08 11:26:02,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +17: [2023-05-08 11:26:02,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:26:02,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:26:02,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:26:02,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 4: [2023-05-08 11:26:02,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 4: [2023-05-08 11:26:02,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 4: [2023-05-08 11:26:02,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +21: [2023-05-08 11:26:02,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +21: [2023-05-08 11:26:02,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +21: [2023-05-08 11:26:02,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +21: [2023-05-08 11:26:02,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +17: [2023-05-08 11:26:02,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +18: [2023-05-08 11:26:02,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +18: [2023-05-08 11:26:02,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +28: [2023-05-08 11:26:02,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +18: [2023-05-08 11:26:02,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +28: [2023-05-08 11:26:02,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +28: [2023-05-08 11:26:02,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +18: [2023-05-08 11:26:02,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +28: [2023-05-08 11:26:02,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +13: [2023-05-08 11:26:02,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +13: [2023-05-08 11:26:02,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +13: [2023-05-08 11:26:02,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +17: [2023-05-08 11:26:02,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +26: [2023-05-08 11:26:02,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +16: [2023-05-08 11:26:02,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:26:02,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +26: [2023-05-08 11:26:02,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 5: [2023-05-08 11:26:02,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 5: [2023-05-08 11:26:02,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +17: [2023-05-08 11:26:02,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +17: [2023-05-08 11:26:02,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 5: [2023-05-08 11:26:02,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 5: [2023-05-08 11:26:02,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +17: [2023-05-08 11:26:02,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +14: [2023-05-08 11:26:02,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +14: [2023-05-08 11:26:02,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +16: [2023-05-08 11:26:02,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:26:02,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +12: [2023-05-08 11:26:02,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +12: [2023-05-08 11:26:02,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +12: [2023-05-08 11:26:02,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +14: [2023-05-08 11:26:02,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 3: [2023-05-08 11:26:02,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +14: [2023-05-08 11:26:02,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 3: [2023-05-08 11:26:02,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 3: [2023-05-08 11:26:02,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +12: [2023-05-08 11:26:02,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +16: [2023-05-08 11:26:02,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:26:02,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +13: [2023-05-08 11:26:02,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +16: [2023-05-08 11:26:02,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:26:02,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 4: [2023-05-08 11:26:02,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:26:02,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +24: [2023-05-08 11:26:02,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 4: [2023-05-08 11:26:02,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:26:02,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:26:02,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +27: [2023-05-08 11:26:02,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 4: [2023-05-08 11:26:02,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +27: [2023-05-08 11:26:02,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +27: [2023-05-08 11:26:02,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 0: [2023-05-08 11:26:02,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +27: [2023-05-08 11:26:02,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 0: [2023-05-08 11:26:02,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 0: [2023-05-08 11:26:02,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 0: [2023-05-08 11:26:02,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +22: [2023-05-08 11:26:02,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +22: [2023-05-08 11:26:02,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +22: [2023-05-08 11:26:02,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +22: [2023-05-08 11:26:02,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. +18: [2023-05-08 11:26:02,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:26:02,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +18: [2023-05-08 11:26:02,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +17: [2023-05-08 11:26:02,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:26:02,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +18: [2023-05-08 11:26:02,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +18: [2023-05-08 11:26:02,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:26:02,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:26:02,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:26:02,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +17: [2023-05-08 11:26:02,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:26:02,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +17: [2023-05-08 11:26:02,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +13: [2023-05-08 11:26:02,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:26:02,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:26:02,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:26:02,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +17: [2023-05-08 11:26:02,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:26:02,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +17: [2023-05-08 11:26:02,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:26:02,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:26:02,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 5: [2023-05-08 11:26:02,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 5: [2023-05-08 11:26:02,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:26:02,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 5: [2023-05-08 11:26:02,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 5: [2023-05-08 11:26:02,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:26:02,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:26:02,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:26:02,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:26:02,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +27: [2023-05-08 11:26:02,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:26:02,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:26:02,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:26:02,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:26:02,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +27: [2023-05-08 11:26:02,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +13: [2023-05-08 11:26:02,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:26:02,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:26:02,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:26:02,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +27: [2023-05-08 11:26:02,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +13: [2023-05-08 11:26:02,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:26:02,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +13: [2023-05-08 11:26:02,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +27: [2023-05-08 11:26:02,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:26:02,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 0: [2023-05-08 11:26:02,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 0: [2023-05-08 11:26:02,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 0: [2023-05-08 11:26:02,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:26:02,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 0: [2023-05-08 11:26:02,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:26:02,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:26:02,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:26:02,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +24: [2023-05-08 11:26:02,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:26:02,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +17: [2023-05-08 11:26:02,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:26:02,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +17: [2023-05-08 11:26:02,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:26:02,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +26: [2023-05-08 11:26:02,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +26: [2023-05-08 11:26:02,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +26: [2023-05-08 11:26:02,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +26: [2023-05-08 11:26:02,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +17: [2023-05-08 11:26:02,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:26:02,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +17: [2023-05-08 11:26:02,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +17: [2023-05-08 11:26:02,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 5: [2023-05-08 11:26:02,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +25: [2023-05-08 11:26:02,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +25: [2023-05-08 11:26:02,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 5: [2023-05-08 11:26:02,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +25: [2023-05-08 11:26:02,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +25: [2023-05-08 11:26:02,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +25: [2023-05-08 11:26:02,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +25: [2023-05-08 11:26:02,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:26:02,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +25: [2023-05-08 11:26:02,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +25: [2023-05-08 11:26:02,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +29: [2023-05-08 11:26:02,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:26:02,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:26:02,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:26:02,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +29: [2023-05-08 11:26:02,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +29: [2023-05-08 11:26:02,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:26:02,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:26:02,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +29: [2023-05-08 11:26:02,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:26:02,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:26:02,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:26:02,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:26:02,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:26:02,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +26: [2023-05-08 11:26:02,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:26:02,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:26:02,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:26:02,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:26:02,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:26:02,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:26:02,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +26: [2023-05-08 11:26:02,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:26:02,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:26:02,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +23: [2023-05-08 11:26:02,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:26:02,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:26:02,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:26:02,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:26:02,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +24: [2023-05-08 11:26:02,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +26: [2023-05-08 11:26:02,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 2: [2023-05-08 11:26:02,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 2: [2023-05-08 11:26:02,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 2: [2023-05-08 11:26:02,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 2: [2023-05-08 11:26:02,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt. + 2: [2023-05-08 11:26:02,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +25: [2023-05-08 11:26:02,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +25: [2023-05-08 11:26:02,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +25: [2023-05-08 11:26:02,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:26:02,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:26:02,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +25: [2023-05-08 11:26:02,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +15: [2023-05-08 11:26:02,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +15: [2023-05-08 11:26:02,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +15: [2023-05-08 11:26:02,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +15: [2023-05-08 11:26:02,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +15: [2023-05-08 11:26:02,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +15: [2023-05-08 11:26:02,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:26:02,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +15: [2023-05-08 11:26:02,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +15: [2023-05-08 11:26:02,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:26:02,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +25: [2023-05-08 11:26:02,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +25: [2023-05-08 11:26:02,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +23: [2023-05-08 11:26:02,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:26:02,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:26:02,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:26:02,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +25: [2023-05-08 11:26:02,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:26:02,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:26:02,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:26:02,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 5: [2023-05-08 11:26:02,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +26: [2023-05-08 11:26:02,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +26: [2023-05-08 11:26:02,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:26:02,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +26: [2023-05-08 11:26:02,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +26: [2023-05-08 11:26:02,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:26:02,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +26: [2023-05-08 11:26:02,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +25: [2023-05-08 11:26:02,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +26: [2023-05-08 11:26:02,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:26:02,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 1: [2023-05-08 11:26:02,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 5: [2023-05-08 11:26:02,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:26:02,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:26:02,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +19: [2023-05-08 11:26:02,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +19: [2023-05-08 11:26:02,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:26:02,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +19: [2023-05-08 11:26:02,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:26:02,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +19: [2023-05-08 11:26:02,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +19: [2023-05-08 11:26:02,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:26:02,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:26:02,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +19: [2023-05-08 11:26:02,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:26:02,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:26:02,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +15: [2023-05-08 11:26:02,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +15: [2023-05-08 11:26:02,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +15: [2023-05-08 11:26:02,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:26:02,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +11: [2023-05-08 11:26:02,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +11: [2023-05-08 11:26:02,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +11: [2023-05-08 11:26:02,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +11: [2023-05-08 11:26:02,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:26:02,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +24: [2023-05-08 11:26:02,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:26:02,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +24: [2023-05-08 11:26:02,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +24: [2023-05-08 11:26:02,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:26:02,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:26:02,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +26: [2023-05-08 11:26:02,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 9: [2023-05-08 11:26:02,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 9: [2023-05-08 11:26:02,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:26:02,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +11: [2023-05-08 11:26:02,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +19: [2023-05-08 11:26:02,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +11: [2023-05-08 11:26:02,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +11: [2023-05-08 11:26:02,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +11: [2023-05-08 11:26:02,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:26:02,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +26: [2023-05-08 11:26:02,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +24: [2023-05-08 11:26:02,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +19: [2023-05-08 11:26:02,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +15: [2023-05-08 11:26:02,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:26:02,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +31: [2023-05-08 11:26:02,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:26:02,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:26:02,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:26:02,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +31: [2023-05-08 11:26:02,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:26:02,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +31: [2023-05-08 11:26:02,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:26:02,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:26:02,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:26:02,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:26:02,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:26:02,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +15: [2023-05-08 11:26:02,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +31: [2023-05-08 11:26:02,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:26:02,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:26:02,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:26:02,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:26:02,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +31: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +31: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +21: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +20: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +13: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +13: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +13: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +20: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +13: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +13: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +20: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +13: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 4: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 4: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +13: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +13: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +13: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +13: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +14: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +13: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +13: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +14: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +20: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +20: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +20: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +20: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +13: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +20: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +14: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +14: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +14: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +20: [2023-05-08 11:26:02,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +14: [2023-05-08 11:26:02,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +14: [2023-05-08 11:26:02,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:26:02,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +14: [2023-05-08 11:26:02,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +14: [2023-05-08 11:26:02,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +14: [2023-05-08 11:26:02,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +14: [2023-05-08 11:26:02,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:26:02,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +14: [2023-05-08 11:26:02,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +21: [2023-05-08 11:26:02,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +27: [2023-05-08 11:26:02,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:26:02,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +27: [2023-05-08 11:26:02,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:26:02,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:26:02,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:26:02,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:26:02,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:26:02,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +27: [2023-05-08 11:26:02,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:26:02,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:26:02,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:26:02,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +12: [2023-05-08 11:26:02,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:26:02,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +27: [2023-05-08 11:26:02,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +27: [2023-05-08 11:26:02,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +12: [2023-05-08 11:26:02,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +21: [2023-05-08 11:26:02,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +20: [2023-05-08 11:26:02,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +14: [2023-05-08 11:26:02,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:26:02,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +13: [2023-05-08 11:26:02,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +13: [2023-05-08 11:26:02,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:26:02,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +14: [2023-05-08 11:26:02,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:26:02,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:26:02,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +14: [2023-05-08 11:26:02,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:26:02,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:26:02,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +12: [2023-05-08 11:26:02,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +12: [2023-05-08 11:26:02,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 3: [2023-05-08 11:26:02,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:26:02,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +13: [2023-05-08 11:26:02,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 3: [2023-05-08 11:26:02,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:26:02,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:26:02,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +27: [2023-05-08 11:26:02,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:26:02,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:26:02,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:26:02,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:26:02,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 3: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +28: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +28: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +20: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +28: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +28: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +26: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:26:02,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +18: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +18: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +18: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +28: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 0: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +19: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 0: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +18: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +18: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 0: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 0: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 8: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 8: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +30: [2023-05-08 11:26:02,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +30: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 3: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 8: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 8: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +30: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 0: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 8: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 8: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +30: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 8: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 0: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 0: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 0: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 0: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 0: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 0: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 0: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 8: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 0: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +30: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +12: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +30: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +30: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 0: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 8: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +22: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +22: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +22: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +30: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +30: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +30: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +30: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 8: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 8: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +30: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +30: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +28: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +22: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 8: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +10: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +22: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +22: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +22: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +22: [2023-05-08 11:26:02,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +10: [2023-05-08 11:26:02,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +10: [2023-05-08 11:26:02,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +22: [2023-05-08 11:26:02,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +10: [2023-05-08 11:26:02,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +10: [2023-05-08 11:26:02,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +10: [2023-05-08 11:26:02,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +10: [2023-05-08 11:26:02,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +10: [2023-05-08 11:26:02,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +10: [2023-05-08 11:26:02,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +19: [2023-05-08 11:26:02,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:26:02,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:26:02,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +18: [2023-05-08 11:26:02,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +10: [2023-05-08 11:26:02,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +10: [2023-05-08 11:26:02,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +10: [2023-05-08 11:26:02,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +26: [2023-05-08 11:26:02,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:26:02,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +19: [2023-05-08 11:26:02,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +22: [2023-05-08 11:26:02,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 0: [2023-05-08 11:26:02,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:26:02,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +10: [2023-05-08 11:26:02,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +26: [2023-05-08 11:26:02,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +10: [2023-05-08 11:26:02,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +10: [2023-05-08 11:26:02,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 0: [2023-05-08 11:26:02,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:26:02,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +11: [2023-05-08 11:26:02,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +22: [2023-05-08 11:26:02,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +22: [2023-05-08 11:26:02,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 4: [2023-05-08 11:26:02,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:26:02,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:26:02,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +19: [2023-05-08 11:26:02,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +14: [2023-05-08 11:26:02,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:26:02,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:26:02,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:26:02,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:26:02,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:26:02,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +24: [2023-05-08 11:26:02,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:26:02,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:26:02,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +24: [2023-05-08 11:26:02,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +21: [2023-05-08 11:26:02,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:26:02,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:26:02,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:26:02,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:26:02,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:26:02,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +13: [2023-05-08 11:26:02,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +13: [2023-05-08 11:26:02,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +31: [2023-05-08 11:26:02,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +11: [2023-05-08 11:26:02,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:26:02,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:26:02,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:26:02,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 3: [2023-05-08 11:26:02,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:26:02,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +30: [2023-05-08 11:26:02,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +30: [2023-05-08 11:26:02,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +30: [2023-05-08 11:26:02,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +10: [2023-05-08 11:26:02,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +30: [2023-05-08 11:26:02,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:26:02,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +18: [2023-05-08 11:26:02,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:26:02,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +22: [2023-05-08 11:26:02,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 9: [2023-05-08 11:26:02,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:26:02,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 8: [2023-05-08 11:26:02,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 8: [2023-05-08 11:26:02,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +28: [2023-05-08 11:26:02,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +10: [2023-05-08 11:26:02,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:26:02,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:26:02,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:26:02,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +19: [2023-05-08 11:26:02,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 8: [2023-05-08 11:26:02,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 8: [2023-05-08 11:26:02,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +10: [2023-05-08 11:26:02,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:26:02,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 9: [2023-05-08 11:26:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +31: [2023-05-08 11:26:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:26:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:26:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +28: [2023-05-08 11:26:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 3: [2023-05-08 11:26:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 4: [2023-05-08 11:26:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:26:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +27: [2023-05-08 11:26:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +27: [2023-05-08 11:26:02,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:26:02,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 3: [2023-05-08 11:26:02,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:26:02,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +20: [2023-05-08 11:26:02,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +20: [2023-05-08 11:26:02,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:26:02,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:26:02,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 0: [2023-05-08 11:26:02,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 0: [2023-05-08 11:26:02,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:26:02,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:26:02,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +13: [2023-05-08 11:26:02,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:26:02,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:26:02,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:26:02,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:26:02,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +30: [2023-05-08 11:26:02,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 3: [2023-05-08 11:26:02,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:26:02,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +18: [2023-05-08 11:26:02,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:26:02,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 9: [2023-05-08 11:26:02,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:26:02,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +28: [2023-05-08 11:26:02,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:26:02,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +18: [2023-05-08 11:26:02,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:26:02,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +18: [2023-05-08 11:26:02,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +13: [2023-05-08 11:26:02,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:26:02,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +13: [2023-05-08 11:26:02,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:26:02,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:26:02,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:26:02,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 8: [2023-05-08 11:26:02,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +13: [2023-05-08 11:26:02,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +10: [2023-05-08 11:26:02,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:26:02,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:26:02,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +10: [2023-05-08 11:26:02,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +28: [2023-05-08 11:26:02,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 4: [2023-05-08 11:26:02,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +21: [2023-05-08 11:26:02,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:26:02,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +12: [2023-05-08 11:26:02,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:26:02,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +10: [2023-05-08 11:26:02,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 0: [2023-05-08 11:26:02,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:26:02,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:26:02,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:26:02,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +14: [2023-05-08 11:26:02,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:26:02,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +13: [2023-05-08 11:26:02,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:26:02,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 9: [2023-05-08 11:26:02,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +20: [2023-05-08 11:26:02,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +18: [2023-05-08 11:26:02,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:26:02,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +14: [2023-05-08 11:26:02,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +28: [2023-05-08 11:26:02,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:26:02,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 0: [2023-05-08 11:26:02,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +14: [2023-05-08 11:26:02,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 4: [2023-05-08 11:26:02,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:26:02,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:26:02,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +22: [2023-05-08 11:26:02,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +27: [2023-05-08 11:26:02,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:26:02,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:26:02,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:26:02,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +13: [2023-05-08 11:26:02,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:26:02,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +22: [2023-05-08 11:26:02,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:26:02,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:26:02,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:26:02,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:26:02,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:26:02,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:26:02,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:26:02,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:26:02,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:26:02,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 6: [2023-05-08 11:26:02,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:26:02,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:26:02,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:26:02,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 6: [2023-05-08 11:26:02,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... + 6: [2023-05-08 11:26:02,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 6: [2023-05-08 11:26:02,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 6: [2023-05-08 11:26:02,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:26:02,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +16: [2023-05-08 11:26:02,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:26:02,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:26:02,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:26:02,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:26:02,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +16: [2023-05-08 11:26:02,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:26:02,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +16: [2023-05-08 11:26:02,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +16: [2023-05-08 11:26:02,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt... +16: [2023-05-08 11:26:02,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:26:02,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +16: [2023-05-08 11:26:02,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:26:02,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:26:02,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 6: [2023-05-08 11:26:02,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +16: [2023-05-08 11:26:02,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +16: [2023-05-08 11:26:02,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... +16: [2023-05-08 11:26:02,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt... + 6: [2023-05-08 11:26:02,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:26:02,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:26:02,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:26:02,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:26:02,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:26:02,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:26:02,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +16: [2023-05-08 11:26:02,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. + 6: [2023-05-08 11:26:02,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:26:02,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +16: [2023-05-08 11:26:02,851] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:26:02,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt. +16: [2023-05-08 11:26:02,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +16: [2023-05-08 11:26:02,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +16: [2023-05-08 11:26:02,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:26:02,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 7: [2023-05-08 11:26:02,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 7: [2023-05-08 11:26:02,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 7: [2023-05-08 11:26:02,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +26: [2023-05-08 11:26:02,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +26: [2023-05-08 11:26:02,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 1: [2023-05-08 11:26:02,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 1: [2023-05-08 11:26:02,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 1: [2023-05-08 11:26:02,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 1: [2023-05-08 11:26:02,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +25: [2023-05-08 11:26:02,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +25: [2023-05-08 11:26:02,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +25: [2023-05-08 11:26:02,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +26: [2023-05-08 11:26:02,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +25: [2023-05-08 11:26:02,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +26: [2023-05-08 11:26:02,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 7: [2023-05-08 11:26:02,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:26:02,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:26:02,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 1: [2023-05-08 11:26:02,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +25: [2023-05-08 11:26:02,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +25: [2023-05-08 11:26:02,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +25: [2023-05-08 11:26:02,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +25: [2023-05-08 11:26:02,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:26:02,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +26: [2023-05-08 11:26:02,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:26:02,893] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 7: [2023-05-08 11:26:02,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 7: [2023-05-08 11:26:02,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +26: [2023-05-08 11:26:02,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +26: [2023-05-08 11:26:02,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 1: [2023-05-08 11:26:02,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +26: [2023-05-08 11:26:02,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +26: [2023-05-08 11:26:02,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +25: [2023-05-08 11:26:02,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +26: [2023-05-08 11:26:02,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +26: [2023-05-08 11:26:02,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +25: [2023-05-08 11:26:02,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 1: [2023-05-08 11:26:02,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 1: [2023-05-08 11:26:02,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 1: [2023-05-08 11:26:02,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +25: [2023-05-08 11:26:02,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +25: [2023-05-08 11:26:02,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +26: [2023-05-08 11:26:02,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +26: [2023-05-08 11:26:02,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +25: [2023-05-08 11:26:02,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +25: [2023-05-08 11:26:02,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:26:02,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +25: [2023-05-08 11:26:02,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +25: [2023-05-08 11:26:02,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 7: [2023-05-08 11:26:02,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +19: [2023-05-08 11:26:02,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +19: [2023-05-08 11:26:02,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +19: [2023-05-08 11:26:02,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +19: [2023-05-08 11:26:02,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +15: [2023-05-08 11:26:02,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +15: [2023-05-08 11:26:02,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +15: [2023-05-08 11:26:02,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +15: [2023-05-08 11:26:02,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 2: [2023-05-08 11:26:02,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 2: [2023-05-08 11:26:02,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 2: [2023-05-08 11:26:02,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +19: [2023-05-08 11:26:02,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:26:02,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:26:02,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 4: [2023-05-08 11:26:02,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 4: [2023-05-08 11:26:02,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:26:02,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 4: [2023-05-08 11:26:02,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:26:02,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 4: [2023-05-08 11:26:02,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:26:02,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +23: [2023-05-08 11:26:02,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +15: [2023-05-08 11:26:02,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:26:02,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +15: [2023-05-08 11:26:02,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +15: [2023-05-08 11:26:02,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +15: [2023-05-08 11:26:02,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:26:02,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:26:02,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +23: [2023-05-08 11:26:02,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:26:02,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 4: [2023-05-08 11:26:02,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:26:02,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:26:02,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:26:02,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +19: [2023-05-08 11:26:02,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +19: [2023-05-08 11:26:02,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +19: [2023-05-08 11:26:02,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 2: [2023-05-08 11:26:02,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +15: [2023-05-08 11:26:02,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:26:02,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:26:02,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:26:02,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:26:02,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +15: [2023-05-08 11:26:02,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 4: [2023-05-08 11:26:02,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:26:02,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +12: [2023-05-08 11:26:02,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +12: [2023-05-08 11:26:02,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +15: [2023-05-08 11:26:02,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +12: [2023-05-08 11:26:02,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:26:02,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +15: [2023-05-08 11:26:02,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +21: [2023-05-08 11:26:02,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:26:02,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:26:02,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +21: [2023-05-08 11:26:02,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:26:02,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +29: [2023-05-08 11:26:02,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +29: [2023-05-08 11:26:02,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +29: [2023-05-08 11:26:02,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +15: [2023-05-08 11:26:02,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +29: [2023-05-08 11:26:02,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +15: [2023-05-08 11:26:02,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +15: [2023-05-08 11:26:02,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +12: [2023-05-08 11:26:02,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +12: [2023-05-08 11:26:02,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +12: [2023-05-08 11:26:02,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:26:02,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +12: [2023-05-08 11:26:02,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 4: [2023-05-08 11:26:02,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:26:02,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 2: [2023-05-08 11:26:02,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 2: [2023-05-08 11:26:02,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:26:02,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +19: [2023-05-08 11:26:02,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:26:02,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +19: [2023-05-08 11:26:02,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +23: [2023-05-08 11:26:02,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:26:02,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:26:02,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +23: [2023-05-08 11:26:02,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +21: [2023-05-08 11:26:02,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:26:02,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:26:02,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:26:02,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:26:02,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +29: [2023-05-08 11:26:02,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +29: [2023-05-08 11:26:02,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +29: [2023-05-08 11:26:02,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 4: [2023-05-08 11:26:02,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +12: [2023-05-08 11:26:02,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:26:02,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +21: [2023-05-08 11:26:02,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +20: [2023-05-08 11:26:02,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +20: [2023-05-08 11:26:02,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +20: [2023-05-08 11:26:02,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +20: [2023-05-08 11:26:02,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:26:02,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:26:02,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +20: [2023-05-08 11:26:02,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:26:02,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:26:02,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:26:02,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:26:02,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:26:02,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +12: [2023-05-08 11:26:02,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:26:02,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +12: [2023-05-08 11:26:02,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +31: [2023-05-08 11:26:02,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +31: [2023-05-08 11:26:02,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +31: [2023-05-08 11:26:02,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +31: [2023-05-08 11:26:02,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:26:02,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:26:02,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +31: [2023-05-08 11:26:02,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:26:02,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +29: [2023-05-08 11:26:02,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +12: [2023-05-08 11:26:02,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:26:02,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +21: [2023-05-08 11:26:02,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +12: [2023-05-08 11:26:02,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +21: [2023-05-08 11:26:02,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:26:02,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:26:02,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +29: [2023-05-08 11:26:02,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +29: [2023-05-08 11:26:02,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +12: [2023-05-08 11:26:02,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +12: [2023-05-08 11:26:02,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:26:02,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +11: [2023-05-08 11:26:02,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +11: [2023-05-08 11:26:02,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +11: [2023-05-08 11:26:02,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +31: [2023-05-08 11:26:02,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +31: [2023-05-08 11:26:03,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +20: [2023-05-08 11:26:03,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +20: [2023-05-08 11:26:03,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +20: [2023-05-08 11:26:03,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +11: [2023-05-08 11:26:03,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +11: [2023-05-08 11:26:03,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:26:03,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +24: [2023-05-08 11:26:03,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +11: [2023-05-08 11:26:03,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +11: [2023-05-08 11:26:03,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:26:03,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +24: [2023-05-08 11:26:03,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 0: [2023-05-08 11:26:03,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:26:03,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +31: [2023-05-08 11:26:03,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +20: [2023-05-08 11:26:03,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +31: [2023-05-08 11:26:03,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +31: [2023-05-08 11:26:03,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +20: [2023-05-08 11:26:03,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +20: [2023-05-08 11:26:03,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:26:03,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:26:03,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:26:03,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +24: [2023-05-08 11:26:03,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +11: [2023-05-08 11:26:03,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +11: [2023-05-08 11:26:03,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +24: [2023-05-08 11:26:03,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +11: [2023-05-08 11:26:03,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +11: [2023-05-08 11:26:03,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +13: [2023-05-08 11:26:03,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +13: [2023-05-08 11:26:03,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +13: [2023-05-08 11:26:03,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +13: [2023-05-08 11:26:03,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +31: [2023-05-08 11:26:03,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 5: [2023-05-08 11:26:03,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +20: [2023-05-08 11:26:03,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 5: [2023-05-08 11:26:03,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +31: [2023-05-08 11:26:03,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +20: [2023-05-08 11:26:03,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +21: [2023-05-08 11:26:03,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +21: [2023-05-08 11:26:03,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +21: [2023-05-08 11:26:03,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +12: [2023-05-08 11:26:03,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +12: [2023-05-08 11:26:03,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +12: [2023-05-08 11:26:03,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +21: [2023-05-08 11:26:03,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 9: [2023-05-08 11:26:03,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 9: [2023-05-08 11:26:03,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 9: [2023-05-08 11:26:03,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +12: [2023-05-08 11:26:03,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +20: [2023-05-08 11:26:03,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +11: [2023-05-08 11:26:03,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +11: [2023-05-08 11:26:03,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +20: [2023-05-08 11:26:03,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +11: [2023-05-08 11:26:03,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +11: [2023-05-08 11:26:03,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +20: [2023-05-08 11:26:03,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:26:03,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +24: [2023-05-08 11:26:03,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 0: [2023-05-08 11:26:03,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 8: [2023-05-08 11:26:03,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 8: [2023-05-08 11:26:03,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 8: [2023-05-08 11:26:03,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +13: [2023-05-08 11:26:03,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:26:03,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:26:03,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +14: [2023-05-08 11:26:03,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:26:03,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +14: [2023-05-08 11:26:03,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +14: [2023-05-08 11:26:03,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:26:03,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +12: [2023-05-08 11:26:03,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:26:03,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:26:03,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:26:03,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:26:03,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:26:03,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:26:03,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:26:03,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +18: [2023-05-08 11:26:03,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +18: [2023-05-08 11:26:03,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +18: [2023-05-08 11:26:03,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +18: [2023-05-08 11:26:03,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:26:03,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:26:03,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +18: [2023-05-08 11:26:03,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +20: [2023-05-08 11:26:03,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +18: [2023-05-08 11:26:03,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:26:03,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:26:03,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +24: [2023-05-08 11:26:03,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +24: [2023-05-08 11:26:03,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +24: [2023-05-08 11:26:03,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +24: [2023-05-08 11:26:03,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +10: [2023-05-08 11:26:03,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +10: [2023-05-08 11:26:03,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +10: [2023-05-08 11:26:03,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +10: [2023-05-08 11:26:03,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:26:03,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +27: [2023-05-08 11:26:03,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:26:03,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +10: [2023-05-08 11:26:03,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +27: [2023-05-08 11:26:03,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +27: [2023-05-08 11:26:03,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +18: [2023-05-08 11:26:03,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +18: [2023-05-08 11:26:03,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:26:03,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +10: [2023-05-08 11:26:03,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:26:03,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +13: [2023-05-08 11:26:03,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +13: [2023-05-08 11:26:03,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:26:03,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +13: [2023-05-08 11:26:03,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:26:03,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +30: [2023-05-08 11:26:03,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +30: [2023-05-08 11:26:03,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +30: [2023-05-08 11:26:03,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 0: [2023-05-08 11:26:03,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +24: [2023-05-08 11:26:03,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +24: [2023-05-08 11:26:03,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +10: [2023-05-08 11:26:03,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +10: [2023-05-08 11:26:03,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +27: [2023-05-08 11:26:03,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:26:03,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +27: [2023-05-08 11:26:03,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +27: [2023-05-08 11:26:03,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:26:03,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +22: [2023-05-08 11:26:03,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +22: [2023-05-08 11:26:03,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:26:03,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:26:03,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +22: [2023-05-08 11:26:03,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:26:03,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +18: [2023-05-08 11:26:03,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +20: [2023-05-08 11:26:03,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +22: [2023-05-08 11:26:03,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +22: [2023-05-08 11:26:03,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:26:03,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +20: [2023-05-08 11:26:03,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +30: [2023-05-08 11:26:03,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:26:03,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +20: [2023-05-08 11:26:03,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +18: [2023-05-08 11:26:03,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 6: [2023-05-08 11:26:03,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +30: [2023-05-08 11:26:03,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:26:03,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +20: [2023-05-08 11:26:03,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +20: [2023-05-08 11:26:03,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 9: [2023-05-08 11:26:03,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +30: [2023-05-08 11:26:03,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:26:03,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 5: [2023-05-08 11:26:03,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +10: [2023-05-08 11:26:03,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +27: [2023-05-08 11:26:03,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:26:03,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +10: [2023-05-08 11:26:03,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +21: [2023-05-08 11:26:03,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:26:03,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:26:03,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +21: [2023-05-08 11:26:03,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +10: [2023-05-08 11:26:03,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +14: [2023-05-08 11:26:03,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +12: [2023-05-08 11:26:03,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +12: [2023-05-08 11:26:03,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +12: [2023-05-08 11:26:03,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +10: [2023-05-08 11:26:03,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +12: [2023-05-08 11:26:03,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +12: [2023-05-08 11:26:03,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +12: [2023-05-08 11:26:03,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +18: [2023-05-08 11:26:03,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +18: [2023-05-08 11:26:03,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 9: [2023-05-08 11:26:03,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +10: [2023-05-08 11:26:03,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +18: [2023-05-08 11:26:03,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +31: [2023-05-08 11:26:03,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +31: [2023-05-08 11:26:03,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +31: [2023-05-08 11:26:03,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 0: [2023-05-08 11:26:03,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 0: [2023-05-08 11:26:03,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +17: [2023-05-08 11:26:03,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 5: [2023-05-08 11:26:03,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +18: [2023-05-08 11:26:03,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 0: [2023-05-08 11:26:03,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +17: [2023-05-08 11:26:03,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +17: [2023-05-08 11:26:03,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +31: [2023-05-08 11:26:03,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +27: [2023-05-08 11:26:03,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +27: [2023-05-08 11:26:03,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +14: [2023-05-08 11:26:03,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +21: [2023-05-08 11:26:03,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:26:03,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 5: [2023-05-08 11:26:03,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +12: [2023-05-08 11:26:03,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:26:03,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +18: [2023-05-08 11:26:03,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +27: [2023-05-08 11:26:03,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +16: [2023-05-08 11:26:03,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +10: [2023-05-08 11:26:03,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 4: [2023-05-08 11:26:03,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 4: [2023-05-08 11:26:03,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +10: [2023-05-08 11:26:03,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +12: [2023-05-08 11:26:03,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +21: [2023-05-08 11:26:03,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +21: [2023-05-08 11:26:03,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 3: [2023-05-08 11:26:03,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +28: [2023-05-08 11:26:03,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 3: [2023-05-08 11:26:03,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 5: [2023-05-08 11:26:03,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +28: [2023-05-08 11:26:03,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 3: [2023-05-08 11:26:03,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 4: [2023-05-08 11:26:03,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +18: [2023-05-08 11:26:03,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:26:03,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +18: [2023-05-08 11:26:03,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +22: [2023-05-08 11:26:03,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 6: [2023-05-08 11:26:03,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:26:03,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +16: [2023-05-08 11:26:03,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +22: [2023-05-08 11:26:03,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +27: [2023-05-08 11:26:03,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 6: [2023-05-08 11:26:03,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:26:03,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +14: [2023-05-08 11:26:03,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:26:03,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 9: [2023-05-08 11:26:03,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +14: [2023-05-08 11:26:03,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:26:03,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:26:03,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:26:03,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:26:03,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. + 9: [2023-05-08 11:26:03,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +27: [2023-05-08 11:26:03,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:26:03,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +14: [2023-05-08 11:26:03,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +17: [2023-05-08 11:26:03,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:26:03,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +28: [2023-05-08 11:26:03,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +20: [2023-05-08 11:26:03,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:26:03,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +18: [2023-05-08 11:26:03,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:26:03,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:26:03,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt. +10: [2023-05-08 11:26:03,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:26:03,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +22: [2023-05-08 11:26:03,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:26:03,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +18: [2023-05-08 11:26:03,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:26:03,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:26:03,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:26:03,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +10: [2023-05-08 11:26:03,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +27: [2023-05-08 11:26:03,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +18: [2023-05-08 11:26:03,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +20: [2023-05-08 11:26:03,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +10: [2023-05-08 11:26:03,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +31: [2023-05-08 11:26:03,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +16: [2023-05-08 11:26:03,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +31: [2023-05-08 11:26:03,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +18: [2023-05-08 11:26:03,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +27: [2023-05-08 11:26:03,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +17: [2023-05-08 11:26:03,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +17: [2023-05-08 11:26:03,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +28: [2023-05-08 11:26:03,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +18: [2023-05-08 11:26:03,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:26:03,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:26:03,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +27: [2023-05-08 11:26:03,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:26:03,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:26:03,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:26:03,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:26:03,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +28: [2023-05-08 11:26:03,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +16: [2023-05-08 11:26:03,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +28: [2023-05-08 11:26:03,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +10: [2023-05-08 11:26:03,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +10: [2023-05-08 11:26:03,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +14: [2023-05-08 11:26:03,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +27: [2023-05-08 11:26:03,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +16: [2023-05-08 11:26:03,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +16: [2023-05-08 11:26:03,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +22: [2023-05-08 11:26:03,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:26:03,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:26:03,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +23: [2023-05-08 11:26:03,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +23: [2023-05-08 11:26:03,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:26:03,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:26:03,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +23: [2023-05-08 11:26:03,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +28: [2023-05-08 11:26:03,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +14: [2023-05-08 11:26:03,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +17: [2023-05-08 11:26:03,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:26:03,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:26:03,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:26:03,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +23: [2023-05-08 11:26:03,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:26:03,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +10: [2023-05-08 11:26:03,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +27: [2023-05-08 11:26:03,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:26:03,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:26:03,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:26:03,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +18: [2023-05-08 11:26:03,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +17: [2023-05-08 11:26:03,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +17: [2023-05-08 11:26:03,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:26:03,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +31: [2023-05-08 11:26:03,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +31: [2023-05-08 11:26:03,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +31: [2023-05-08 11:26:03,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +31: [2023-05-08 11:26:03,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +31: [2023-05-08 11:26:03,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +31: [2023-05-08 11:26:03,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 5: [2023-05-08 11:26:03,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:26:03,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:26:03,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +18: [2023-05-08 11:26:03,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +18: [2023-05-08 11:26:03,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +27: [2023-05-08 11:26:03,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +10: [2023-05-08 11:26:03,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +22: [2023-05-08 11:26:03,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:26:03,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 5: [2023-05-08 11:26:03,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:26:03,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +27: [2023-05-08 11:26:03,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:26:03,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +17: [2023-05-08 11:26:03,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +17: [2023-05-08 11:26:03,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +17: [2023-05-08 11:26:03,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:26:03,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:26:03,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 4: [2023-05-08 11:26:03,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +17: [2023-05-08 11:26:03,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +17: [2023-05-08 11:26:03,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:26:03,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 0: [2023-05-08 11:26:03,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +22: [2023-05-08 11:26:03,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +14: [2023-05-08 11:26:03,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +14: [2023-05-08 11:26:03,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +23: [2023-05-08 11:26:03,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +18: [2023-05-08 11:26:03,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +18: [2023-05-08 11:26:03,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +22: [2023-05-08 11:26:03,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +22: [2023-05-08 11:26:03,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +22: [2023-05-08 11:26:03,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +22: [2023-05-08 11:26:03,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +14: [2023-05-08 11:26:03,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +31: [2023-05-08 11:26:03,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +14: [2023-05-08 11:26:03,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +27: [2023-05-08 11:26:03,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:26:03,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +14: [2023-05-08 11:26:03,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +14: [2023-05-08 11:26:03,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +14: [2023-05-08 11:26:03,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +14: [2023-05-08 11:26:03,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 0: [2023-05-08 11:26:03,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +27: [2023-05-08 11:26:03,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:26:03,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +31: [2023-05-08 11:26:03,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 5: [2023-05-08 11:26:03,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +22: [2023-05-08 11:26:03,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:26:03,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +18: [2023-05-08 11:26:03,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +27: [2023-05-08 11:26:03,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +27: [2023-05-08 11:26:03,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +17: [2023-05-08 11:26:03,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +17: [2023-05-08 11:26:03,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:26:03,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:26:03,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:26:03,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +17: [2023-05-08 11:26:03,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +23: [2023-05-08 11:26:03,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 5: [2023-05-08 11:26:03,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +30: [2023-05-08 11:26:03,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +30: [2023-05-08 11:26:03,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +30: [2023-05-08 11:26:03,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +30: [2023-05-08 11:26:03,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +30: [2023-05-08 11:26:03,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +30: [2023-05-08 11:26:03,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +30: [2023-05-08 11:26:03,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:26:03,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:26:03,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +30: [2023-05-08 11:26:03,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +30: [2023-05-08 11:26:03,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +30: [2023-05-08 11:26:03,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +30: [2023-05-08 11:26:03,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +30: [2023-05-08 11:26:03,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +23: [2023-05-08 11:26:03,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +17: [2023-05-08 11:26:03,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +17: [2023-05-08 11:26:03,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +30: [2023-05-08 11:26:03,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:26:03,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +17: [2023-05-08 11:26:03,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:26:03,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +23: [2023-05-08 11:26:03,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +22: [2023-05-08 11:26:03,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +17: [2023-05-08 11:26:03,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:26:03,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +23: [2023-05-08 11:26:03,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +17: [2023-05-08 11:26:03,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +17: [2023-05-08 11:26:03,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +30: [2023-05-08 11:26:03,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +30: [2023-05-08 11:26:03,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +23: [2023-05-08 11:26:03,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:26:03,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:26:03,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +30: [2023-05-08 11:26:03,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:26:03,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +16: [2023-05-08 11:26:03,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +16: [2023-05-08 11:26:03,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +16: [2023-05-08 11:26:03,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +16: [2023-05-08 11:26:03,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +16: [2023-05-08 11:26:03,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +16: [2023-05-08 11:26:03,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +16: [2023-05-08 11:26:03,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +16: [2023-05-08 11:26:03,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +16: [2023-05-08 11:26:03,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +16: [2023-05-08 11:26:03,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +16: [2023-05-08 11:26:03,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:26:03,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 8: [2023-05-08 11:26:03,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 3: [2023-05-08 11:26:03,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +16: [2023-05-08 11:26:03,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +16: [2023-05-08 11:26:03,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +16: [2023-05-08 11:26:03,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +30: [2023-05-08 11:26:03,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +30: [2023-05-08 11:26:03,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 3: [2023-05-08 11:26:03,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 3: [2023-05-08 11:26:03,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 8: [2023-05-08 11:26:03,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 3: [2023-05-08 11:26:03,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 3: [2023-05-08 11:26:03,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 6: [2023-05-08 11:26:03,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 6: [2023-05-08 11:26:03,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 6: [2023-05-08 11:26:03,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 7: [2023-05-08 11:26:03,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:26:03,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +30: [2023-05-08 11:26:03,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +16: [2023-05-08 11:26:03,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:26:03,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:26:03,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:26:03,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:26:03,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +16: [2023-05-08 11:26:03,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:26:03,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:26:03,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +28: [2023-05-08 11:26:03,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +16: [2023-05-08 11:26:03,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:26:03,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 3: [2023-05-08 11:26:03,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:26:03,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:26:03,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:26:03,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:26:03,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:26:03,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:26:03,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +28: [2023-05-08 11:26:03,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +28: [2023-05-08 11:26:03,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +28: [2023-05-08 11:26:03,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:26:03,157] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:26:03,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +11: [2023-05-08 11:26:03,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:26:03,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:26:03,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:26:03,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:26:03,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:26:03,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:26:03,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:26:03,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:26:03,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +13: [2023-05-08 11:26:03,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:26:03,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:26:03,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:26:03,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:26:03,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +13: [2023-05-08 11:26:03,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +13: [2023-05-08 11:26:03,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:26:03,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:26:03,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:26:03,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:26:03,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +13: [2023-05-08 11:26:03,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt... +13: [2023-05-08 11:26:03,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +13: [2023-05-08 11:26:03,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +13: [2023-05-08 11:26:03,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... + 2: [2023-05-08 11:26:03,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:26:03,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:26:03,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt... +28: [2023-05-08 11:26:03,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:26:03,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:26:03,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +11: [2023-05-08 11:26:03,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:26:03,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:26:03,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +28: [2023-05-08 11:26:03,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:26:03,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +13: [2023-05-08 11:26:03,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:26:03,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +28: [2023-05-08 11:26:03,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:26:03,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:26:03,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +13: [2023-05-08 11:26:03,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +11: [2023-05-08 11:26:03,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:26:03,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:26:03,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +28: [2023-05-08 11:26:03,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt. +24: [2023-05-08 11:26:03,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:26:03,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:26:03,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:26:03,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +13: [2023-05-08 11:26:03,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:26:03,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:26:03,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:26:03,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:26:03,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +13: [2023-05-08 11:26:03,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:26:03,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +13: [2023-05-08 11:26:03,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +13: [2023-05-08 11:26:03,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:26:03,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 1: [2023-05-08 11:26:03,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 1: [2023-05-08 11:26:03,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 1: [2023-05-08 11:26:03,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 1: [2023-05-08 11:26:03,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:26:03,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:26:03,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:26:03,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:26:03,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +25: [2023-05-08 11:26:03,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:26:03,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:26:03,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:26:03,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +25: [2023-05-08 11:26:03,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:26:03,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:26:03,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +25: [2023-05-08 11:26:03,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +25: [2023-05-08 11:26:03,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 1: [2023-05-08 11:26:03,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:26:03,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +25: [2023-05-08 11:26:03,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +25: [2023-05-08 11:26:03,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +24: [2023-05-08 11:26:03,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:26:03,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:26:03,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:26:03,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +25: [2023-05-08 11:26:03,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:26:03,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:26:03,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +25: [2023-05-08 11:26:03,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:26:03,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +26: [2023-05-08 11:26:03,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:26:03,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 1: [2023-05-08 11:26:03,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +26: [2023-05-08 11:26:03,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:26:03,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +26: [2023-05-08 11:26:03,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:26:03,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 7: [2023-05-08 11:26:03,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +26: [2023-05-08 11:26:03,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +26: [2023-05-08 11:26:03,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 7: [2023-05-08 11:26:03,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 1: [2023-05-08 11:26:03,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:26:03,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:26:03,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:26:03,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:26:03,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:26:03,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 7: [2023-05-08 11:26:03,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +25: [2023-05-08 11:26:03,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 7: [2023-05-08 11:26:03,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:26:03,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:26:03,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +26: [2023-05-08 11:26:03,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +26: [2023-05-08 11:26:03,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +26: [2023-05-08 11:26:03,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +25: [2023-05-08 11:26:03,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:26:03,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +26: [2023-05-08 11:26:03,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:26:03,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:26:03,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:26:03,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 9: [2023-05-08 11:26:03,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:26:03,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +15: [2023-05-08 11:26:03,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 9: [2023-05-08 11:26:03,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +15: [2023-05-08 11:26:03,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 9: [2023-05-08 11:26:03,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +26: [2023-05-08 11:26:03,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +25: [2023-05-08 11:26:03,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:26:03,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +25: [2023-05-08 11:26:03,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:26:03,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 7: [2023-05-08 11:26:03,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:26:03,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +15: [2023-05-08 11:26:03,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:26:03,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:26:03,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:26:03,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:26:03,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:26:03,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +26: [2023-05-08 11:26:03,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +26: [2023-05-08 11:26:03,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:26:03,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:26:03,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:26:03,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:26:03,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:26:03,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +26: [2023-05-08 11:26:03,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:26:03,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:26:03,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +25: [2023-05-08 11:26:03,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +19: [2023-05-08 11:26:03,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +19: [2023-05-08 11:26:03,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +19: [2023-05-08 11:26:03,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +26: [2023-05-08 11:26:03,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +19: [2023-05-08 11:26:03,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 9: [2023-05-08 11:26:03,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:26:03,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:26:03,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +15: [2023-05-08 11:26:03,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:26:03,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +26: [2023-05-08 11:26:03,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:26:03,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:26:03,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +26: [2023-05-08 11:26:03,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 9: [2023-05-08 11:26:03,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:26:03,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +19: [2023-05-08 11:26:03,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +15: [2023-05-08 11:26:03,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:26:03,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:26:03,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +15: [2023-05-08 11:26:03,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +19: [2023-05-08 11:26:03,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +26: [2023-05-08 11:26:03,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:26:03,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,285] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +15: [2023-05-08 11:26:03,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:26:03,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:26:03,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:26:03,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 9: [2023-05-08 11:26:03,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:26:03,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:26:03,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:26:03,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +15: [2023-05-08 11:26:03,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:26:03,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:26:03,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +26: [2023-05-08 11:26:03,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 9: [2023-05-08 11:26:03,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +15: [2023-05-08 11:26:03,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:26:03,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:26:03,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:26:03,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:26:03,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +20: [2023-05-08 11:26:03,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +20: [2023-05-08 11:26:03,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:26:03,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:26:03,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:26:03,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:26:03,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:26:03,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:26:03,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:26:03,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:26:03,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:26:03,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:26:03,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:26:03,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:26:03,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +20: [2023-05-08 11:26:03,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:26:03,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:26:03,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +12: [2023-05-08 11:26:03,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +12: [2023-05-08 11:26:03,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +12: [2023-05-08 11:26:03,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +12: [2023-05-08 11:26:03,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +12: [2023-05-08 11:26:03,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +12: [2023-05-08 11:26:03,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +12: [2023-05-08 11:26:03,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +12: [2023-05-08 11:26:03,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:26:03,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +23: [2023-05-08 11:26:03,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +23: [2023-05-08 11:26:03,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +23: [2023-05-08 11:26:03,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +23: [2023-05-08 11:26:03,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +20: [2023-05-08 11:26:03,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:26:03,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:26:03,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:26:03,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:26:03,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:26:03,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +11: [2023-05-08 11:26:03,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +20: [2023-05-08 11:26:03,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:26:03,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +12: [2023-05-08 11:26:03,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:26:03,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +29: [2023-05-08 11:26:03,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:26:03,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:26:03,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:26:03,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:26:03,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:26:03,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:26:03,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +17: [2023-05-08 11:26:03,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +17: [2023-05-08 11:26:03,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:26:03,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:26:03,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:26:03,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:26:03,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:26:03,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +12: [2023-05-08 11:26:03,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +12: [2023-05-08 11:26:03,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:26:03,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +29: [2023-05-08 11:26:03,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:26:03,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +19: [2023-05-08 11:26:03,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +19: [2023-05-08 11:26:03,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +19: [2023-05-08 11:26:03,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +19: [2023-05-08 11:26:03,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +19: [2023-05-08 11:26:03,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +19: [2023-05-08 11:26:03,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +19: [2023-05-08 11:26:03,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +19: [2023-05-08 11:26:03,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +19: [2023-05-08 11:26:03,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +19: [2023-05-08 11:26:03,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +19: [2023-05-08 11:26:03,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +19: [2023-05-08 11:26:03,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +19: [2023-05-08 11:26:03,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +11: [2023-05-08 11:26:03,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:26:03,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +19: [2023-05-08 11:26:03,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +19: [2023-05-08 11:26:03,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +19: [2023-05-08 11:26:03,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 7: [2023-05-08 11:26:03,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:26:03,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:26:03,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 2: [2023-05-08 11:26:03,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 2: [2023-05-08 11:26:03,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +29: [2023-05-08 11:26:03,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +29: [2023-05-08 11:26:03,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +29: [2023-05-08 11:26:03,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +29: [2023-05-08 11:26:03,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:26:03,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 2: [2023-05-08 11:26:03,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +21: [2023-05-08 11:26:03,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +21: [2023-05-08 11:26:03,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +21: [2023-05-08 11:26:03,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +21: [2023-05-08 11:26:03,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +17: [2023-05-08 11:26:03,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +20: [2023-05-08 11:26:03,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +20: [2023-05-08 11:26:03,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +20: [2023-05-08 11:26:03,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +20: [2023-05-08 11:26:03,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 2: [2023-05-08 11:26:03,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:26:03,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:26:03,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +17: [2023-05-08 11:26:03,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +19: [2023-05-08 11:26:03,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +19: [2023-05-08 11:26:03,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:26:03,365] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 4: [2023-05-08 11:26:03,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 4: [2023-05-08 11:26:03,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 4: [2023-05-08 11:26:03,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +12: [2023-05-08 11:26:03,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:26:03,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:26:03,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 2: [2023-05-08 11:26:03,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +21: [2023-05-08 11:26:03,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +19: [2023-05-08 11:26:03,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:26:03,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:26:03,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +21: [2023-05-08 11:26:03,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:26:03,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +23: [2023-05-08 11:26:03,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:26:03,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:26:03,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:26:03,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +11: [2023-05-08 11:26:03,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:26:03,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 5: [2023-05-08 11:26:03,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:26:03,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +23: [2023-05-08 11:26:03,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +11: [2023-05-08 11:26:03,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:26:03,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:26:03,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +23: [2023-05-08 11:26:03,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 5: [2023-05-08 11:26:03,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +31: [2023-05-08 11:26:03,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:26:03,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +31: [2023-05-08 11:26:03,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:26:03,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +31: [2023-05-08 11:26:03,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:26:03,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +31: [2023-05-08 11:26:03,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +31: [2023-05-08 11:26:03,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:26:03,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:26:03,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +31: [2023-05-08 11:26:03,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:26:03,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:26:03,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +29: [2023-05-08 11:26:03,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:26:03,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:26:03,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +17: [2023-05-08 11:26:03,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +21: [2023-05-08 11:26:03,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +21: [2023-05-08 11:26:03,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +19: [2023-05-08 11:26:03,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:26:03,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:26:03,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +12: [2023-05-08 11:26:03,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +17: [2023-05-08 11:26:03,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +19: [2023-05-08 11:26:03,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:26:03,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:26:03,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +23: [2023-05-08 11:26:03,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 7: [2023-05-08 11:26:03,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +20: [2023-05-08 11:26:03,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:26:03,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 2: [2023-05-08 11:26:03,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:26:03,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:26:03,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:26:03,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:26:03,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:26:03,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:26:03,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:26:03,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +23: [2023-05-08 11:26:03,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:26:03,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:26:03,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:26:03,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:26:03,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:26:03,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +29: [2023-05-08 11:26:03,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:26:03,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:26:03,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +29: [2023-05-08 11:26:03,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +31: [2023-05-08 11:26:03,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:26:03,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +29: [2023-05-08 11:26:03,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 5: [2023-05-08 11:26:03,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +29: [2023-05-08 11:26:03,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +31: [2023-05-08 11:26:03,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:26:03,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +24: [2023-05-08 11:26:03,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +24: [2023-05-08 11:26:03,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +24: [2023-05-08 11:26:03,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 5: [2023-05-08 11:26:03,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 2: [2023-05-08 11:26:03,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:26:03,399] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +31: [2023-05-08 11:26:03,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:26:03,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:26:03,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:26:03,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:26:03,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +20: [2023-05-08 11:26:03,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +20: [2023-05-08 11:26:03,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +23: [2023-05-08 11:26:03,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +31: [2023-05-08 11:26:03,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:26:03,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:26:03,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:26:03,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +18: [2023-05-08 11:26:03,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:26:03,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +18: [2023-05-08 11:26:03,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +18: [2023-05-08 11:26:03,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:26:03,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:26:03,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:26:03,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:26:03,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:26:03,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:26:03,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:26:03,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:26:03,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:26:03,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:26:03,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:26:03,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:26:03,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:26:03,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:26:03,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:26:03,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:26:03,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:26:03,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:26:03,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +21: [2023-05-08 11:26:03,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +21: [2023-05-08 11:26:03,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +18: [2023-05-08 11:26:03,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 4: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:26:03,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +21: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +21: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +23: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +21: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +21: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +21: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +18: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 3: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 4: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +21: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +21: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +21: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +21: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 4: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 4: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 4: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 4: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 4: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +21: [2023-05-08 11:26:03,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +21: [2023-05-08 11:26:03,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +21: [2023-05-08 11:26:03,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +12: [2023-05-08 11:26:03,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 4: [2023-05-08 11:26:03,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +12: [2023-05-08 11:26:03,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +12: [2023-05-08 11:26:03,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +12: [2023-05-08 11:26:03,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +21: [2023-05-08 11:26:03,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +21: [2023-05-08 11:26:03,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +23: [2023-05-08 11:26:03,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:26:03,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +11: [2023-05-08 11:26:03,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:26:03,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +12: [2023-05-08 11:26:03,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:26:03,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:26:03,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +16: [2023-05-08 11:26:03,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +16: [2023-05-08 11:26:03,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +16: [2023-05-08 11:26:03,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +24: [2023-05-08 11:26:03,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +24: [2023-05-08 11:26:03,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +24: [2023-05-08 11:26:03,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:26:03,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:26:03,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:26:03,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:26:03,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +12: [2023-05-08 11:26:03,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:26:03,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +24: [2023-05-08 11:26:03,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:26:03,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +18: [2023-05-08 11:26:03,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:26:03,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:26:03,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +21: [2023-05-08 11:26:03,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +23: [2023-05-08 11:26:03,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:26:03,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:26:03,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +18: [2023-05-08 11:26:03,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:26:03,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +17: [2023-05-08 11:26:03,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +17: [2023-05-08 11:26:03,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +17: [2023-05-08 11:26:03,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +12: [2023-05-08 11:26:03,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:26:03,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +14: [2023-05-08 11:26:03,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +14: [2023-05-08 11:26:03,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +14: [2023-05-08 11:26:03,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +24: [2023-05-08 11:26:03,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +14: [2023-05-08 11:26:03,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +24: [2023-05-08 11:26:03,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:26:03,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +11: [2023-05-08 11:26:03,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:26:03,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +21: [2023-05-08 11:26:03,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +21: [2023-05-08 11:26:03,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +22: [2023-05-08 11:26:03,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 9: [2023-05-08 11:26:03,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +10: [2023-05-08 11:26:03,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +22: [2023-05-08 11:26:03,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +22: [2023-05-08 11:26:03,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 9: [2023-05-08 11:26:03,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +10: [2023-05-08 11:26:03,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +22: [2023-05-08 11:26:03,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 9: [2023-05-08 11:26:03,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +10: [2023-05-08 11:26:03,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +10: [2023-05-08 11:26:03,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +21: [2023-05-08 11:26:03,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +16: [2023-05-08 11:26:03,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +11: [2023-05-08 11:26:03,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:26:03,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 5: [2023-05-08 11:26:03,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 5: [2023-05-08 11:26:03,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 5: [2023-05-08 11:26:03,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +18: [2023-05-08 11:26:03,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +16: [2023-05-08 11:26:03,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:26:03,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:26:03,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:26:03,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +27: [2023-05-08 11:26:03,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 0: [2023-05-08 11:26:03,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +27: [2023-05-08 11:26:03,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 8: [2023-05-08 11:26:03,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 8: [2023-05-08 11:26:03,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +27: [2023-05-08 11:26:03,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 4: [2023-05-08 11:26:03,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +27: [2023-05-08 11:26:03,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 0: [2023-05-08 11:26:03,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +11: [2023-05-08 11:26:03,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +27: [2023-05-08 11:26:03,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +27: [2023-05-08 11:26:03,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 8: [2023-05-08 11:26:03,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 4: [2023-05-08 11:26:03,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +27: [2023-05-08 11:26:03,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +27: [2023-05-08 11:26:03,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:26:03,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +12: [2023-05-08 11:26:03,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:26:03,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 6: [2023-05-08 11:26:03,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +12: [2023-05-08 11:26:03,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +17: [2023-05-08 11:26:03,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:26:03,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:26:03,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:26:03,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:26:03,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 6: [2023-05-08 11:26:03,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +14: [2023-05-08 11:26:03,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:26:03,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:26:03,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:26:03,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +31: [2023-05-08 11:26:03,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +14: [2023-05-08 11:26:03,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:26:03,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:26:03,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +31: [2023-05-08 11:26:03,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +31: [2023-05-08 11:26:03,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +25: [2023-05-08 11:26:03,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:26:03,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:26:03,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:26:03,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:26:03,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:26:03,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:26:03,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +25: [2023-05-08 11:26:03,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:26:03,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:26:03,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:26:03,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:26:03,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +21: [2023-05-08 11:26:03,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +10: [2023-05-08 11:26:03,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:26:03,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:26:03,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +27: [2023-05-08 11:26:03,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +27: [2023-05-08 11:26:03,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +27: [2023-05-08 11:26:03,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +21: [2023-05-08 11:26:03,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:26:03,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +12: [2023-05-08 11:26:03,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +12: [2023-05-08 11:26:03,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +12: [2023-05-08 11:26:03,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +12: [2023-05-08 11:26:03,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 5: [2023-05-08 11:26:03,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +21: [2023-05-08 11:26:03,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +18: [2023-05-08 11:26:03,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +18: [2023-05-08 11:26:03,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 1: [2023-05-08 11:26:03,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +27: [2023-05-08 11:26:03,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +18: [2023-05-08 11:26:03,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +18: [2023-05-08 11:26:03,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 5: [2023-05-08 11:26:03,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:26:03,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +27: [2023-05-08 11:26:03,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +27: [2023-05-08 11:26:03,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:26:03,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:26:03,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +24: [2023-05-08 11:26:03,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +27: [2023-05-08 11:26:03,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +27: [2023-05-08 11:26:03,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +24: [2023-05-08 11:26:03,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +24: [2023-05-08 11:26:03,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +31: [2023-05-08 11:26:03,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +31: [2023-05-08 11:26:03,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +27: [2023-05-08 11:26:03,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,453] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:26:03,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:26:03,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +17: [2023-05-08 11:26:03,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +17: [2023-05-08 11:26:03,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +17: [2023-05-08 11:26:03,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +21: [2023-05-08 11:26:03,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +17: [2023-05-08 11:26:03,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +17: [2023-05-08 11:26:03,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +18: [2023-05-08 11:26:03,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:26:03,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +25: [2023-05-08 11:26:03,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 9: [2023-05-08 11:26:03,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 9: [2023-05-08 11:26:03,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +17: [2023-05-08 11:26:03,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +17: [2023-05-08 11:26:03,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +18: [2023-05-08 11:26:03,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:26:03,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:26:03,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +12: [2023-05-08 11:26:03,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 5: [2023-05-08 11:26:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +24: [2023-05-08 11:26:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +27: [2023-05-08 11:26:03,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:26:03,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +31: [2023-05-08 11:26:03,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:26:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +25: [2023-05-08 11:26:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +25: [2023-05-08 11:26:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:26:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:26:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +24: [2023-05-08 11:26:03,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +27: [2023-05-08 11:26:03,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 5: [2023-05-08 11:26:03,468] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +27: [2023-05-08 11:26:03,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:26:03,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:26:03,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +31: [2023-05-08 11:26:03,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +31: [2023-05-08 11:26:03,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +24: [2023-05-08 11:26:03,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:26:03,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +22: [2023-05-08 11:26:03,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +22: [2023-05-08 11:26:03,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +22: [2023-05-08 11:26:03,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +22: [2023-05-08 11:26:03,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +22: [2023-05-08 11:26:03,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +22: [2023-05-08 11:26:03,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +22: [2023-05-08 11:26:03,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:26:03,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:26:03,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +22: [2023-05-08 11:26:03,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +22: [2023-05-08 11:26:03,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +22: [2023-05-08 11:26:03,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:26:03,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +25: [2023-05-08 11:26:03,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +25: [2023-05-08 11:26:03,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 8: [2023-05-08 11:26:03,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +22: [2023-05-08 11:26:03,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 8: [2023-05-08 11:26:03,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 8: [2023-05-08 11:26:03,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +27: [2023-05-08 11:26:03,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +27: [2023-05-08 11:26:03,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 0: [2023-05-08 11:26:03,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 0: [2023-05-08 11:26:03,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 0: [2023-05-08 11:26:03,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 0: [2023-05-08 11:26:03,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +22: [2023-05-08 11:26:03,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:26:03,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +27: [2023-05-08 11:26:03,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +27: [2023-05-08 11:26:03,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +27: [2023-05-08 11:26:03,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +27: [2023-05-08 11:26:03,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +27: [2023-05-08 11:26:03,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 1: [2023-05-08 11:26:03,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +31: [2023-05-08 11:26:03,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +27: [2023-05-08 11:26:03,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +27: [2023-05-08 11:26:03,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +27: [2023-05-08 11:26:03,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +27: [2023-05-08 11:26:03,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +31: [2023-05-08 11:26:03,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +27: [2023-05-08 11:26:03,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +30: [2023-05-08 11:26:03,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +30: [2023-05-08 11:26:03,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +30: [2023-05-08 11:26:03,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +30: [2023-05-08 11:26:03,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. + 8: [2023-05-08 11:26:03,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +22: [2023-05-08 11:26:03,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +22: [2023-05-08 11:26:03,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +13: [2023-05-08 11:26:03,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +13: [2023-05-08 11:26:03,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +15: [2023-05-08 11:26:03,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:26:03,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +15: [2023-05-08 11:26:03,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +28: [2023-05-08 11:26:03,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +28: [2023-05-08 11:26:03,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +28: [2023-05-08 11:26:03,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +15: [2023-05-08 11:26:03,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +15: [2023-05-08 11:26:03,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +28: [2023-05-08 11:26:03,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +13: [2023-05-08 11:26:03,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt. +15: [2023-05-08 11:26:03,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +15: [2023-05-08 11:26:03,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +15: [2023-05-08 11:26:03,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +15: [2023-05-08 11:26:03,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:26:03,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +13: [2023-05-08 11:26:03,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +13: [2023-05-08 11:26:03,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:26:03,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +30: [2023-05-08 11:26:03,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:26:03,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +10: [2023-05-08 11:26:03,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:26:03,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +10: [2023-05-08 11:26:03,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +10: [2023-05-08 11:26:03,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +10: [2023-05-08 11:26:03,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +10: [2023-05-08 11:26:03,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +10: [2023-05-08 11:26:03,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +30: [2023-05-08 11:26:03,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:26:03,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +10: [2023-05-08 11:26:03,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +30: [2023-05-08 11:26:03,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:26:03,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +10: [2023-05-08 11:26:03,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +13: [2023-05-08 11:26:03,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +13: [2023-05-08 11:26:03,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +10: [2023-05-08 11:26:03,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +13: [2023-05-08 11:26:03,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +10: [2023-05-08 11:26:03,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +10: [2023-05-08 11:26:03,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:26:03,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +28: [2023-05-08 11:26:03,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:26:03,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +13: [2023-05-08 11:26:03,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +13: [2023-05-08 11:26:03,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:26:03,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +13: [2023-05-08 11:26:03,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:26:03,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +10: [2023-05-08 11:26:03,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +20: [2023-05-08 11:26:03,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +20: [2023-05-08 11:26:03,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +20: [2023-05-08 11:26:03,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +20: [2023-05-08 11:26:03,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:26:03,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:26:03,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:26:03,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +20: [2023-05-08 11:26:03,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:26:03,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:26:03,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:26:03,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +13: [2023-05-08 11:26:03,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:26:03,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:26:03,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:26:03,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 3: [2023-05-08 11:26:03,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +16: [2023-05-08 11:26:03,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:26:03,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +14: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 6: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +16: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +14: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +14: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +14: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 6: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +14: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +14: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +14: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +14: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +14: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +14: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +14: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 3: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +14: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +28: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:26:03,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:26:03,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +28: [2023-05-08 11:26:03,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:26:03,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +16: [2023-05-08 11:26:03,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +14: [2023-05-08 11:26:03,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +16: [2023-05-08 11:26:03,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 0: [2023-05-08 11:26:03,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:26:03,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +30: [2023-05-08 11:26:03,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +30: [2023-05-08 11:26:03,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +30: [2023-05-08 11:26:03,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +30: [2023-05-08 11:26:03,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +22: [2023-05-08 11:26:03,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +14: [2023-05-08 11:26:03,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:26:03,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +15: [2023-05-08 11:26:03,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:26:03,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +10: [2023-05-08 11:26:03,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +30: [2023-05-08 11:26:03,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +13: [2023-05-08 11:26:03,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +30: [2023-05-08 11:26:03,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +30: [2023-05-08 11:26:03,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +30: [2023-05-08 11:26:03,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:26:03,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +28: [2023-05-08 11:26:03,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +28: [2023-05-08 11:26:03,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +28: [2023-05-08 11:26:03,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +16: [2023-05-08 11:26:03,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +10: [2023-05-08 11:26:03,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +30: [2023-05-08 11:26:03,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +30: [2023-05-08 11:26:03,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt... +15: [2023-05-08 11:26:03,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +15: [2023-05-08 11:26:03,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +30: [2023-05-08 11:26:03,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:26:03,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:26:03,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:26:03,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:26:03,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:26:03,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:26:03,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +13: [2023-05-08 11:26:03,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:26:03,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:26:03,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +30: [2023-05-08 11:26:03,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +30: [2023-05-08 11:26:03,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +30: [2023-05-08 11:26:03,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +30: [2023-05-08 11:26:03,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 3: [2023-05-08 11:26:03,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +13: [2023-05-08 11:26:03,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 9: [2023-05-08 11:26:03,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:26:03,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:26:03,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:26:03,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +26: [2023-05-08 11:26:03,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +10: [2023-05-08 11:26:03,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +10: [2023-05-08 11:26:03,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +15: [2023-05-08 11:26:03,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:26:03,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:26:03,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 1: [2023-05-08 11:26:03,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 1: [2023-05-08 11:26:03,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 1: [2023-05-08 11:26:03,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +15: [2023-05-08 11:26:03,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +14: [2023-05-08 11:26:03,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +14: [2023-05-08 11:26:03,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:26:03,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +30: [2023-05-08 11:26:03,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:26:03,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:26:03,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:26:03,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +15: [2023-05-08 11:26:03,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:26:03,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:26:03,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +13: [2023-05-08 11:26:03,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +28: [2023-05-08 11:26:03,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +10: [2023-05-08 11:26:03,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:26:03,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +28: [2023-05-08 11:26:03,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +28: [2023-05-08 11:26:03,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +28: [2023-05-08 11:26:03,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... + 3: [2023-05-08 11:26:03,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:26:03,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +20: [2023-05-08 11:26:03,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +20: [2023-05-08 11:26:03,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +20: [2023-05-08 11:26:03,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +10: [2023-05-08 11:26:03,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:26:03,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:26:03,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:26:03,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:26:03,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:26:03,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +28: [2023-05-08 11:26:03,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +28: [2023-05-08 11:26:03,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +29: [2023-05-08 11:26:03,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +29: [2023-05-08 11:26:03,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +29: [2023-05-08 11:26:03,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:26:03,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:26:03,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +28: [2023-05-08 11:26:03,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +28: [2023-05-08 11:26:03,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +29: [2023-05-08 11:26:03,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:26:03,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:26:03,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:26:03,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +16: [2023-05-08 11:26:03,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +16: [2023-05-08 11:26:03,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:26:03,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +28: [2023-05-08 11:26:03,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:26:03,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +16: [2023-05-08 11:26:03,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +14: [2023-05-08 11:26:03,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +28: [2023-05-08 11:26:03,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:26:03,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +25: [2023-05-08 11:26:03,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +25: [2023-05-08 11:26:03,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +16: [2023-05-08 11:26:03,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:26:03,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:26:03,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:26:03,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +14: [2023-05-08 11:26:03,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +20: [2023-05-08 11:26:03,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:26:03,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:26:03,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:26:03,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +10: [2023-05-08 11:26:03,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:26:03,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:26:03,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:26:03,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:26:03,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +28: [2023-05-08 11:26:03,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +26: [2023-05-08 11:26:03,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:26:03,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +13: [2023-05-08 11:26:03,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 3: [2023-05-08 11:26:03,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:26:03,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:26:03,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +28: [2023-05-08 11:26:03,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +14: [2023-05-08 11:26:03,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:26:03,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +13: [2023-05-08 11:26:03,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:26:03,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:26:03,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:26:03,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:26:03,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:26:03,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:26:03,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:26:03,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:26:03,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 1: [2023-05-08 11:26:03,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +30: [2023-05-08 11:26:03,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:26:03,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +19: [2023-05-08 11:26:03,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:26:03,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +19: [2023-05-08 11:26:03,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +19: [2023-05-08 11:26:03,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +19: [2023-05-08 11:26:03,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:26:03,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:26:03,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:26:03,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:26:03,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:26:03,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:26:03,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +28: [2023-05-08 11:26:03,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt. +12: [2023-05-08 11:26:03,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +14: [2023-05-08 11:26:03,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:26:03,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +28: [2023-05-08 11:26:03,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt... +28: [2023-05-08 11:26:03,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:26:03,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +29: [2023-05-08 11:26:03,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +29: [2023-05-08 11:26:03,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +31: [2023-05-08 11:26:03,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +31: [2023-05-08 11:26:03,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +31: [2023-05-08 11:26:03,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:26:03,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +31: [2023-05-08 11:26:03,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +31: [2023-05-08 11:26:03,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:26:03,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +31: [2023-05-08 11:26:03,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:26:03,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:26:03,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:26:03,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:26:03,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:26:03,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +25: [2023-05-08 11:26:03,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +29: [2023-05-08 11:26:03,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:26:03,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +19: [2023-05-08 11:26:03,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +29: [2023-05-08 11:26:03,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:26:03,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +29: [2023-05-08 11:26:03,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:26:03,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +29: [2023-05-08 11:26:03,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:26:03,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +31: [2023-05-08 11:26:03,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:26:03,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:26:03,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +31: [2023-05-08 11:26:03,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +19: [2023-05-08 11:26:03,578] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:26:03,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:26:03,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +29: [2023-05-08 11:26:03,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +25: [2023-05-08 11:26:03,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +19: [2023-05-08 11:26:03,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:26:03,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:26:03,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:26:03,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +19: [2023-05-08 11:26:03,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +19: [2023-05-08 11:26:03,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +31: [2023-05-08 11:26:03,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +31: [2023-05-08 11:26:03,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:26:03,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +12: [2023-05-08 11:26:03,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:26:03,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +31: [2023-05-08 11:26:03,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +31: [2023-05-08 11:26:03,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:26:03,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +19: [2023-05-08 11:26:03,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:26:03,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:26:03,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:26:03,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:26:03,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +19: [2023-05-08 11:26:03,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +31: [2023-05-08 11:26:03,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:26:03,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +15: [2023-05-08 11:26:03,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +15: [2023-05-08 11:26:03,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +15: [2023-05-08 11:26:03,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +15: [2023-05-08 11:26:03,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +21: [2023-05-08 11:26:03,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:26:03,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:26:03,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +21: [2023-05-08 11:26:03,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +21: [2023-05-08 11:26:03,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:26:03,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:26:03,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +21: [2023-05-08 11:26:03,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:26:03,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +20: [2023-05-08 11:26:03,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +20: [2023-05-08 11:26:03,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +20: [2023-05-08 11:26:03,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +15: [2023-05-08 11:26:03,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +15: [2023-05-08 11:26:03,614] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:26:03,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +26: [2023-05-08 11:26:03,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +26: [2023-05-08 11:26:03,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +26: [2023-05-08 11:26:03,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 2: [2023-05-08 11:26:03,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 2: [2023-05-08 11:26:03,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 2: [2023-05-08 11:26:03,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 2: [2023-05-08 11:26:03,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 7: [2023-05-08 11:26:03,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 7: [2023-05-08 11:26:03,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 7: [2023-05-08 11:26:03,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +15: [2023-05-08 11:26:03,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +17: [2023-05-08 11:26:03,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:26:03,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:26:03,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +17: [2023-05-08 11:26:03,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +17: [2023-05-08 11:26:03,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:26:03,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:26:03,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +19: [2023-05-08 11:26:03,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +17: [2023-05-08 11:26:03,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:26:03,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:26:03,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:26:03,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +27: [2023-05-08 11:26:03,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +27: [2023-05-08 11:26:03,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +27: [2023-05-08 11:26:03,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +27: [2023-05-08 11:26:03,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +15: [2023-05-08 11:26:03,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +18: [2023-05-08 11:26:03,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +18: [2023-05-08 11:26:03,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +18: [2023-05-08 11:26:03,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +18: [2023-05-08 11:26:03,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +27: [2023-05-08 11:26:03,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +27: [2023-05-08 11:26:03,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:26:03,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +27: [2023-05-08 11:26:03,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +21: [2023-05-08 11:26:03,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:26:03,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:26:03,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +21: [2023-05-08 11:26:03,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +21: [2023-05-08 11:26:03,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +24: [2023-05-08 11:26:03,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +24: [2023-05-08 11:26:03,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +24: [2023-05-08 11:26:03,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +24: [2023-05-08 11:26:03,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +24: [2023-05-08 11:26:03,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +24: [2023-05-08 11:26:03,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:26:03,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:26:03,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:26:03,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:26:03,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:26:03,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:26:03,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:26:03,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:26:03,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:26:03,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +15: [2023-05-08 11:26:03,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +15: [2023-05-08 11:26:03,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 7: [2023-05-08 11:26:03,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +21: [2023-05-08 11:26:03,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +19: [2023-05-08 11:26:03,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:26:03,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 5: [2023-05-08 11:26:03,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +17: [2023-05-08 11:26:03,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:26:03,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +18: [2023-05-08 11:26:03,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +18: [2023-05-08 11:26:03,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +15: [2023-05-08 11:26:03,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +19: [2023-05-08 11:26:03,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +15: [2023-05-08 11:26:03,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 5: [2023-05-08 11:26:03,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:26:03,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +15: [2023-05-08 11:26:03,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +15: [2023-05-08 11:26:03,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 4: [2023-05-08 11:26:03,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:26:03,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:26:03,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:26:03,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:26:03,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:26:03,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:26:03,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:26:03,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +23: [2023-05-08 11:26:03,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:26:03,632] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:26:03,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:26:03,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +25: [2023-05-08 11:26:03,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:26:03,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +23: [2023-05-08 11:26:03,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +24: [2023-05-08 11:26:03,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +25: [2023-05-08 11:26:03,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +25: [2023-05-08 11:26:03,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:26:03,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +19: [2023-05-08 11:26:03,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:26:03,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:26:03,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +27: [2023-05-08 11:26:03,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:26:03,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +17: [2023-05-08 11:26:03,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +11: [2023-05-08 11:26:03,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +11: [2023-05-08 11:26:03,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +17: [2023-05-08 11:26:03,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +17: [2023-05-08 11:26:03,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:26:03,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:26:03,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 4: [2023-05-08 11:26:03,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +17: [2023-05-08 11:26:03,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +21: [2023-05-08 11:26:03,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:26:03,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +17: [2023-05-08 11:26:03,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 9: [2023-05-08 11:26:03,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +24: [2023-05-08 11:26:03,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +24: [2023-05-08 11:26:03,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 9: [2023-05-08 11:26:03,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +24: [2023-05-08 11:26:03,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 4: [2023-05-08 11:26:03,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 4: [2023-05-08 11:26:03,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +19: [2023-05-08 11:26:03,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 9: [2023-05-08 11:26:03,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 9: [2023-05-08 11:26:03,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 4: [2023-05-08 11:26:03,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +25: [2023-05-08 11:26:03,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:26:03,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:26:03,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:26:03,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +26: [2023-05-08 11:26:03,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +23: [2023-05-08 11:26:03,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 5: [2023-05-08 11:26:03,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:26:03,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:26:03,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +23: [2023-05-08 11:26:03,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +17: [2023-05-08 11:26:03,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +21: [2023-05-08 11:26:03,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:26:03,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +15: [2023-05-08 11:26:03,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:26:03,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +29: [2023-05-08 11:26:03,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +29: [2023-05-08 11:26:03,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +29: [2023-05-08 11:26:03,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +26: [2023-05-08 11:26:03,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:26:03,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +15: [2023-05-08 11:26:03,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +21: [2023-05-08 11:26:03,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +24: [2023-05-08 11:26:03,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:26:03,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:26:03,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:26:03,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 7: [2023-05-08 11:26:03,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:26:03,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:26:03,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +17: [2023-05-08 11:26:03,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +27: [2023-05-08 11:26:03,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 4: [2023-05-08 11:26:03,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +27: [2023-05-08 11:26:03,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +27: [2023-05-08 11:26:03,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +25: [2023-05-08 11:26:03,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +20: [2023-05-08 11:26:03,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +20: [2023-05-08 11:26:03,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +20: [2023-05-08 11:26:03,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +20: [2023-05-08 11:26:03,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 2: [2023-05-08 11:26:03,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +20: [2023-05-08 11:26:03,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +20: [2023-05-08 11:26:03,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 9: [2023-05-08 11:26:03,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +12: [2023-05-08 11:26:03,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +21: [2023-05-08 11:26:03,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:26:03,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +17: [2023-05-08 11:26:03,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:26:03,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +24: [2023-05-08 11:26:03,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:26:03,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 2: [2023-05-08 11:26:03,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:26:03,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +17: [2023-05-08 11:26:03,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:26:03,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +18: [2023-05-08 11:26:03,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:26:03,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:26:03,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:26:03,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:26:03,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +24: [2023-05-08 11:26:03,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +18: [2023-05-08 11:26:03,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:26:03,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:26:03,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +18: [2023-05-08 11:26:03,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +18: [2023-05-08 11:26:03,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:26:03,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:26:03,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:26:03,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +18: [2023-05-08 11:26:03,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 9: [2023-05-08 11:26:03,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +18: [2023-05-08 11:26:03,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 4: [2023-05-08 11:26:03,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:26:03,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +18: [2023-05-08 11:26:03,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +17: [2023-05-08 11:26:03,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +17: [2023-05-08 11:26:03,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +27: [2023-05-08 11:26:03,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:26:03,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:26:03,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +24: [2023-05-08 11:26:03,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:26:03,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +29: [2023-05-08 11:26:03,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 2: [2023-05-08 11:26:03,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:26:03,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:26:03,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +29: [2023-05-08 11:26:03,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +26: [2023-05-08 11:26:03,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +23: [2023-05-08 11:26:03,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:26:03,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +23: [2023-05-08 11:26:03,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +29: [2023-05-08 11:26:03,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 2: [2023-05-08 11:26:03,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:26:03,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:26:03,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:26:03,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +27: [2023-05-08 11:26:03,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +11: [2023-05-08 11:26:03,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +20: [2023-05-08 11:26:03,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +25: [2023-05-08 11:26:03,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +11: [2023-05-08 11:26:03,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +23: [2023-05-08 11:26:03,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:26:03,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +11: [2023-05-08 11:26:03,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +12: [2023-05-08 11:26:03,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:26:03,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +12: [2023-05-08 11:26:03,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +24: [2023-05-08 11:26:03,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:26:03,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:26:03,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:26:03,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 7: [2023-05-08 11:26:03,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:26:03,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:26:03,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +23: [2023-05-08 11:26:03,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:26:03,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +25: [2023-05-08 11:26:03,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:26:03,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +24: [2023-05-08 11:26:03,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 7: [2023-05-08 11:26:03,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 5: [2023-05-08 11:26:03,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 4: [2023-05-08 11:26:03,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:26:03,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 7: [2023-05-08 11:26:03,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +18: [2023-05-08 11:26:03,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:26:03,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:26:03,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:26:03,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:26:03,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +11: [2023-05-08 11:26:03,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:26:03,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +21: [2023-05-08 11:26:03,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:26:03,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:26:03,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +21: [2023-05-08 11:26:03,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +21: [2023-05-08 11:26:03,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +21: [2023-05-08 11:26:03,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 7: [2023-05-08 11:26:03,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 9: [2023-05-08 11:26:03,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 9: [2023-05-08 11:26:03,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +29: [2023-05-08 11:26:03,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +29: [2023-05-08 11:26:03,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +24: [2023-05-08 11:26:03,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +24: [2023-05-08 11:26:03,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 7: [2023-05-08 11:26:03,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:26:03,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:26:03,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +25: [2023-05-08 11:26:03,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +18: [2023-05-08 11:26:03,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:26:03,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:26:03,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:26:03,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +29: [2023-05-08 11:26:03,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +29: [2023-05-08 11:26:03,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +29: [2023-05-08 11:26:03,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +29: [2023-05-08 11:26:03,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +23: [2023-05-08 11:26:03,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:26:03,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:26:03,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +24: [2023-05-08 11:26:03,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:26:03,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +22: [2023-05-08 11:26:03,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +22: [2023-05-08 11:26:03,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +22: [2023-05-08 11:26:03,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +22: [2023-05-08 11:26:03,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +25: [2023-05-08 11:26:03,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:26:03,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:26:03,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +22: [2023-05-08 11:26:03,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +17: [2023-05-08 11:26:03,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +17: [2023-05-08 11:26:03,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +22: [2023-05-08 11:26:03,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +17: [2023-05-08 11:26:03,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +22: [2023-05-08 11:26:03,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +17: [2023-05-08 11:26:03,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +24: [2023-05-08 11:26:03,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +24: [2023-05-08 11:26:03,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +24: [2023-05-08 11:26:03,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +24: [2023-05-08 11:26:03,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +24: [2023-05-08 11:26:03,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +24: [2023-05-08 11:26:03,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +24: [2023-05-08 11:26:03,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:26:03,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:26:03,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:26:03,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +29: [2023-05-08 11:26:03,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:26:03,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +12: [2023-05-08 11:26:03,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +12: [2023-05-08 11:26:03,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 4: [2023-05-08 11:26:03,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 4: [2023-05-08 11:26:03,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 8: [2023-05-08 11:26:03,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,682] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +25: [2023-05-08 11:26:03,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:26:03,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +23: [2023-05-08 11:26:03,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:26:03,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +23: [2023-05-08 11:26:03,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +12: [2023-05-08 11:26:03,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:26:03,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +23: [2023-05-08 11:26:03,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +18: [2023-05-08 11:26:03,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:26:03,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:26:03,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:26:03,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:26:03,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:26:03,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:26:03,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:26:03,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +11: [2023-05-08 11:26:03,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:26:03,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:26:03,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +24: [2023-05-08 11:26:03,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:26:03,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +13: [2023-05-08 11:26:03,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:26:03,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:26:03,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:26:03,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:26:03,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:26:03,688] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:26:03,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:26:03,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:26:03,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:26:03,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +22: [2023-05-08 11:26:03,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +11: [2023-05-08 11:26:03,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:26:03,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:26:03,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +15: [2023-05-08 11:26:03,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +15: [2023-05-08 11:26:03,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +15: [2023-05-08 11:26:03,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +15: [2023-05-08 11:26:03,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +15: [2023-05-08 11:26:03,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +15: [2023-05-08 11:26:03,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +15: [2023-05-08 11:26:03,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:26:03,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:26:03,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:26:03,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:26:03,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +15: [2023-05-08 11:26:03,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:26:03,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:26:03,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:26:03,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:26:03,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +10: [2023-05-08 11:26:03,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:26:03,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:26:03,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +10: [2023-05-08 11:26:03,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:26:03,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:26:03,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:26:03,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:26:03,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +11: [2023-05-08 11:26:03,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +11: [2023-05-08 11:26:03,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +11: [2023-05-08 11:26:03,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:26:03,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +15: [2023-05-08 11:26:03,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:26:03,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +22: [2023-05-08 11:26:03,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:26:03,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:26:03,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +10: [2023-05-08 11:26:03,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:26:03,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:26:03,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:26:03,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +15: [2023-05-08 11:26:03,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:26:03,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:26:03,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:26:03,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:26:03,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:26:03,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:26:03,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:26:03,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:26:03,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:26:03,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:26:03,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +15: [2023-05-08 11:26:03,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:26:03,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:26:03,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +15: [2023-05-08 11:26:03,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +15: [2023-05-08 11:26:03,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:26:03,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:26:03,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:26:03,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +10: [2023-05-08 11:26:03,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:26:03,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:26:03,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:26:03,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:26:03,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +30: [2023-05-08 11:26:03,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +10: [2023-05-08 11:26:03,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:26:03,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +15: [2023-05-08 11:26:03,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +30: [2023-05-08 11:26:03,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:26:03,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:26:03,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:26:03,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:26:03,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:26:03,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +28: [2023-05-08 11:26:03,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:26:03,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:26:03,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +28: [2023-05-08 11:26:03,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +28: [2023-05-08 11:26:03,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:26:03,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:26:03,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +26: [2023-05-08 11:26:03,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +15: [2023-05-08 11:26:03,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:26:03,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +30: [2023-05-08 11:26:03,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:26:03,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +26: [2023-05-08 11:26:03,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:26:03,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +16: [2023-05-08 11:26:03,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +16: [2023-05-08 11:26:03,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +16: [2023-05-08 11:26:03,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:26:03,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:26:03,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:26:03,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +16: [2023-05-08 11:26:03,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +16: [2023-05-08 11:26:03,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +16: [2023-05-08 11:26:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +16: [2023-05-08 11:26:03,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +16: [2023-05-08 11:26:03,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:26:03,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +30: [2023-05-08 11:26:03,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:26:03,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +31: [2023-05-08 11:26:03,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +28: [2023-05-08 11:26:03,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:26:03,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:26:03,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +28: [2023-05-08 11:26:03,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 8: [2023-05-08 11:26:03,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +30: [2023-05-08 11:26:03,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 8: [2023-05-08 11:26:03,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +31: [2023-05-08 11:26:03,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +30: [2023-05-08 11:26:03,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +30: [2023-05-08 11:26:03,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +30: [2023-05-08 11:26:03,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +31: [2023-05-08 11:26:03,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +16: [2023-05-08 11:26:03,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 0: [2023-05-08 11:26:03,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 0: [2023-05-08 11:26:03,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 0: [2023-05-08 11:26:03,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +28: [2023-05-08 11:26:03,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:26:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:26:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:26:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:26:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +14: [2023-05-08 11:26:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +14: [2023-05-08 11:26:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:26:03,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +14: [2023-05-08 11:26:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:26:03,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:26:03,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:26:03,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:26:03,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:26:03,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +31: [2023-05-08 11:26:03,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:26:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:26:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:26:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:26:03,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +28: [2023-05-08 11:26:03,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +16: [2023-05-08 11:26:03,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +30: [2023-05-08 11:26:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +16: [2023-05-08 11:26:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:26:03,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:26:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +16: [2023-05-08 11:26:03,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +14: [2023-05-08 11:26:03,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 5: [2023-05-08 11:26:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +27: [2023-05-08 11:26:03,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 6: [2023-05-08 11:26:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 5: [2023-05-08 11:26:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 5: [2023-05-08 11:26:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 6: [2023-05-08 11:26:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 6: [2023-05-08 11:26:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +31: [2023-05-08 11:26:03,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:26:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +13: [2023-05-08 11:26:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +16: [2023-05-08 11:26:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +31: [2023-05-08 11:26:03,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +27: [2023-05-08 11:26:03,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 6: [2023-05-08 11:26:03,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +31: [2023-05-08 11:26:03,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +31: [2023-05-08 11:26:03,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +27: [2023-05-08 11:26:03,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +27: [2023-05-08 11:26:03,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +13: [2023-05-08 11:26:03,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +16: [2023-05-08 11:26:03,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:26:03,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 8: [2023-05-08 11:26:03,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:26:03,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +30: [2023-05-08 11:26:03,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:26:03,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +16: [2023-05-08 11:26:03,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:26:03,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +27: [2023-05-08 11:26:03,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +14: [2023-05-08 11:26:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +28: [2023-05-08 11:26:03,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +28: [2023-05-08 11:26:03,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +28: [2023-05-08 11:26:03,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 5: [2023-05-08 11:26:03,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:26:03,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +20: [2023-05-08 11:26:03,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 3: [2023-05-08 11:26:03,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +16: [2023-05-08 11:26:03,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:26:03,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:26:03,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +28: [2023-05-08 11:26:03,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +20: [2023-05-08 11:26:03,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:26:03,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +20: [2023-05-08 11:26:03,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:26:03,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +31: [2023-05-08 11:26:03,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +22: [2023-05-08 11:26:03,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +22: [2023-05-08 11:26:03,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +22: [2023-05-08 11:26:03,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 5: [2023-05-08 11:26:03,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:26:03,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +30: [2023-05-08 11:26:03,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:26:03,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +14: [2023-05-08 11:26:03,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 0: [2023-05-08 11:26:03,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 0: [2023-05-08 11:26:03,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:26:03,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +30: [2023-05-08 11:26:03,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:26:03,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +30: [2023-05-08 11:26:03,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +30: [2023-05-08 11:26:03,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 8: [2023-05-08 11:26:03,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +10: [2023-05-08 11:26:03,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +10: [2023-05-08 11:26:03,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +14: [2023-05-08 11:26:03,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +20: [2023-05-08 11:26:03,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 0: [2023-05-08 11:26:03,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +14: [2023-05-08 11:26:03,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +20: [2023-05-08 11:26:03,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 8: [2023-05-08 11:26:03,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +13: [2023-05-08 11:26:03,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +10: [2023-05-08 11:26:03,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +14: [2023-05-08 11:26:03,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 6: [2023-05-08 11:26:03,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +31: [2023-05-08 11:26:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +27: [2023-05-08 11:26:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +27: [2023-05-08 11:26:03,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +31: [2023-05-08 11:26:03,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 3: [2023-05-08 11:26:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +16: [2023-05-08 11:26:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +16: [2023-05-08 11:26:03,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +16: [2023-05-08 11:26:03,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. +13: [2023-05-08 11:26:03,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +16: [2023-05-08 11:26:03,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt. + 8: [2023-05-08 11:26:03,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +28: [2023-05-08 11:26:03,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:26:03,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +13: [2023-05-08 11:26:03,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +28: [2023-05-08 11:26:03,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +28: [2023-05-08 11:26:03,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:26:03,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:26:03,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:26:03,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:26:03,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:26:03,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:26:03,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +14: [2023-05-08 11:26:03,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +14: [2023-05-08 11:26:03,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +22: [2023-05-08 11:26:03,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 3: [2023-05-08 11:26:03,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 0: [2023-05-08 11:26:03,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +10: [2023-05-08 11:26:03,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +14: [2023-05-08 11:26:03,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +14: [2023-05-08 11:26:03,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +10: [2023-05-08 11:26:03,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +10: [2023-05-08 11:26:03,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +27: [2023-05-08 11:26:03,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:26:03,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 9: [2023-05-08 11:26:03,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +27: [2023-05-08 11:26:03,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +13: [2023-05-08 11:26:03,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 6: [2023-05-08 11:26:03,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +16: [2023-05-08 11:26:03,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +16: [2023-05-08 11:26:03,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 6: [2023-05-08 11:26:03,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 5: [2023-05-08 11:26:03,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +16: [2023-05-08 11:26:03,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... +16: [2023-05-08 11:26:03,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 5: [2023-05-08 11:26:03,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 9: [2023-05-08 11:26:03,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +13: [2023-05-08 11:26:03,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:26:03,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 0: [2023-05-08 11:26:03,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:26:03,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:26:03,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +20: [2023-05-08 11:26:03,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +20: [2023-05-08 11:26:03,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:26:03,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +27: [2023-05-08 11:26:03,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:26:03,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +27: [2023-05-08 11:26:03,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 0: [2023-05-08 11:26:03,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 5: [2023-05-08 11:26:03,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:26:03,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +12: [2023-05-08 11:26:03,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:26:03,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:26:03,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +10: [2023-05-08 11:26:03,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +10: [2023-05-08 11:26:03,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +13: [2023-05-08 11:26:03,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:26:03,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +12: [2023-05-08 11:26:03,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:26:03,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:26:03,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:26:03,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +27: [2023-05-08 11:26:03,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +12: [2023-05-08 11:26:03,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:26:03,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:26:03,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +27: [2023-05-08 11:26:03,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +14: [2023-05-08 11:26:03,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:26:03,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +22: [2023-05-08 11:26:03,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +22: [2023-05-08 11:26:03,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +22: [2023-05-08 11:26:03,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +22: [2023-05-08 11:26:03,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 6: [2023-05-08 11:26:03,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 6: [2023-05-08 11:26:03,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 6: [2023-05-08 11:26:03,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +28: [2023-05-08 11:26:03,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:26:03,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +28: [2023-05-08 11:26:03,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:26:03,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +10: [2023-05-08 11:26:03,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:26:03,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +16: [2023-05-08 11:26:03,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:26:03,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:26:03,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +16: [2023-05-08 11:26:03,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 3: [2023-05-08 11:26:03,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +27: [2023-05-08 11:26:03,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +27: [2023-05-08 11:26:03,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 3: [2023-05-08 11:26:03,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:26:03,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:26:03,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +14: [2023-05-08 11:26:03,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +14: [2023-05-08 11:26:03,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 3: [2023-05-08 11:26:03,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... + 3: [2023-05-08 11:26:03,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +20: [2023-05-08 11:26:03,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:26:03,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:26:03,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +22: [2023-05-08 11:26:03,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +28: [2023-05-08 11:26:03,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +28: [2023-05-08 11:26:03,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +16: [2023-05-08 11:26:03,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +16: [2023-05-08 11:26:03,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +16: [2023-05-08 11:26:03,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +16: [2023-05-08 11:26:03,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +12: [2023-05-08 11:26:03,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,837] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +10: [2023-05-08 11:26:03,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +22: [2023-05-08 11:26:03,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +10: [2023-05-08 11:26:03,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +20: [2023-05-08 11:26:03,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:26:03,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. +16: [2023-05-08 11:26:03,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +22: [2023-05-08 11:26:03,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +20: [2023-05-08 11:26:03,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt. + 3: [2023-05-08 11:26:03,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt... +12: [2023-05-08 11:26:03,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:26:03,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:26:03,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:26:03,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +20: [2023-05-08 11:26:03,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:26:03,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:26:03,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:26:03,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:26:03,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:26:03,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +19: [2023-05-08 11:26:03,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +19: [2023-05-08 11:26:03,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +19: [2023-05-08 11:26:03,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +19: [2023-05-08 11:26:03,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:26:03,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:26:03,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:26:03,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:26:03,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:26:03,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:26:03,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:26:03,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:26:03,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:26:03,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:26:03,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:26:03,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +21: [2023-05-08 11:26:03,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:26:03,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:26:03,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:26:03,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +21: [2023-05-08 11:26:03,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +21: [2023-05-08 11:26:03,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +21: [2023-05-08 11:26:03,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:26:03,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +29: [2023-05-08 11:26:03,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:26:03,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +29: [2023-05-08 11:26:03,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:26:03,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:26:03,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:26:03,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +29: [2023-05-08 11:26:03,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:26:03,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:26:03,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:26:03,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:26:03,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:26:03,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:26:03,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:26:03,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:26:03,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:26:03,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:26:03,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:26:03,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:26:03,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:26:03,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:26:03,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:26:03,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:26:03,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +21: [2023-05-08 11:26:03,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +21: [2023-05-08 11:26:03,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:26:03,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:26:03,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:26:03,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:26:03,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:26:03,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:26:03,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +17: [2023-05-08 11:26:03,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:26:03,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +17: [2023-05-08 11:26:03,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:26:03,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:26:03,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:26:03,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:26:03,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:26:03,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:26:03,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:26:03,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:26:03,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:26:03,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:26:03,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:26:03,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +19: [2023-05-08 11:26:03,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +29: [2023-05-08 11:26:03,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:26:03,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:26:03,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:26:03,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:26:03,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +25: [2023-05-08 11:26:03,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +25: [2023-05-08 11:26:03,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +25: [2023-05-08 11:26:03,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +19: [2023-05-08 11:26:03,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +17: [2023-05-08 11:26:03,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +21: [2023-05-08 11:26:03,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:26:03,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:26:03,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:26:03,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:26:03,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:26:03,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +29: [2023-05-08 11:26:03,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:26:03,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:26:03,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:26:03,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +21: [2023-05-08 11:26:03,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 1: [2023-05-08 11:26:03,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 1: [2023-05-08 11:26:03,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +25: [2023-05-08 11:26:03,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +19: [2023-05-08 11:26:03,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:26:03,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:26:03,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:26:03,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +29: [2023-05-08 11:26:03,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:26:03,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +25: [2023-05-08 11:26:03,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +17: [2023-05-08 11:26:03,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:26:03,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:26:03,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:26:03,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +15: [2023-05-08 11:26:03,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:26:03,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:26:03,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:26:03,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:26:03,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +13: [2023-05-08 11:26:03,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +15: [2023-05-08 11:26:03,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:26:03,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:26:03,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +19: [2023-05-08 11:26:03,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:26:03,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:26:03,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:26:03,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +26: [2023-05-08 11:26:03,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +17: [2023-05-08 11:26:03,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:26:03,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +19: [2023-05-08 11:26:03,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:26:03,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:26:03,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +15: [2023-05-08 11:26:03,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +15: [2023-05-08 11:26:03,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +26: [2023-05-08 11:26:03,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +19: [2023-05-08 11:26:03,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:26:03,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:26:03,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:26:03,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:26:03,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:26:03,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:26:03,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:26:03,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:26:03,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +20: [2023-05-08 11:26:03,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +20: [2023-05-08 11:26:03,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +22: [2023-05-08 11:26:03,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +25: [2023-05-08 11:26:03,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:26:03,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +21: [2023-05-08 11:26:03,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +21: [2023-05-08 11:26:03,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +13: [2023-05-08 11:26:03,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +13: [2023-05-08 11:26:03,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +15: [2023-05-08 11:26:03,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +25: [2023-05-08 11:26:03,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:26:03,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +27: [2023-05-08 11:26:03,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:26:03,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:26:03,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +25: [2023-05-08 11:26:03,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:26:03,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +25: [2023-05-08 11:26:03,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +26: [2023-05-08 11:26:03,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:26:03,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +25: [2023-05-08 11:26:03,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:26:03,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +25: [2023-05-08 11:26:03,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +21: [2023-05-08 11:26:03,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +21: [2023-05-08 11:26:03,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +27: [2023-05-08 11:26:03,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:26:03,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:26:03,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +17: [2023-05-08 11:26:03,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 4: [2023-05-08 11:26:03,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +26: [2023-05-08 11:26:03,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:26:03,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +18: [2023-05-08 11:26:03,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 1: [2023-05-08 11:26:03,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +15: [2023-05-08 11:26:03,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +15: [2023-05-08 11:26:03,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:26:03,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +26: [2023-05-08 11:26:03,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 1: [2023-05-08 11:26:03,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +15: [2023-05-08 11:26:03,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:26:03,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +27: [2023-05-08 11:26:03,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:26:03,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:26:03,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +27: [2023-05-08 11:26:03,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +27: [2023-05-08 11:26:03,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:26:03,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +15: [2023-05-08 11:26:03,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +20: [2023-05-08 11:26:03,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:26:03,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +21: [2023-05-08 11:26:03,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:26:03,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:26:03,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:26:03,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +27: [2023-05-08 11:26:03,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:26:03,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +21: [2023-05-08 11:26:03,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +21: [2023-05-08 11:26:03,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:26:03,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:26:03,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:26:03,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:26:03,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:26:03,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +27: [2023-05-08 11:26:03,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:26:03,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:26:03,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +18: [2023-05-08 11:26:03,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:26:03,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:26:03,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:26:03,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:26:03,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:26:03,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:26:03,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:26:03,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:26:03,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:26:03,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:26:03,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:26:03,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +26: [2023-05-08 11:26:03,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:26:03,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 7: [2023-05-08 11:26:03,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 7: [2023-05-08 11:26:03,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +26: [2023-05-08 11:26:03,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:26:03,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +15: [2023-05-08 11:26:03,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +15: [2023-05-08 11:26:03,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 1: [2023-05-08 11:26:03,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 1: [2023-05-08 11:26:03,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +20: [2023-05-08 11:26:03,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +20: [2023-05-08 11:26:03,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 1: [2023-05-08 11:26:03,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:26:03,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:26:03,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:26:03,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:26:03,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:26:03,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:26:03,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:26:03,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:26:03,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:26:03,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +24: [2023-05-08 11:26:03,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:26:03,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +24: [2023-05-08 11:26:03,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:26:03,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +26: [2023-05-08 11:26:03,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +27: [2023-05-08 11:26:03,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 2: [2023-05-08 11:26:03,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +20: [2023-05-08 11:26:03,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +20: [2023-05-08 11:26:03,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +20: [2023-05-08 11:26:03,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +20: [2023-05-08 11:26:03,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:26:03,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:26:03,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:26:03,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +20: [2023-05-08 11:26:03,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:26:03,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +15: [2023-05-08 11:26:03,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 2: [2023-05-08 11:26:03,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:26:03,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +20: [2023-05-08 11:26:03,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 5: [2023-05-08 11:26:03,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:26:03,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:26:03,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +15: [2023-05-08 11:26:03,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:26:03,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +15: [2023-05-08 11:26:03,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +22: [2023-05-08 11:26:03,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:26:03,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 5: [2023-05-08 11:26:03,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:26:03,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +25: [2023-05-08 11:26:03,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:26:03,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:26:03,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +15: [2023-05-08 11:26:03,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +22: [2023-05-08 11:26:03,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 9: [2023-05-08 11:26:03,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 9: [2023-05-08 11:26:03,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 9: [2023-05-08 11:26:03,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +23: [2023-05-08 11:26:03,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +23: [2023-05-08 11:26:03,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:26:03,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:26:03,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +23: [2023-05-08 11:26:03,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:26:03,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:26:03,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:26:03,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +20: [2023-05-08 11:26:03,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +23: [2023-05-08 11:26:03,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +23: [2023-05-08 11:26:03,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:26:03,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:26:03,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:26:03,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:26:03,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:26:03,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +21: [2023-05-08 11:26:03,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +21: [2023-05-08 11:26:03,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:26:03,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 8: [2023-05-08 11:26:03,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:26:03,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +27: [2023-05-08 11:26:03,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:26:03,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +11: [2023-05-08 11:26:03,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +11: [2023-05-08 11:26:03,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 8: [2023-05-08 11:26:03,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +11: [2023-05-08 11:26:03,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:26:03,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +26: [2023-05-08 11:26:03,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:26:03,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +26: [2023-05-08 11:26:03,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +11: [2023-05-08 11:26:03,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +11: [2023-05-08 11:26:03,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +11: [2023-05-08 11:26:03,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:26:03,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:26:03,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:26:03,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:26:03,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:26:03,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +31: [2023-05-08 11:26:03,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +31: [2023-05-08 11:26:03,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +31: [2023-05-08 11:26:03,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:26:03,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:26:03,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:26:03,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:26:03,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +26: [2023-05-08 11:26:03,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:26:03,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:26:03,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +26: [2023-05-08 11:26:03,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +26: [2023-05-08 11:26:03,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +26: [2023-05-08 11:26:03,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +12: [2023-05-08 11:26:03,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 2: [2023-05-08 11:26:03,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:26:03,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +12: [2023-05-08 11:26:03,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:26:03,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +12: [2023-05-08 11:26:03,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:26:03,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +27: [2023-05-08 11:26:03,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:26:03,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 9: [2023-05-08 11:26:03,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +24: [2023-05-08 11:26:03,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 4: [2023-05-08 11:26:03,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 4: [2023-05-08 11:26:03,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 9: [2023-05-08 11:26:03,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +25: [2023-05-08 11:26:03,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:26:03,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +26: [2023-05-08 11:26:03,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +26: [2023-05-08 11:26:03,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 2: [2023-05-08 11:26:03,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:26:03,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +24: [2023-05-08 11:26:03,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 2: [2023-05-08 11:26:03,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:26:03,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +10: [2023-05-08 11:26:03,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:26:03,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:26:03,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:26:03,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +10: [2023-05-08 11:26:03,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +10: [2023-05-08 11:26:03,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 5: [2023-05-08 11:26:03,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:26:03,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +10: [2023-05-08 11:26:03,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 7: [2023-05-08 11:26:03,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:26:03,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +31: [2023-05-08 11:26:03,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +20: [2023-05-08 11:26:03,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 7: [2023-05-08 11:26:03,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:26:03,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +18: [2023-05-08 11:26:03,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:26:03,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:26:03,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 4: [2023-05-08 11:26:03,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +31: [2023-05-08 11:26:03,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +18: [2023-05-08 11:26:04,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 9: [2023-05-08 11:26:04,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +10: [2023-05-08 11:26:04,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:26:03,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +23: [2023-05-08 11:26:03,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +23: [2023-05-08 11:26:03,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +27: [2023-05-08 11:26:04,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +23: [2023-05-08 11:26:04,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 9: [2023-05-08 11:26:04,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:26:04,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +24: [2023-05-08 11:26:04,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +24: [2023-05-08 11:26:04,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 7: [2023-05-08 11:26:04,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 9: [2023-05-08 11:26:04,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:26:04,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:26:04,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:26:04,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:26:04,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +28: [2023-05-08 11:26:04,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +24: [2023-05-08 11:26:04,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:26:04,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:26:04,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +20: [2023-05-08 11:26:04,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +20: [2023-05-08 11:26:04,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +27: [2023-05-08 11:26:04,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:26:04,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +11: [2023-05-08 11:26:04,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +30: [2023-05-08 11:26:04,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +30: [2023-05-08 11:26:04,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +30: [2023-05-08 11:26:04,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:26:04,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +30: [2023-05-08 11:26:04,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 7: [2023-05-08 11:26:04,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +30: [2023-05-08 11:26:04,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:26:04,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:26:04,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +30: [2023-05-08 11:26:04,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +30: [2023-05-08 11:26:04,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:26:04,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:26:04,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +30: [2023-05-08 11:26:04,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +30: [2023-05-08 11:26:04,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 9: [2023-05-08 11:26:04,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:26:04,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 9: [2023-05-08 11:26:04,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +30: [2023-05-08 11:26:04,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +29: [2023-05-08 11:26:04,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +29: [2023-05-08 11:26:04,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +29: [2023-05-08 11:26:04,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +26: [2023-05-08 11:26:04,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:26:04,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:26:04,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +17: [2023-05-08 11:26:04,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +17: [2023-05-08 11:26:04,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +12: [2023-05-08 11:26:04,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +17: [2023-05-08 11:26:04,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +29: [2023-05-08 11:26:04,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +17: [2023-05-08 11:26:04,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 5: [2023-05-08 11:26:04,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +23: [2023-05-08 11:26:04,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 2: [2023-05-08 11:26:04,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 2: [2023-05-08 11:26:04,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +11: [2023-05-08 11:26:04,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +11: [2023-05-08 11:26:04,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:26:04,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +31: [2023-05-08 11:26:04,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:26:04,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +10: [2023-05-08 11:26:04,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:26:04,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +20: [2023-05-08 11:26:04,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +20: [2023-05-08 11:26:04,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:26:04,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +20: [2023-05-08 11:26:04,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +20: [2023-05-08 11:26:04,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:26:04,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +23: [2023-05-08 11:26:04,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +24: [2023-05-08 11:26:04,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +31: [2023-05-08 11:26:04,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +23: [2023-05-08 11:26:04,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +26: [2023-05-08 11:26:04,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +10: [2023-05-08 11:26:04,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:26:04,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +12: [2023-05-08 11:26:04,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,013] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +23: [2023-05-08 11:26:04,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:26:04,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +26: [2023-05-08 11:26:04,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +11: [2023-05-08 11:26:04,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:26:04,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:26:04,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 8: [2023-05-08 11:26:04,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 8: [2023-05-08 11:26:04,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +30: [2023-05-08 11:26:04,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:26:04,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +17: [2023-05-08 11:26:04,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 7: [2023-05-08 11:26:04,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +31: [2023-05-08 11:26:04,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +23: [2023-05-08 11:26:04,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:26:04,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:26:04,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +23: [2023-05-08 11:26:04,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:26:04,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:26:04,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +31: [2023-05-08 11:26:04,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:26:04,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:26:04,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +28: [2023-05-08 11:26:04,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:26:04,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +24: [2023-05-08 11:26:04,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +11: [2023-05-08 11:26:04,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +29: [2023-05-08 11:26:04,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +17: [2023-05-08 11:26:04,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:26:04,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +25: [2023-05-08 11:26:04,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +24: [2023-05-08 11:26:04,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +28: [2023-05-08 11:26:04,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +28: [2023-05-08 11:26:04,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:26:04,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +29: [2023-05-08 11:26:04,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +29: [2023-05-08 11:26:04,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +31: [2023-05-08 11:26:04,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:26:04,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +10: [2023-05-08 11:26:04,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +11: [2023-05-08 11:26:04,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +10: [2023-05-08 11:26:04,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +30: [2023-05-08 11:26:04,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +24: [2023-05-08 11:26:04,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +20: [2023-05-08 11:26:04,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:26:04,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +24: [2023-05-08 11:26:04,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 7: [2023-05-08 11:26:04,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:26:04,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +23: [2023-05-08 11:26:04,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +26: [2023-05-08 11:26:04,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +12: [2023-05-08 11:26:04,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +23: [2023-05-08 11:26:04,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +12: [2023-05-08 11:26:04,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 2: [2023-05-08 11:26:04,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +20: [2023-05-08 11:26:04,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:26:04,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +30: [2023-05-08 11:26:04,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +24: [2023-05-08 11:26:04,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +24: [2023-05-08 11:26:04,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:26:04,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +31: [2023-05-08 11:26:04,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +23: [2023-05-08 11:26:04,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 9: [2023-05-08 11:26:04,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:26:04,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +10: [2023-05-08 11:26:04,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +31: [2023-05-08 11:26:04,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:26:04,027] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:26:04,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 7: [2023-05-08 11:26:04,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +10: [2023-05-08 11:26:04,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:26:04,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:26:04,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:26:04,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +28: [2023-05-08 11:26:04,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +11: [2023-05-08 11:26:04,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +11: [2023-05-08 11:26:04,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +11: [2023-05-08 11:26:04,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +11: [2023-05-08 11:26:04,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +28: [2023-05-08 11:26:04,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 9: [2023-05-08 11:26:04,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:26:04,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:26:04,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:26:04,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +31: [2023-05-08 11:26:04,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:26:04,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +17: [2023-05-08 11:26:04,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:26:04,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +26: [2023-05-08 11:26:04,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:26:04,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:26:04,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:26:04,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +30: [2023-05-08 11:26:04,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +30: [2023-05-08 11:26:04,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 7: [2023-05-08 11:26:04,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +30: [2023-05-08 11:26:04,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +30: [2023-05-08 11:26:04,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +30: [2023-05-08 11:26:04,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +29: [2023-05-08 11:26:04,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:26:04,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +29: [2023-05-08 11:26:04,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +30: [2023-05-08 11:26:04,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:26:04,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:26:04,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +24: [2023-05-08 11:26:04,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +29: [2023-05-08 11:26:04,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +17: [2023-05-08 11:26:04,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +17: [2023-05-08 11:26:04,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +30: [2023-05-08 11:26:04,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +22: [2023-05-08 11:26:04,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +10: [2023-05-08 11:26:04,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:26:04,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +22: [2023-05-08 11:26:04,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +31: [2023-05-08 11:26:04,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:26:04,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 5: [2023-05-08 11:26:04,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 5: [2023-05-08 11:26:04,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 5: [2023-05-08 11:26:04,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +11: [2023-05-08 11:26:04,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 2: [2023-05-08 11:26:04,039] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +30: [2023-05-08 11:26:04,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:26:04,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:26:04,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +23: [2023-05-08 11:26:04,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:26:04,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +23: [2023-05-08 11:26:04,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 8: [2023-05-08 11:26:04,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 8: [2023-05-08 11:26:04,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +14: [2023-05-08 11:26:04,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +28: [2023-05-08 11:26:04,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:26:04,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:26:04,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +14: [2023-05-08 11:26:04,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +14: [2023-05-08 11:26:04,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 9: [2023-05-08 11:26:04,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +23: [2023-05-08 11:26:04,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +23: [2023-05-08 11:26:04,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +14: [2023-05-08 11:26:04,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +28: [2023-05-08 11:26:04,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +28: [2023-05-08 11:26:04,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 6: [2023-05-08 11:26:04,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +28: [2023-05-08 11:26:04,041] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +11: [2023-05-08 11:26:04,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +11: [2023-05-08 11:26:04,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:26:04,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +14: [2023-05-08 11:26:04,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:26:04,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:26:04,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:26:04,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +14: [2023-05-08 11:26:04,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +14: [2023-05-08 11:26:04,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:26:04,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +14: [2023-05-08 11:26:04,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:26:04,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +13: [2023-05-08 11:26:04,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +14: [2023-05-08 11:26:04,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:26:04,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 2: [2023-05-08 11:26:04,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +30: [2023-05-08 11:26:04,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:26:04,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +12: [2023-05-08 11:26:04,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 9: [2023-05-08 11:26:04,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +12: [2023-05-08 11:26:04,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 2: [2023-05-08 11:26:04,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:26:04,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +31: [2023-05-08 11:26:04,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +31: [2023-05-08 11:26:04,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 2: [2023-05-08 11:26:04,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +22: [2023-05-08 11:26:04,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +24: [2023-05-08 11:26:04,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:26:04,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +29: [2023-05-08 11:26:04,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:26:04,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +24: [2023-05-08 11:26:04,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 5: [2023-05-08 11:26:04,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:26:04,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +24: [2023-05-08 11:26:04,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:26:04,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +29: [2023-05-08 11:26:04,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +22: [2023-05-08 11:26:04,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +28: [2023-05-08 11:26:04,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +28: [2023-05-08 11:26:04,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +22: [2023-05-08 11:26:04,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +28: [2023-05-08 11:26:04,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +24: [2023-05-08 11:26:04,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +14: [2023-05-08 11:26:04,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +30: [2023-05-08 11:26:04,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +23: [2023-05-08 11:26:04,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:26:04,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:26:04,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:26:04,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:26:04,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:26:04,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +11: [2023-05-08 11:26:04,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:26:04,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 0: [2023-05-08 11:26:04,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:26:04,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +13: [2023-05-08 11:26:04,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +10: [2023-05-08 11:26:04,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +10: [2023-05-08 11:26:04,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +11: [2023-05-08 11:26:04,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:26:04,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +10: [2023-05-08 11:26:04,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 3: [2023-05-08 11:26:04,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +13: [2023-05-08 11:26:04,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. + 3: [2023-05-08 11:26:04,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +27: [2023-05-08 11:26:04,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +27: [2023-05-08 11:26:04,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +27: [2023-05-08 11:26:04,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +11: [2023-05-08 11:26:04,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +13: [2023-05-08 11:26:04,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +10: [2023-05-08 11:26:04,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +11: [2023-05-08 11:26:04,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:26:04,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +11: [2023-05-08 11:26:04,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +16: [2023-05-08 11:26:04,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +16: [2023-05-08 11:26:04,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +16: [2023-05-08 11:26:04,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +11: [2023-05-08 11:26:04,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +16: [2023-05-08 11:26:04,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +12: [2023-05-08 11:26:04,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:26:04,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:26:04,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +27: [2023-05-08 11:26:04,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt. +14: [2023-05-08 11:26:04,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 0: [2023-05-08 11:26:04,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +14: [2023-05-08 11:26:04,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:26:04,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:26:04,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +10: [2023-05-08 11:26:04,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +27: [2023-05-08 11:26:04,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +10: [2023-05-08 11:26:04,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +27: [2023-05-08 11:26:04,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:26:04,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:26:04,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:26:04,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 4: [2023-05-08 11:26:04,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:26:04,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +13: [2023-05-08 11:26:04,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 5: [2023-05-08 11:26:04,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +14: [2023-05-08 11:26:04,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +14: [2023-05-08 11:26:04,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:26:04,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:26:04,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:26:04,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:26:04,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +22: [2023-05-08 11:26:04,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:26:04,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:26:04,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:26:04,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +22: [2023-05-08 11:26:04,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +13: [2023-05-08 11:26:04,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:26:04,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +13: [2023-05-08 11:26:04,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 5: [2023-05-08 11:26:04,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 5: [2023-05-08 11:26:04,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +16: [2023-05-08 11:26:04,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +16: [2023-05-08 11:26:04,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +10: [2023-05-08 11:26:04,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +16: [2023-05-08 11:26:04,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 3: [2023-05-08 11:26:04,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:26:04,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:26:04,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:26:04,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:26:04,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +28: [2023-05-08 11:26:04,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 3: [2023-05-08 11:26:04,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:26:04,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +12: [2023-05-08 11:26:04,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +16: [2023-05-08 11:26:04,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +14: [2023-05-08 11:26:04,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +14: [2023-05-08 11:26:04,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +22: [2023-05-08 11:26:04,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +14: [2023-05-08 11:26:04,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +22: [2023-05-08 11:26:04,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +14: [2023-05-08 11:26:04,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 0: [2023-05-08 11:26:04,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:26:04,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +10: [2023-05-08 11:26:04,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:26:04,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:26:04,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +28: [2023-05-08 11:26:04,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +28: [2023-05-08 11:26:04,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +10: [2023-05-08 11:26:04,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 3: [2023-05-08 11:26:04,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +14: [2023-05-08 11:26:04,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:26:04,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +10: [2023-05-08 11:26:04,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +14: [2023-05-08 11:26:04,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:26:04,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +10: [2023-05-08 11:26:04,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 4: [2023-05-08 11:26:04,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +27: [2023-05-08 11:26:04,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +29: [2023-05-08 11:26:04,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +29: [2023-05-08 11:26:04,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +29: [2023-05-08 11:26:04,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +29: [2023-05-08 11:26:04,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:26:04,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +29: [2023-05-08 11:26:04,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:26:04,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:26:04,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:26:04,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +12: [2023-05-08 11:26:04,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +12: [2023-05-08 11:26:04,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +13: [2023-05-08 11:26:04,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:26:04,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +29: [2023-05-08 11:26:04,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:26:04,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:26:04,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +13: [2023-05-08 11:26:04,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 3: [2023-05-08 11:26:04,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:26:04,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:26:04,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +27: [2023-05-08 11:26:04,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +27: [2023-05-08 11:26:04,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 4: [2023-05-08 11:26:04,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:26:04,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:26:04,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:26:04,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:26:04,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:26:04,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:26:04,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:26:04,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:26:04,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:26:04,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:26:04,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:26:04,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:26:04,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:26:04,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:26:04,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:26:04,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:26:04,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:26:04,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +10: [2023-05-08 11:26:04,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +27: [2023-05-08 11:26:04,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:26:04,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +29: [2023-05-08 11:26:04,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 3: [2023-05-08 11:26:04,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 3: [2023-05-08 11:26:04,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +27: [2023-05-08 11:26:04,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +18: [2023-05-08 11:26:04,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:26:04,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:26:04,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:26:04,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +29: [2023-05-08 11:26:04,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +29: [2023-05-08 11:26:04,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +29: [2023-05-08 11:26:04,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +29: [2023-05-08 11:26:04,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:26:04,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:26:04,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:26:04,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +29: [2023-05-08 11:26:04,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:26:04,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:26:04,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:26:04,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:26:04,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:26:04,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:26:04,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:26:04,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +21: [2023-05-08 11:26:04,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:26:04,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:26:04,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:26:04,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:26:04,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:26:04,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +21: [2023-05-08 11:26:04,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:26:04,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:26:04,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:26:04,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:26:04,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 6: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 6: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +16: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +16: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +16: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +16: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:26:04,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:26:04,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +16: [2023-05-08 11:26:04,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +16: [2023-05-08 11:26:04,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:26:04,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... + 6: [2023-05-08 11:26:04,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt... +16: [2023-05-08 11:26:04,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +13: [2023-05-08 11:26:04,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:26:04,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +16: [2023-05-08 11:26:04,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +21: [2023-05-08 11:26:04,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:26:04,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:26:04,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:26:04,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +13: [2023-05-08 11:26:04,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:26:04,140] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:26:04,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +13: [2023-05-08 11:26:04,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +16: [2023-05-08 11:26:04,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt... +21: [2023-05-08 11:26:04,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:26:04,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +21: [2023-05-08 11:26:04,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:26:04,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:26:04,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:26:04,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:26:04,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:26:04,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:26:04,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +16: [2023-05-08 11:26:04,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +13: [2023-05-08 11:26:04,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:26:04,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:26:04,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +21: [2023-05-08 11:26:04,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:26:04,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +16: [2023-05-08 11:26:04,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +16: [2023-05-08 11:26:04,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,164] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +16: [2023-05-08 11:26:04,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +16: [2023-05-08 11:26:04,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +13: [2023-05-08 11:26:04,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:26:04,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:26:04,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt. +19: [2023-05-08 11:26:04,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +19: [2023-05-08 11:26:04,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +19: [2023-05-08 11:26:04,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +19: [2023-05-08 11:26:04,173] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 6: [2023-05-08 11:26:04,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:26:04,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:26:04,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:26:04,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:26:04,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:26:04,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:26:04,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:26:04,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:26:04,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:26:04,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:26:04,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:26:04,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:26:04,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:26:04,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:26:04,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +25: [2023-05-08 11:26:04,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +20: [2023-05-08 11:26:04,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +10: [2023-05-08 11:26:04,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:26:04,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +10: [2023-05-08 11:26:04,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:26:04,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +10: [2023-05-08 11:26:04,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:26:04,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:26:04,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:26:04,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:26:04,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +25: [2023-05-08 11:26:04,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +22: [2023-05-08 11:26:04,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +20: [2023-05-08 11:26:04,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +10: [2023-05-08 11:26:04,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:26:04,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:26:04,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:26:04,200] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +27: [2023-05-08 11:26:04,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +27: [2023-05-08 11:26:04,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +27: [2023-05-08 11:26:04,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +27: [2023-05-08 11:26:04,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +27: [2023-05-08 11:26:04,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +27: [2023-05-08 11:26:04,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +27: [2023-05-08 11:26:04,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:26:04,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +27: [2023-05-08 11:26:04,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:26:04,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:26:04,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +20: [2023-05-08 11:26:04,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:26:04,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:26:04,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:26:04,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:26:04,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:26:04,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +10: [2023-05-08 11:26:04,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:26:04,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:26:04,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +20: [2023-05-08 11:26:04,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:26:04,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:26:04,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +20: [2023-05-08 11:26:04,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:26:04,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:26:04,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:26:04,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +27: [2023-05-08 11:26:04,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:26:04,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +19: [2023-05-08 11:26:04,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 8: [2023-05-08 11:26:04,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:26:04,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +27: [2023-05-08 11:26:04,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +27: [2023-05-08 11:26:04,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:26:04,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:26:04,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:26:04,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +22: [2023-05-08 11:26:04,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:26:04,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +21: [2023-05-08 11:26:04,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +21: [2023-05-08 11:26:04,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +21: [2023-05-08 11:26:04,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +21: [2023-05-08 11:26:04,221] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 8: [2023-05-08 11:26:04,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +22: [2023-05-08 11:26:04,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +25: [2023-05-08 11:26:04,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +27: [2023-05-08 11:26:04,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +25: [2023-05-08 11:26:04,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +25: [2023-05-08 11:26:04,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +20: [2023-05-08 11:26:04,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +20: [2023-05-08 11:26:04,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +25: [2023-05-08 11:26:04,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 8: [2023-05-08 11:26:04,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +27: [2023-05-08 11:26:04,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +27: [2023-05-08 11:26:04,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:26:04,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:26:04,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:26:04,226] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:26:04,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:26:04,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:26:04,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:26:04,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:26:04,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:26:04,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:26:04,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +15: [2023-05-08 11:26:04,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +15: [2023-05-08 11:26:04,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +15: [2023-05-08 11:26:04,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +21: [2023-05-08 11:26:04,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +21: [2023-05-08 11:26:04,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +27: [2023-05-08 11:26:04,232] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:26:04,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:26:04,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 1: [2023-05-08 11:26:04,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 1: [2023-05-08 11:26:04,233] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 1: [2023-05-08 11:26:04,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +21: [2023-05-08 11:26:04,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:26:04,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +31: [2023-05-08 11:26:04,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +31: [2023-05-08 11:26:04,234] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +31: [2023-05-08 11:26:04,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:26:04,234] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:26:04,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +30: [2023-05-08 11:26:04,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +31: [2023-05-08 11:26:04,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +31: [2023-05-08 11:26:04,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +30: [2023-05-08 11:26:04,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:26:04,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +30: [2023-05-08 11:26:04,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:26:04,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +30: [2023-05-08 11:26:04,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:26:04,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +31: [2023-05-08 11:26:04,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:26:04,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:26:04,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +30: [2023-05-08 11:26:04,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +30: [2023-05-08 11:26:04,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +25: [2023-05-08 11:26:04,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +10: [2023-05-08 11:26:04,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +30: [2023-05-08 11:26:04,238] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:26:04,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +30: [2023-05-08 11:26:04,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:26:04,239] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:26:04,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +27: [2023-05-08 11:26:04,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:26:04,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:26:04,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:26:04,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:26:04,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:26:04,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:26:04,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:26:04,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +28: [2023-05-08 11:26:04,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:26:04,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +24: [2023-05-08 11:26:04,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +24: [2023-05-08 11:26:04,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 1: [2023-05-08 11:26:04,246] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +28: [2023-05-08 11:26:04,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:26:04,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +28: [2023-05-08 11:26:04,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 7: [2023-05-08 11:26:04,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 7: [2023-05-08 11:26:04,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +30: [2023-05-08 11:26:04,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:26:04,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +30: [2023-05-08 11:26:04,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +28: [2023-05-08 11:26:04,248] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 4: [2023-05-08 11:26:04,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 4: [2023-05-08 11:26:04,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +14: [2023-05-08 11:26:04,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:26:04,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +31: [2023-05-08 11:26:04,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:26:04,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:26:04,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:26:04,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +21: [2023-05-08 11:26:04,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +15: [2023-05-08 11:26:04,251] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:26:04,252] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +23: [2023-05-08 11:26:04,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +23: [2023-05-08 11:26:04,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +23: [2023-05-08 11:26:04,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +23: [2023-05-08 11:26:04,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +21: [2023-05-08 11:26:04,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:26:04,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 3: [2023-05-08 11:26:04,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:26:04,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:26:04,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:26:04,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +24: [2023-05-08 11:26:04,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:26:04,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:26:04,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +30: [2023-05-08 11:26:04,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +21: [2023-05-08 11:26:04,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:26:04,257] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +28: [2023-05-08 11:26:04,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:26:04,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:26:04,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 4: [2023-05-08 11:26:04,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:26:04,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:26:04,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:26:04,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:26:04,260] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:26:04,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:26:04,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:26:04,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,261] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +30: [2023-05-08 11:26:04,262] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +23: [2023-05-08 11:26:04,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:26:04,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:26:04,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +30: [2023-05-08 11:26:04,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +23: [2023-05-08 11:26:04,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:26:04,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:26:04,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +15: [2023-05-08 11:26:04,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +28: [2023-05-08 11:26:04,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:26:04,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +30: [2023-05-08 11:26:04,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:26:04,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +31: [2023-05-08 11:26:04,270] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 1: [2023-05-08 11:26:04,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +31: [2023-05-08 11:26:04,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 7: [2023-05-08 11:26:04,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 9: [2023-05-08 11:26:04,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 7: [2023-05-08 11:26:04,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 9: [2023-05-08 11:26:04,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 0: [2023-05-08 11:26:04,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 4: [2023-05-08 11:26:04,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +16: [2023-05-08 11:26:04,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:26:04,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +16: [2023-05-08 11:26:04,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +16: [2023-05-08 11:26:04,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +16: [2023-05-08 11:26:04,275] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +30: [2023-05-08 11:26:04,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 3: [2023-05-08 11:26:04,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +24: [2023-05-08 11:26:04,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +16: [2023-05-08 11:26:04,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +16: [2023-05-08 11:26:04,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:26:04,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +30: [2023-05-08 11:26:04,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +24: [2023-05-08 11:26:04,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 0: [2023-05-08 11:26:04,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +16: [2023-05-08 11:26:04,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:26:04,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +16: [2023-05-08 11:26:04,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:26:04,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 4: [2023-05-08 11:26:04,279] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:26:04,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 4: [2023-05-08 11:26:04,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +28: [2023-05-08 11:26:04,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 3: [2023-05-08 11:26:04,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:26:04,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +24: [2023-05-08 11:26:04,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +24: [2023-05-08 11:26:04,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +23: [2023-05-08 11:26:04,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:26:04,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +23: [2023-05-08 11:26:04,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +23: [2023-05-08 11:26:04,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:26:04,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +23: [2023-05-08 11:26:04,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:26:04,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +23: [2023-05-08 11:26:04,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +23: [2023-05-08 11:26:04,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +23: [2023-05-08 11:26:04,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +23: [2023-05-08 11:26:04,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:26:04,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +23: [2023-05-08 11:26:04,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +23: [2023-05-08 11:26:04,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +28: [2023-05-08 11:26:04,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +16: [2023-05-08 11:26:04,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +16: [2023-05-08 11:26:04,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +26: [2023-05-08 11:26:04,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +26: [2023-05-08 11:26:04,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +23: [2023-05-08 11:26:04,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +23: [2023-05-08 11:26:04,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +26: [2023-05-08 11:26:04,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 7: [2023-05-08 11:26:04,292] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +26: [2023-05-08 11:26:04,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 7: [2023-05-08 11:26:04,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +24: [2023-05-08 11:26:04,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +24: [2023-05-08 11:26:04,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +24: [2023-05-08 11:26:04,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +23: [2023-05-08 11:26:04,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +24: [2023-05-08 11:26:04,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +24: [2023-05-08 11:26:04,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +24: [2023-05-08 11:26:04,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,298] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 5: [2023-05-08 11:26:04,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +16: [2023-05-08 11:26:04,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +16: [2023-05-08 11:26:04,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +24: [2023-05-08 11:26:04,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +25: [2023-05-08 11:26:04,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +25: [2023-05-08 11:26:04,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +25: [2023-05-08 11:26:04,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +25: [2023-05-08 11:26:04,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +24: [2023-05-08 11:26:04,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +25: [2023-05-08 11:26:04,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +25: [2023-05-08 11:26:04,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +23: [2023-05-08 11:26:04,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +25: [2023-05-08 11:26:04,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +25: [2023-05-08 11:26:04,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:26:04,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:26:04,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:26:04,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:26:04,303] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +23: [2023-05-08 11:26:04,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:26:04,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:26:04,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +17: [2023-05-08 11:26:04,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +17: [2023-05-08 11:26:04,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +18: [2023-05-08 11:26:04,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +18: [2023-05-08 11:26:04,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +17: [2023-05-08 11:26:04,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:26:04,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +17: [2023-05-08 11:26:04,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:26:04,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +26: [2023-05-08 11:26:04,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:26:04,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +17: [2023-05-08 11:26:04,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +17: [2023-05-08 11:26:04,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:26:04,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +26: [2023-05-08 11:26:04,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:26:04,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +11: [2023-05-08 11:26:04,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +11: [2023-05-08 11:26:04,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +11: [2023-05-08 11:26:04,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +11: [2023-05-08 11:26:04,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +20: [2023-05-08 11:26:04,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:26:04,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +17: [2023-05-08 11:26:04,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +11: [2023-05-08 11:26:04,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +11: [2023-05-08 11:26:04,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +11: [2023-05-08 11:26:04,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +23: [2023-05-08 11:26:04,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +15: [2023-05-08 11:26:04,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +15: [2023-05-08 11:26:04,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:26:04,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +15: [2023-05-08 11:26:04,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +24: [2023-05-08 11:26:04,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +23: [2023-05-08 11:26:04,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +15: [2023-05-08 11:26:04,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +15: [2023-05-08 11:26:04,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +11: [2023-05-08 11:26:04,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:26:04,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:26:04,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +16: [2023-05-08 11:26:04,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +16: [2023-05-08 11:26:04,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:26:04,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 0: [2023-05-08 11:26:04,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +27: [2023-05-08 11:26:04,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +27: [2023-05-08 11:26:04,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +27: [2023-05-08 11:26:04,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 6: [2023-05-08 11:26:04,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +26: [2023-05-08 11:26:04,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:26:04,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:26:04,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +12: [2023-05-08 11:26:04,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +12: [2023-05-08 11:26:04,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:26:04,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:26:04,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:26:04,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:26:04,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:26:04,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:26:04,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:26:04,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:26:04,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +24: [2023-05-08 11:26:04,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +12: [2023-05-08 11:26:04,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +12: [2023-05-08 11:26:04,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +23: [2023-05-08 11:26:04,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +12: [2023-05-08 11:26:04,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:26:04,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:26:04,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +23: [2023-05-08 11:26:04,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:26:04,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:26:04,317] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:26:04,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +17: [2023-05-08 11:26:04,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +11: [2023-05-08 11:26:04,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +25: [2023-05-08 11:26:04,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +25: [2023-05-08 11:26:04,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +24: [2023-05-08 11:26:04,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +23: [2023-05-08 11:26:04,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:26:04,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +11: [2023-05-08 11:26:04,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +16: [2023-05-08 11:26:04,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +26: [2023-05-08 11:26:04,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:26:04,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +11: [2023-05-08 11:26:04,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:26:04,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +15: [2023-05-08 11:26:04,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:26:04,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:26:04,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:26:04,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +17: [2023-05-08 11:26:04,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +24: [2023-05-08 11:26:04,323] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +26: [2023-05-08 11:26:04,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +27: [2023-05-08 11:26:04,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +26: [2023-05-08 11:26:04,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 7: [2023-05-08 11:26:04,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:26:04,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:26:04,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:26:04,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:26:04,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +28: [2023-05-08 11:26:04,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +28: [2023-05-08 11:26:04,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +28: [2023-05-08 11:26:04,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +28: [2023-05-08 11:26:04,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +17: [2023-05-08 11:26:04,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:26:04,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:26:04,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +11: [2023-05-08 11:26:04,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +24: [2023-05-08 11:26:04,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:26:04,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:26:04,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +25: [2023-05-08 11:26:04,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:26:04,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +25: [2023-05-08 11:26:04,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:26:04,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +26: [2023-05-08 11:26:04,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +15: [2023-05-08 11:26:04,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +12: [2023-05-08 11:26:04,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +12: [2023-05-08 11:26:04,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +12: [2023-05-08 11:26:04,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +17: [2023-05-08 11:26:04,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +27: [2023-05-08 11:26:04,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:26:04,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:26:04,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +25: [2023-05-08 11:26:04,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +26: [2023-05-08 11:26:04,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:26:04,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 0: [2023-05-08 11:26:04,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 0: [2023-05-08 11:26:04,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 0: [2023-05-08 11:26:04,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +24: [2023-05-08 11:26:04,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +15: [2023-05-08 11:26:04,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:26:04,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:26:04,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:26:04,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 4: [2023-05-08 11:26:04,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:26:04,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 3: [2023-05-08 11:26:04,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 3: [2023-05-08 11:26:04,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 9: [2023-05-08 11:26:04,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 9: [2023-05-08 11:26:04,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +26: [2023-05-08 11:26:04,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:26:04,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:26:04,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +24: [2023-05-08 11:26:04,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +26: [2023-05-08 11:26:04,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 6: [2023-05-08 11:26:04,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +12: [2023-05-08 11:26:04,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:26:04,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +28: [2023-05-08 11:26:04,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:26:04,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:26:04,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:26:04,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:26:04,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +13: [2023-05-08 11:26:04,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +13: [2023-05-08 11:26:04,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +12: [2023-05-08 11:26:04,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:26:04,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:26:04,341] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +27: [2023-05-08 11:26:04,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +27: [2023-05-08 11:26:04,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +27: [2023-05-08 11:26:04,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +12: [2023-05-08 11:26:04,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:26:04,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +17: [2023-05-08 11:26:04,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +26: [2023-05-08 11:26:04,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:26:04,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +26: [2023-05-08 11:26:04,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +18: [2023-05-08 11:26:04,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +12: [2023-05-08 11:26:04,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +24: [2023-05-08 11:26:04,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +26: [2023-05-08 11:26:04,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +20: [2023-05-08 11:26:04,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:26:04,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +15: [2023-05-08 11:26:04,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:26:04,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +26: [2023-05-08 11:26:04,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:26:04,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +15: [2023-05-08 11:26:04,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +11: [2023-05-08 11:26:04,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,350] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +25: [2023-05-08 11:26:04,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:26:04,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:26:04,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +18: [2023-05-08 11:26:04,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:26:04,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:26:04,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:26:04,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +27: [2023-05-08 11:26:04,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +13: [2023-05-08 11:26:04,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +18: [2023-05-08 11:26:04,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:26:04,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:26:04,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:26:04,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +20: [2023-05-08 11:26:04,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +15: [2023-05-08 11:26:04,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +20: [2023-05-08 11:26:04,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:26:04,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:26:04,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +13: [2023-05-08 11:26:04,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +21: [2023-05-08 11:26:04,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +21: [2023-05-08 11:26:04,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:26:04,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +21: [2023-05-08 11:26:04,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +21: [2023-05-08 11:26:04,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +21: [2023-05-08 11:26:04,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:26:04,357] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +22: [2023-05-08 11:26:04,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:26:04,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:26:04,357] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:26:04,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +22: [2023-05-08 11:26:04,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +22: [2023-05-08 11:26:04,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +22: [2023-05-08 11:26:04,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:26:04,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:26:04,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +21: [2023-05-08 11:26:04,358] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:26:04,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +31: [2023-05-08 11:26:04,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 4: [2023-05-08 11:26:04,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +31: [2023-05-08 11:26:04,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +28: [2023-05-08 11:26:04,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +31: [2023-05-08 11:26:04,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 8: [2023-05-08 11:26:04,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:26:04,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:26:04,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +13: [2023-05-08 11:26:04,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:26:04,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 0: [2023-05-08 11:26:04,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:26:04,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +18: [2023-05-08 11:26:04,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:26:04,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:26:04,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:26:04,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:26:04,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:26:04,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +29: [2023-05-08 11:26:04,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +29: [2023-05-08 11:26:04,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:26:04,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:26:04,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:26:04,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:26:04,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:26:04,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:26:04,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +28: [2023-05-08 11:26:04,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:26:04,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:26:04,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +28: [2023-05-08 11:26:04,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 8: [2023-05-08 11:26:04,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:26:04,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:26:04,372] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:26:04,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:26:04,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:26:04,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:26:04,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:26:04,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +21: [2023-05-08 11:26:04,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +21: [2023-05-08 11:26:04,373] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +19: [2023-05-08 11:26:04,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:26:04,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:26:04,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:26:04,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +14: [2023-05-08 11:26:04,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +14: [2023-05-08 11:26:04,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:26:04,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +14: [2023-05-08 11:26:04,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:26:04,374] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:26:04,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:26:04,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:26:04,375] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +22: [2023-05-08 11:26:04,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:26:04,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:26:04,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:26:04,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:26:04,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:26:04,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:26:04,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:26:04,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +13: [2023-05-08 11:26:04,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 8: [2023-05-08 11:26:04,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:26:04,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +13: [2023-05-08 11:26:04,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +30: [2023-05-08 11:26:04,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +30: [2023-05-08 11:26:04,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +21: [2023-05-08 11:26:04,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:26:04,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +29: [2023-05-08 11:26:04,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:26:04,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:26:04,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +11: [2023-05-08 11:26:04,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +31: [2023-05-08 11:26:04,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +11: [2023-05-08 11:26:04,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 3: [2023-05-08 11:26:04,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +31: [2023-05-08 11:26:04,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +19: [2023-05-08 11:26:04,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:26:04,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +31: [2023-05-08 11:26:04,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +31: [2023-05-08 11:26:04,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +30: [2023-05-08 11:26:04,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +18: [2023-05-08 11:26:04,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:26:04,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +31: [2023-05-08 11:26:04,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +31: [2023-05-08 11:26:04,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +10: [2023-05-08 11:26:04,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +11: [2023-05-08 11:26:04,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +10: [2023-05-08 11:26:04,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +10: [2023-05-08 11:26:04,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 8: [2023-05-08 11:26:04,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:26:04,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:26:04,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:26:04,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +10: [2023-05-08 11:26:04,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +14: [2023-05-08 11:26:04,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +14: [2023-05-08 11:26:04,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 0: [2023-05-08 11:26:04,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +14: [2023-05-08 11:26:04,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +14: [2023-05-08 11:26:04,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:26:04,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 8: [2023-05-08 11:26:04,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +31: [2023-05-08 11:26:04,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +31: [2023-05-08 11:26:04,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 8: [2023-05-08 11:26:04,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 8: [2023-05-08 11:26:04,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 0: [2023-05-08 11:26:04,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:26:04,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:26:04,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:26:04,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:26:04,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:26:04,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:26:04,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:26:04,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:26:04,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:26:04,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +18: [2023-05-08 11:26:04,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +19: [2023-05-08 11:26:04,395] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:26:04,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:26:04,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:26:04,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:26:04,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +19: [2023-05-08 11:26:04,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:26:04,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +11: [2023-05-08 11:26:04,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:26:04,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:26:04,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +10: [2023-05-08 11:26:04,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:26:04,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:26:04,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:26:04,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:26:04,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +14: [2023-05-08 11:26:04,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:26:04,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +10: [2023-05-08 11:26:04,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +19: [2023-05-08 11:26:04,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +11: [2023-05-08 11:26:04,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:26:04,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +31: [2023-05-08 11:26:04,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +31: [2023-05-08 11:26:04,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:26:04,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +31: [2023-05-08 11:26:04,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:26:04,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +31: [2023-05-08 11:26:04,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +31: [2023-05-08 11:26:04,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:26:04,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:26:04,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:26:04,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +31: [2023-05-08 11:26:04,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:26:04,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:26:04,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:26:04,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:26:04,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:26:04,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +28: [2023-05-08 11:26:04,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:26:04,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +28: [2023-05-08 11:26:04,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:26:04,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +28: [2023-05-08 11:26:04,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:26:04,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:26:04,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +11: [2023-05-08 11:26:04,409] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +11: [2023-05-08 11:26:04,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +22: [2023-05-08 11:26:04,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +10: [2023-05-08 11:26:04,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:26:04,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +22: [2023-05-08 11:26:04,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +10: [2023-05-08 11:26:04,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +10: [2023-05-08 11:26:04,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +10: [2023-05-08 11:26:04,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +22: [2023-05-08 11:26:04,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +28: [2023-05-08 11:26:04,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:26:04,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +14: [2023-05-08 11:26:04,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +14: [2023-05-08 11:26:04,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +19: [2023-05-08 11:26:04,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 5: [2023-05-08 11:26:04,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +30: [2023-05-08 11:26:04,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +29: [2023-05-08 11:26:04,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 5: [2023-05-08 11:26:04,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 5: [2023-05-08 11:26:04,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +30: [2023-05-08 11:26:04,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +30: [2023-05-08 11:26:04,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:26:04,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +30: [2023-05-08 11:26:04,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +12: [2023-05-08 11:26:04,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +10: [2023-05-08 11:26:04,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +10: [2023-05-08 11:26:04,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +10: [2023-05-08 11:26:04,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:26:04,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +12: [2023-05-08 11:26:04,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 2: [2023-05-08 11:26:04,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +12: [2023-05-08 11:26:04,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 3: [2023-05-08 11:26:04,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 3: [2023-05-08 11:26:04,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +11: [2023-05-08 11:26:04,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +11: [2023-05-08 11:26:04,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +29: [2023-05-08 11:26:04,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +29: [2023-05-08 11:26:04,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 2: [2023-05-08 11:26:04,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +29: [2023-05-08 11:26:04,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 2: [2023-05-08 11:26:04,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +10: [2023-05-08 11:26:04,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:26:04,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +30: [2023-05-08 11:26:04,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 3: [2023-05-08 11:26:04,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 6: [2023-05-08 11:26:04,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 6: [2023-05-08 11:26:04,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +16: [2023-05-08 11:26:04,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 6: [2023-05-08 11:26:04,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 8: [2023-05-08 11:26:04,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +16: [2023-05-08 11:26:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. + 8: [2023-05-08 11:26:04,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +11: [2023-05-08 11:26:04,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +11: [2023-05-08 11:26:04,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 3: [2023-05-08 11:26:04,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:26:04,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:26:04,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:26:04,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +30: [2023-05-08 11:26:04,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +30: [2023-05-08 11:26:04,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:26:04,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:26:04,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:26:04,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +30: [2023-05-08 11:26:04,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +17: [2023-05-08 11:26:04,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +17: [2023-05-08 11:26:04,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +17: [2023-05-08 11:26:04,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +17: [2023-05-08 11:26:04,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +22: [2023-05-08 11:26:04,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:26:04,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:26:04,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:26:04,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +31: [2023-05-08 11:26:04,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:26:04,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +30: [2023-05-08 11:26:04,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:26:04,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +14: [2023-05-08 11:26:04,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:26:04,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:26:04,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +16: [2023-05-08 11:26:04,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +29: [2023-05-08 11:26:04,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +13: [2023-05-08 11:26:04,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:26:04,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +14: [2023-05-08 11:26:04,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +16: [2023-05-08 11:26:04,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt. +22: [2023-05-08 11:26:04,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +11: [2023-05-08 11:26:04,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:26:04,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +12: [2023-05-08 11:26:04,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +28: [2023-05-08 11:26:04,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:26:04,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:26:04,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:26:04,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +12: [2023-05-08 11:26:04,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:26:04,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +12: [2023-05-08 11:26:04,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +14: [2023-05-08 11:26:04,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:26:04,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +11: [2023-05-08 11:26:04,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +10: [2023-05-08 11:26:04,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:26:04,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 5: [2023-05-08 11:26:04,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:26:04,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +29: [2023-05-08 11:26:04,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +10: [2023-05-08 11:26:04,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +10: [2023-05-08 11:26:04,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 3: [2023-05-08 11:26:04,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +10: [2023-05-08 11:26:04,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +13: [2023-05-08 11:26:04,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +10: [2023-05-08 11:26:04,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +17: [2023-05-08 11:26:04,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +16: [2023-05-08 11:26:04,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:26:04,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +16: [2023-05-08 11:26:04,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +17: [2023-05-08 11:26:04,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +31: [2023-05-08 11:26:04,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +31: [2023-05-08 11:26:04,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:26:04,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +22: [2023-05-08 11:26:04,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +16: [2023-05-08 11:26:04,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +10: [2023-05-08 11:26:04,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:26:04,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +16: [2023-05-08 11:26:04,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt... +30: [2023-05-08 11:26:04,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +28: [2023-05-08 11:26:04,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:26:04,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:26:04,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:26:04,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:26:04,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +31: [2023-05-08 11:26:04,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:26:04,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +10: [2023-05-08 11:26:04,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +13: [2023-05-08 11:26:04,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +10: [2023-05-08 11:26:04,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +31: [2023-05-08 11:26:04,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:26:04,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +29: [2023-05-08 11:26:04,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +10: [2023-05-08 11:26:04,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +14: [2023-05-08 11:26:04,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:26:04,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 3: [2023-05-08 11:26:04,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:26:04,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:26:04,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +22: [2023-05-08 11:26:04,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +22: [2023-05-08 11:26:04,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +14: [2023-05-08 11:26:04,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:26:04,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:26:04,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +14: [2023-05-08 11:26:04,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 5: [2023-05-08 11:26:04,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 5: [2023-05-08 11:26:04,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +13: [2023-05-08 11:26:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +28: [2023-05-08 11:26:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +17: [2023-05-08 11:26:04,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:26:04,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +10: [2023-05-08 11:26:04,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +17: [2023-05-08 11:26:04,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +30: [2023-05-08 11:26:04,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +12: [2023-05-08 11:26:04,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +12: [2023-05-08 11:26:04,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +16: [2023-05-08 11:26:04,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +16: [2023-05-08 11:26:04,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +17: [2023-05-08 11:26:04,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +12: [2023-05-08 11:26:04,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +12: [2023-05-08 11:26:04,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +17: [2023-05-08 11:26:04,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +16: [2023-05-08 11:26:04,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +16: [2023-05-08 11:26:04,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +30: [2023-05-08 11:26:04,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:26:04,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 2: [2023-05-08 11:26:04,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +16: [2023-05-08 11:26:04,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:26:04,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +10: [2023-05-08 11:26:04,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +16: [2023-05-08 11:26:04,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:26:04,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +31: [2023-05-08 11:26:04,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:26:04,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +29: [2023-05-08 11:26:04,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +29: [2023-05-08 11:26:04,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +29: [2023-05-08 11:26:04,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +29: [2023-05-08 11:26:04,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +29: [2023-05-08 11:26:04,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +29: [2023-05-08 11:26:04,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +16: [2023-05-08 11:26:04,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +16: [2023-05-08 11:26:04,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +16: [2023-05-08 11:26:04,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +12: [2023-05-08 11:26:04,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +12: [2023-05-08 11:26:04,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +28: [2023-05-08 11:26:04,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +12: [2023-05-08 11:26:04,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +12: [2023-05-08 11:26:04,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +13: [2023-05-08 11:26:04,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... + 5: [2023-05-08 11:26:04,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +14: [2023-05-08 11:26:04,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +14: [2023-05-08 11:26:04,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +22: [2023-05-08 11:26:04,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +17: [2023-05-08 11:26:04,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +30: [2023-05-08 11:26:04,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +17: [2023-05-08 11:26:04,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +22: [2023-05-08 11:26:04,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +21: [2023-05-08 11:26:04,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +21: [2023-05-08 11:26:04,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +21: [2023-05-08 11:26:04,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +10: [2023-05-08 11:26:04,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:26:04,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 6: [2023-05-08 11:26:04,455] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +16: [2023-05-08 11:26:04,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +16: [2023-05-08 11:26:04,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +16: [2023-05-08 11:26:04,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +16: [2023-05-08 11:26:04,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:26:04,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +21: [2023-05-08 11:26:04,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +16: [2023-05-08 11:26:04,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt. +16: [2023-05-08 11:26:04,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +16: [2023-05-08 11:26:04,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +16: [2023-05-08 11:26:04,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt... +21: [2023-05-08 11:26:04,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:26:04,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:26:04,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +16: [2023-05-08 11:26:04,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +19: [2023-05-08 11:26:04,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +19: [2023-05-08 11:26:04,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +19: [2023-05-08 11:26:04,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +19: [2023-05-08 11:26:04,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +16: [2023-05-08 11:26:04,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:26:04,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +16: [2023-05-08 11:26:04,478] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +20: [2023-05-08 11:26:04,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +20: [2023-05-08 11:26:04,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +20: [2023-05-08 11:26:04,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +20: [2023-05-08 11:26:04,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +19: [2023-05-08 11:26:04,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:26:04,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:26:04,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:26:04,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:26:04,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +21: [2023-05-08 11:26:04,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +21: [2023-05-08 11:26:04,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +19: [2023-05-08 11:26:04,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:26:04,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +21: [2023-05-08 11:26:04,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +16: [2023-05-08 11:26:04,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +20: [2023-05-08 11:26:04,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:26:04,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +21: [2023-05-08 11:26:04,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +21: [2023-05-08 11:26:04,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +20: [2023-05-08 11:26:04,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:26:04,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:26:04,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:26:04,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +27: [2023-05-08 11:26:04,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:26:04,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:26:04,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:26:04,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +27: [2023-05-08 11:26:04,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +27: [2023-05-08 11:26:04,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +27: [2023-05-08 11:26:04,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:26:04,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:26:04,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:26:04,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:26:04,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +19: [2023-05-08 11:26:04,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +20: [2023-05-08 11:26:04,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:26:04,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 6: [2023-05-08 11:26:04,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:26:04,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:26:04,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +20: [2023-05-08 11:26:04,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:26:04,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +20: [2023-05-08 11:26:04,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:26:04,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:26:04,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +19: [2023-05-08 11:26:04,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +27: [2023-05-08 11:26:04,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:26:04,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,514] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:26:04,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:26:04,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:26:04,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:26:04,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:26:04,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:26:04,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +24: [2023-05-08 11:26:04,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +24: [2023-05-08 11:26:04,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +24: [2023-05-08 11:26:04,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +23: [2023-05-08 11:26:04,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +23: [2023-05-08 11:26:04,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +24: [2023-05-08 11:26:04,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +23: [2023-05-08 11:26:04,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +23: [2023-05-08 11:26:04,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +23: [2023-05-08 11:26:04,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +23: [2023-05-08 11:26:04,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:26:04,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +23: [2023-05-08 11:26:04,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:26:04,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +23: [2023-05-08 11:26:04,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:26:04,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +23: [2023-05-08 11:26:04,570] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +23: [2023-05-08 11:26:04,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:26:04,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +24: [2023-05-08 11:26:04,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +24: [2023-05-08 11:26:04,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:26:04,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +20: [2023-05-08 11:26:04,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +20: [2023-05-08 11:26:04,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +20: [2023-05-08 11:26:04,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +20: [2023-05-08 11:26:04,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +20: [2023-05-08 11:26:04,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +20: [2023-05-08 11:26:04,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +20: [2023-05-08 11:26:04,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:26:04,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +23: [2023-05-08 11:26:04,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +24: [2023-05-08 11:26:04,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +24: [2023-05-08 11:26:04,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:26:04,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:26:04,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +24: [2023-05-08 11:26:04,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +24: [2023-05-08 11:26:04,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:26:04,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:26:04,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:26:04,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +24: [2023-05-08 11:26:04,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:26:04,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:26:04,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +24: [2023-05-08 11:26:04,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:26:04,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +24: [2023-05-08 11:26:04,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:26:04,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 1: [2023-05-08 11:26:04,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:26:04,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 1: [2023-05-08 11:26:04,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +23: [2023-05-08 11:26:04,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:26:04,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +18: [2023-05-08 11:26:04,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +20: [2023-05-08 11:26:04,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +20: [2023-05-08 11:26:04,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +20: [2023-05-08 11:26:04,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +23: [2023-05-08 11:26:04,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +20: [2023-05-08 11:26:04,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:26:04,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +24: [2023-05-08 11:26:04,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 1: [2023-05-08 11:26:04,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:26:04,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:26:04,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:26:04,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:26:04,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +23: [2023-05-08 11:26:04,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +20: [2023-05-08 11:26:04,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:26:04,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:26:04,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:26:04,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:26:04,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:26:04,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +23: [2023-05-08 11:26:04,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +23: [2023-05-08 11:26:04,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +23: [2023-05-08 11:26:04,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:26:04,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:26:04,614] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:26:04,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:26:04,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:26:04,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:26:04,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +23: [2023-05-08 11:26:04,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +23: [2023-05-08 11:26:04,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +23: [2023-05-08 11:26:04,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:26:04,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:26:04,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +23: [2023-05-08 11:26:04,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +18: [2023-05-08 11:26:04,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +23: [2023-05-08 11:26:04,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:26:04,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:26:04,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +24: [2023-05-08 11:26:04,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:26:04,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +18: [2023-05-08 11:26:04,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +24: [2023-05-08 11:26:04,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +24: [2023-05-08 11:26:04,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:26:04,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +24: [2023-05-08 11:26:04,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:26:04,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:26:04,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 2: [2023-05-08 11:26:04,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 1: [2023-05-08 11:26:04,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 1: [2023-05-08 11:26:04,647] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 2: [2023-05-08 11:26:04,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 2: [2023-05-08 11:26:04,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,648] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +25: [2023-05-08 11:26:04,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:26:04,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +25: [2023-05-08 11:26:04,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:26:04,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:26:04,653] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:26:04,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +25: [2023-05-08 11:26:04,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +25: [2023-05-08 11:26:04,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 4: [2023-05-08 11:26:04,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 1: [2023-05-08 11:26:04,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 4: [2023-05-08 11:26:04,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 1: [2023-05-08 11:26:04,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 3: [2023-05-08 11:26:04,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 3: [2023-05-08 11:26:04,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 3: [2023-05-08 11:26:04,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 2: [2023-05-08 11:26:04,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:26:04,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +11: [2023-05-08 11:26:04,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:26:04,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:26:04,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +11: [2023-05-08 11:26:04,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +11: [2023-05-08 11:26:04,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:26:04,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:26:04,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:26:04,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +11: [2023-05-08 11:26:04,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +25: [2023-05-08 11:26:04,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:26:04,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 1: [2023-05-08 11:26:04,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 1: [2023-05-08 11:26:04,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +17: [2023-05-08 11:26:04,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:26:04,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:26:04,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:26:04,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:26:04,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:26:04,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:26:04,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:26:04,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +18: [2023-05-08 11:26:04,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:26:04,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:26:04,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:26:04,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:26:04,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:26:04,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:26:04,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:26:04,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +25: [2023-05-08 11:26:04,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:26:04,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:26:04,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 2: [2023-05-08 11:26:04,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 4: [2023-05-08 11:26:04,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 4: [2023-05-08 11:26:04,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +11: [2023-05-08 11:26:04,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +11: [2023-05-08 11:26:04,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:26:04,687] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +17: [2023-05-08 11:26:04,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:26:04,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,692] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +18: [2023-05-08 11:26:04,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 9: [2023-05-08 11:26:04,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 9: [2023-05-08 11:26:04,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 9: [2023-05-08 11:26:04,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 7: [2023-05-08 11:26:04,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +11: [2023-05-08 11:26:04,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 7: [2023-05-08 11:26:04,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +11: [2023-05-08 11:26:04,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +17: [2023-05-08 11:26:04,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:26:04,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:26:04,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:26:04,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 7: [2023-05-08 11:26:04,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +25: [2023-05-08 11:26:04,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:26:04,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:26:04,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +11: [2023-05-08 11:26:04,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +17: [2023-05-08 11:26:04,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:26:04,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 0: [2023-05-08 11:26:04,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 3: [2023-05-08 11:26:04,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 3: [2023-05-08 11:26:04,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +27: [2023-05-08 11:26:04,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +27: [2023-05-08 11:26:04,698] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +17: [2023-05-08 11:26:04,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +27: [2023-05-08 11:26:04,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 0: [2023-05-08 11:26:04,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 0: [2023-05-08 11:26:04,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +11: [2023-05-08 11:26:04,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +31: [2023-05-08 11:26:04,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +31: [2023-05-08 11:26:04,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +31: [2023-05-08 11:26:04,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +28: [2023-05-08 11:26:04,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +28: [2023-05-08 11:26:04,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +28: [2023-05-08 11:26:04,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +11: [2023-05-08 11:26:04,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:26:04,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:26:04,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +31: [2023-05-08 11:26:04,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 9: [2023-05-08 11:26:04,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:26:04,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +25: [2023-05-08 11:26:04,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +25: [2023-05-08 11:26:04,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +29: [2023-05-08 11:26:04,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:26:04,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:26:04,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +29: [2023-05-08 11:26:04,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +29: [2023-05-08 11:26:04,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +29: [2023-05-08 11:26:04,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:26:04,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:26:04,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +18: [2023-05-08 11:26:04,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:26:04,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:26:04,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:26:04,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +11: [2023-05-08 11:26:04,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +25: [2023-05-08 11:26:04,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +13: [2023-05-08 11:26:04,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +13: [2023-05-08 11:26:04,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +11: [2023-05-08 11:26:04,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +11: [2023-05-08 11:26:04,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +11: [2023-05-08 11:26:04,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +25: [2023-05-08 11:26:04,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +17: [2023-05-08 11:26:04,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +17: [2023-05-08 11:26:04,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:26:04,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +13: [2023-05-08 11:26:04,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +27: [2023-05-08 11:26:04,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +27: [2023-05-08 11:26:04,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +27: [2023-05-08 11:26:04,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:26:04,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +11: [2023-05-08 11:26:04,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:26:04,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +17: [2023-05-08 11:26:04,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:26:04,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:26:04,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:26:04,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:26:04,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:26:04,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:26:04,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +25: [2023-05-08 11:26:04,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +25: [2023-05-08 11:26:04,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +13: [2023-05-08 11:26:04,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +29: [2023-05-08 11:26:04,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +15: [2023-05-08 11:26:04,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 9: [2023-05-08 11:26:04,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +15: [2023-05-08 11:26:04,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +15: [2023-05-08 11:26:04,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +13: [2023-05-08 11:26:04,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +25: [2023-05-08 11:26:04,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:26:04,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 7: [2023-05-08 11:26:04,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:26:04,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +25: [2023-05-08 11:26:04,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +27: [2023-05-08 11:26:04,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 9: [2023-05-08 11:26:04,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 9: [2023-05-08 11:26:04,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +13: [2023-05-08 11:26:04,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +13: [2023-05-08 11:26:04,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 7: [2023-05-08 11:26:04,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 7: [2023-05-08 11:26:04,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +12: [2023-05-08 11:26:04,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:26:04,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:26:04,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:26:04,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:26:04,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +12: [2023-05-08 11:26:04,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:26:04,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:26:04,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:26:04,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +28: [2023-05-08 11:26:04,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +28: [2023-05-08 11:26:04,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:26:04,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:26:04,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:26:04,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +31: [2023-05-08 11:26:04,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +12: [2023-05-08 11:26:04,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:26:04,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +13: [2023-05-08 11:26:04,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +13: [2023-05-08 11:26:04,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +13: [2023-05-08 11:26:04,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:26:04,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:26:04,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +31: [2023-05-08 11:26:04,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:26:04,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +13: [2023-05-08 11:26:04,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +31: [2023-05-08 11:26:04,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +13: [2023-05-08 11:26:04,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +31: [2023-05-08 11:26:04,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +11: [2023-05-08 11:26:04,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +11: [2023-05-08 11:26:04,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:26:04,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:26:04,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +11: [2023-05-08 11:26:04,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +11: [2023-05-08 11:26:04,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +15: [2023-05-08 11:26:04,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +11: [2023-05-08 11:26:04,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +11: [2023-05-08 11:26:04,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +14: [2023-05-08 11:26:04,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:26:04,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:26:04,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:26:04,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +14: [2023-05-08 11:26:04,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:26:04,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:26:04,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:26:04,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:26:04,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:26:04,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:26:04,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +14: [2023-05-08 11:26:04,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +14: [2023-05-08 11:26:04,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +31: [2023-05-08 11:26:04,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:26:04,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +31: [2023-05-08 11:26:04,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 9: [2023-05-08 11:26:04,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 0: [2023-05-08 11:26:04,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:26:04,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +25: [2023-05-08 11:26:04,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:26:04,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +15: [2023-05-08 11:26:04,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +25: [2023-05-08 11:26:04,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +25: [2023-05-08 11:26:04,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +25: [2023-05-08 11:26:04,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +27: [2023-05-08 11:26:04,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:26:04,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +28: [2023-05-08 11:26:04,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:26:04,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:26:04,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +28: [2023-05-08 11:26:04,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +28: [2023-05-08 11:26:04,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:26:04,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +28: [2023-05-08 11:26:04,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +28: [2023-05-08 11:26:04,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +26: [2023-05-08 11:26:04,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +11: [2023-05-08 11:26:04,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +13: [2023-05-08 11:26:04,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:26:04,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +31: [2023-05-08 11:26:04,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +11: [2023-05-08 11:26:04,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +27: [2023-05-08 11:26:04,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +25: [2023-05-08 11:26:04,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +25: [2023-05-08 11:26:04,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +25: [2023-05-08 11:26:04,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +25: [2023-05-08 11:26:04,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +29: [2023-05-08 11:26:04,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +31: [2023-05-08 11:26:04,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +29: [2023-05-08 11:26:04,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:26:04,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +17: [2023-05-08 11:26:04,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +27: [2023-05-08 11:26:04,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +28: [2023-05-08 11:26:04,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +28: [2023-05-08 11:26:04,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +26: [2023-05-08 11:26:04,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:26:04,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:26:04,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:26:04,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +26: [2023-05-08 11:26:04,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:26:04,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:26:04,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 0: [2023-05-08 11:26:04,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +26: [2023-05-08 11:26:04,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +31: [2023-05-08 11:26:04,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:26:04,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +31: [2023-05-08 11:26:04,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +31: [2023-05-08 11:26:04,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +31: [2023-05-08 11:26:04,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +31: [2023-05-08 11:26:04,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +31: [2023-05-08 11:26:04,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:26:04,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +31: [2023-05-08 11:26:04,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +12: [2023-05-08 11:26:04,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:26:04,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:26:04,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +31: [2023-05-08 11:26:04,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:26:04,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:26:04,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:26:04,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +13: [2023-05-08 11:26:04,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:26:04,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:26:04,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:26:04,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:26:04,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:26:04,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +22: [2023-05-08 11:26:04,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +22: [2023-05-08 11:26:04,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +26: [2023-05-08 11:26:04,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:26:04,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +26: [2023-05-08 11:26:04,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:26:04,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +13: [2023-05-08 11:26:04,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +13: [2023-05-08 11:26:04,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:26:04,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:26:04,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +21: [2023-05-08 11:26:04,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +21: [2023-05-08 11:26:04,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +21: [2023-05-08 11:26:04,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:26:04,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +29: [2023-05-08 11:26:04,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:26:04,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:26:04,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +22: [2023-05-08 11:26:04,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +22: [2023-05-08 11:26:04,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:26:04,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +22: [2023-05-08 11:26:04,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +21: [2023-05-08 11:26:04,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +21: [2023-05-08 11:26:04,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +21: [2023-05-08 11:26:04,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +12: [2023-05-08 11:26:04,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:26:04,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +22: [2023-05-08 11:26:04,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:26:04,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:26:04,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:26:04,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:26:04,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +29: [2023-05-08 11:26:04,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +29: [2023-05-08 11:26:04,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +21: [2023-05-08 11:26:04,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:26:04,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:26:04,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 5: [2023-05-08 11:26:04,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 5: [2023-05-08 11:26:04,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +15: [2023-05-08 11:26:04,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +14: [2023-05-08 11:26:04,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:26:04,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:26:04,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +17: [2023-05-08 11:26:04,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +17: [2023-05-08 11:26:04,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +29: [2023-05-08 11:26:04,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +26: [2023-05-08 11:26:04,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +15: [2023-05-08 11:26:04,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 8: [2023-05-08 11:26:04,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 8: [2023-05-08 11:26:04,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 8: [2023-05-08 11:26:04,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 8: [2023-05-08 11:26:04,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +13: [2023-05-08 11:26:04,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:26:04,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:26:04,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:26:04,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +12: [2023-05-08 11:26:04,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:26:04,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +12: [2023-05-08 11:26:04,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:26:04,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:26:04,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +27: [2023-05-08 11:26:04,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +14: [2023-05-08 11:26:04,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +27: [2023-05-08 11:26:04,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:26:04,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +27: [2023-05-08 11:26:04,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:26:04,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:26:04,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +27: [2023-05-08 11:26:04,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +22: [2023-05-08 11:26:04,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +22: [2023-05-08 11:26:04,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +30: [2023-05-08 11:26:04,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +30: [2023-05-08 11:26:04,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:26:04,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +31: [2023-05-08 11:26:04,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +31: [2023-05-08 11:26:04,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +30: [2023-05-08 11:26:04,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:26:04,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +27: [2023-05-08 11:26:04,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:26:04,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:26:04,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +30: [2023-05-08 11:26:04,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +15: [2023-05-08 11:26:04,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +27: [2023-05-08 11:26:04,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 2: [2023-05-08 11:26:04,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 8: [2023-05-08 11:26:04,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 2: [2023-05-08 11:26:04,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 8: [2023-05-08 11:26:04,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,757] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +30: [2023-05-08 11:26:04,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:26:04,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +13: [2023-05-08 11:26:04,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:26:04,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:26:04,758] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +15: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +15: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +15: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +14: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 3: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +19: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +19: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +10: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +14: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +19: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +28: [2023-05-08 11:26:04,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +19: [2023-05-08 11:26:04,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +19: [2023-05-08 11:26:04,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:26:04,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:26:04,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:26:04,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +19: [2023-05-08 11:26:04,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:26:04,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +19: [2023-05-08 11:26:04,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:26:04,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:26:04,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +10: [2023-05-08 11:26:04,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 9: [2023-05-08 11:26:04,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +14: [2023-05-08 11:26:04,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:26:04,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:26:04,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:26:04,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +16: [2023-05-08 11:26:04,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:26:04,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:26:04,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +10: [2023-05-08 11:26:04,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:26:04,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:26:04,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +17: [2023-05-08 11:26:04,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 9: [2023-05-08 11:26:04,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:26:04,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +17: [2023-05-08 11:26:04,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 8: [2023-05-08 11:26:04,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:26:04,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:26:04,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +14: [2023-05-08 11:26:04,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +16: [2023-05-08 11:26:04,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:26:04,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +21: [2023-05-08 11:26:04,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +21: [2023-05-08 11:26:04,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:26:04,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +31: [2023-05-08 11:26:04,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:26:04,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +17: [2023-05-08 11:26:04,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +26: [2023-05-08 11:26:04,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +31: [2023-05-08 11:26:04,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +16: [2023-05-08 11:26:04,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +12: [2023-05-08 11:26:04,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:26:04,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:26:04,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:26:04,765] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +22: [2023-05-08 11:26:04,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:26:04,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +27: [2023-05-08 11:26:04,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:26:04,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +28: [2023-05-08 11:26:04,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +30: [2023-05-08 11:26:04,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +31: [2023-05-08 11:26:04,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +30: [2023-05-08 11:26:04,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +22: [2023-05-08 11:26:04,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +30: [2023-05-08 11:26:04,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +30: [2023-05-08 11:26:04,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +30: [2023-05-08 11:26:04,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +14: [2023-05-08 11:26:04,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:26:04,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +30: [2023-05-08 11:26:04,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 6: [2023-05-08 11:26:04,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:26:04,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +12: [2023-05-08 11:26:04,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +12: [2023-05-08 11:26:04,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 3: [2023-05-08 11:26:04,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:26:04,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +21: [2023-05-08 11:26:04,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:26:04,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 6: [2023-05-08 11:26:04,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +10: [2023-05-08 11:26:04,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +10: [2023-05-08 11:26:04,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +10: [2023-05-08 11:26:04,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +21: [2023-05-08 11:26:04,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:26:04,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +19: [2023-05-08 11:26:04,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:26:04,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +14: [2023-05-08 11:26:04,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:26:04,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:26:04,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:26:04,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:26:04,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +14: [2023-05-08 11:26:04,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:26:04,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:26:04,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +19: [2023-05-08 11:26:04,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +22: [2023-05-08 11:26:04,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:26:04,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:26:04,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +22: [2023-05-08 11:26:04,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +22: [2023-05-08 11:26:04,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +22: [2023-05-08 11:26:04,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +27: [2023-05-08 11:26:04,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:26:04,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +19: [2023-05-08 11:26:04,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +21: [2023-05-08 11:26:04,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:26:04,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:26:04,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:26:04,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:26:04,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +31: [2023-05-08 11:26:04,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +29: [2023-05-08 11:26:04,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:26:04,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:26:04,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:26:04,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +15: [2023-05-08 11:26:04,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +15: [2023-05-08 11:26:04,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +15: [2023-05-08 11:26:04,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +17: [2023-05-08 11:26:04,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 2: [2023-05-08 11:26:04,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +29: [2023-05-08 11:26:04,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +29: [2023-05-08 11:26:04,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:26:04,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +29: [2023-05-08 11:26:04,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +29: [2023-05-08 11:26:04,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +22: [2023-05-08 11:26:04,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:26:04,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +27: [2023-05-08 11:26:04,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +27: [2023-05-08 11:26:04,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:26:04,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +26: [2023-05-08 11:26:04,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 2: [2023-05-08 11:26:04,781] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 6: [2023-05-08 11:26:04,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +12: [2023-05-08 11:26:04,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +30: [2023-05-08 11:26:04,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:26:04,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +12: [2023-05-08 11:26:04,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:26:04,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +30: [2023-05-08 11:26:04,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +30: [2023-05-08 11:26:04,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +10: [2023-05-08 11:26:04,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +27: [2023-05-08 11:26:04,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 7: [2023-05-08 11:26:04,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +19: [2023-05-08 11:26:04,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:26:04,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +19: [2023-05-08 11:26:04,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:26:04,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:26:04,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 8: [2023-05-08 11:26:04,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +22: [2023-05-08 11:26:04,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:26:04,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +22: [2023-05-08 11:26:04,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:26:04,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. + 2: [2023-05-08 11:26:04,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +14: [2023-05-08 11:26:04,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +26: [2023-05-08 11:26:04,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:26:04,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +16: [2023-05-08 11:26:04,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt. +19: [2023-05-08 11:26:04,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:26:04,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:26:04,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +22: [2023-05-08 11:26:04,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:26:04,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +10: [2023-05-08 11:26:04,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +30: [2023-05-08 11:26:04,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +19: [2023-05-08 11:26:04,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:26:04,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +19: [2023-05-08 11:26:04,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +19: [2023-05-08 11:26:04,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +19: [2023-05-08 11:26:04,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +19: [2023-05-08 11:26:04,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 6: [2023-05-08 11:26:04,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:26:04,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:26:04,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +15: [2023-05-08 11:26:04,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:26:04,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:26:04,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +20: [2023-05-08 11:26:04,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +20: [2023-05-08 11:26:04,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +15: [2023-05-08 11:26:04,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:26:04,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 4: [2023-05-08 11:26:04,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +20: [2023-05-08 11:26:04,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 8: [2023-05-08 11:26:04,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +30: [2023-05-08 11:26:04,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +27: [2023-05-08 11:26:04,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:26:04,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +27: [2023-05-08 11:26:04,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 5: [2023-05-08 11:26:04,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:26:04,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +24: [2023-05-08 11:26:04,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +24: [2023-05-08 11:26:04,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +24: [2023-05-08 11:26:04,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +23: [2023-05-08 11:26:04,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +21: [2023-05-08 11:26:04,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +21: [2023-05-08 11:26:04,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +21: [2023-05-08 11:26:04,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +21: [2023-05-08 11:26:04,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +29: [2023-05-08 11:26:04,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +23: [2023-05-08 11:26:04,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +23: [2023-05-08 11:26:04,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +23: [2023-05-08 11:26:04,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 9: [2023-05-08 11:26:04,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +27: [2023-05-08 11:26:04,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +15: [2023-05-08 11:26:04,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:26:04,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +15: [2023-05-08 11:26:04,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:26:04,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +14: [2023-05-08 11:26:04,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +14: [2023-05-08 11:26:04,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +14: [2023-05-08 11:26:04,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +14: [2023-05-08 11:26:04,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 0: [2023-05-08 11:26:04,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:26:04,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +14: [2023-05-08 11:26:04,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:26:04,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:26:04,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:26:04,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 6: [2023-05-08 11:26:04,795] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:26:04,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:26:04,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +16: [2023-05-08 11:26:04,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +22: [2023-05-08 11:26:04,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:26:04,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:26:04,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 5: [2023-05-08 11:26:04,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +19: [2023-05-08 11:26:04,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +14: [2023-05-08 11:26:04,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +10: [2023-05-08 11:26:04,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:26:04,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt... +19: [2023-05-08 11:26:04,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:26:04,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 0: [2023-05-08 11:26:04,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:26:04,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +19: [2023-05-08 11:26:04,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:26:04,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +19: [2023-05-08 11:26:04,798] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:26:04,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 7: [2023-05-08 11:26:04,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:26:04,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 6: [2023-05-08 11:26:04,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +16: [2023-05-08 11:26:04,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 6: [2023-05-08 11:26:04,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 6: [2023-05-08 11:26:04,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 9: [2023-05-08 11:26:04,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:26:04,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:26:04,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:26:04,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +20: [2023-05-08 11:26:04,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:26:04,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:26:04,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:26:04,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 0: [2023-05-08 11:26:04,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 9: [2023-05-08 11:26:04,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:26:04,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 8: [2023-05-08 11:26:04,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +15: [2023-05-08 11:26:04,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +24: [2023-05-08 11:26:04,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:26:04,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +20: [2023-05-08 11:26:04,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +24: [2023-05-08 11:26:04,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:26:04,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +23: [2023-05-08 11:26:04,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:26:04,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:26:04,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +10: [2023-05-08 11:26:04,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +24: [2023-05-08 11:26:04,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:26:04,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +12: [2023-05-08 11:26:04,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +23: [2023-05-08 11:26:04,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +12: [2023-05-08 11:26:04,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +12: [2023-05-08 11:26:04,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +12: [2023-05-08 11:26:04,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +10: [2023-05-08 11:26:04,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +12: [2023-05-08 11:26:04,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:26:04,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +12: [2023-05-08 11:26:04,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +10: [2023-05-08 11:26:04,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +10: [2023-05-08 11:26:04,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +21: [2023-05-08 11:26:04,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:26:04,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +23: [2023-05-08 11:26:04,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +10: [2023-05-08 11:26:04,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +10: [2023-05-08 11:26:04,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +10: [2023-05-08 11:26:04,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 5: [2023-05-08 11:26:04,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. + 5: [2023-05-08 11:26:04,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 5: [2023-05-08 11:26:04,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 9: [2023-05-08 11:26:04,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:26:04,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:26:04,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:26:04,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:26:04,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:26:04,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +16: [2023-05-08 11:26:04,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +30: [2023-05-08 11:26:04,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... + 0: [2023-05-08 11:26:04,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:26:04,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +21: [2023-05-08 11:26:04,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +16: [2023-05-08 11:26:04,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +19: [2023-05-08 11:26:04,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:26:04,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +19: [2023-05-08 11:26:04,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +19: [2023-05-08 11:26:04,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +19: [2023-05-08 11:26:04,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +20: [2023-05-08 11:26:04,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +21: [2023-05-08 11:26:04,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +21: [2023-05-08 11:26:04,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +21: [2023-05-08 11:26:04,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +21: [2023-05-08 11:26:04,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +20: [2023-05-08 11:26:04,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +20: [2023-05-08 11:26:04,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +20: [2023-05-08 11:26:04,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +18: [2023-05-08 11:26:04,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +16: [2023-05-08 11:26:04,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +16: [2023-05-08 11:26:04,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +16: [2023-05-08 11:26:04,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +16: [2023-05-08 11:26:04,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +18: [2023-05-08 11:26:04,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +18: [2023-05-08 11:26:04,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +23: [2023-05-08 11:26:04,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:26:04,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:26:04,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +23: [2023-05-08 11:26:04,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +19: [2023-05-08 11:26:04,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +23: [2023-05-08 11:26:04,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +23: [2023-05-08 11:26:04,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +19: [2023-05-08 11:26:04,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +21: [2023-05-08 11:26:04,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +21: [2023-05-08 11:26:04,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +24: [2023-05-08 11:26:04,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +23: [2023-05-08 11:26:04,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:26:04,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:26:04,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +24: [2023-05-08 11:26:04,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +23: [2023-05-08 11:26:04,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +19: [2023-05-08 11:26:04,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:26:04,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:26:04,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +20: [2023-05-08 11:26:04,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +19: [2023-05-08 11:26:04,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +23: [2023-05-08 11:26:04,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +23: [2023-05-08 11:26:04,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +20: [2023-05-08 11:26:04,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +16: [2023-05-08 11:26:04,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt. +20: [2023-05-08 11:26:04,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +24: [2023-05-08 11:26:04,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:26:04,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:26:04,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:26:04,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt... +18: [2023-05-08 11:26:04,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +24: [2023-05-08 11:26:04,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +20: [2023-05-08 11:26:04,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +18: [2023-05-08 11:26:04,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:26:04,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:26:04,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:26:04,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +18: [2023-05-08 11:26:04,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +18: [2023-05-08 11:26:04,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:26:04,847] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 3: [2023-05-08 11:26:04,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 3: [2023-05-08 11:26:04,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 3: [2023-05-08 11:26:04,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 3: [2023-05-08 11:26:04,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +18: [2023-05-08 11:26:04,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:26:04,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 3: [2023-05-08 11:26:04,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 1: [2023-05-08 11:26:04,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 1: [2023-05-08 11:26:04,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 1: [2023-05-08 11:26:04,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 1: [2023-05-08 11:26:04,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:26:04,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 3: [2023-05-08 11:26:04,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 3: [2023-05-08 11:26:04,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 3: [2023-05-08 11:26:04,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 3: [2023-05-08 11:26:04,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 1: [2023-05-08 11:26:04,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 1: [2023-05-08 11:26:04,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 1: [2023-05-08 11:26:04,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 1: [2023-05-08 11:26:04,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +23: [2023-05-08 11:26:04,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:26:04,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:26:04,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:26:04,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:26:04,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:26:04,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:26:04,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:26:04,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:26:04,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:26:04,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:26:04,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:26:04,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +29: [2023-05-08 11:26:04,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:26:04,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:26:04,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:26:04,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:26:04,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:26:04,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:26:04,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:26:04,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:26:04,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +20: [2023-05-08 11:26:04,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +20: [2023-05-08 11:26:04,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +20: [2023-05-08 11:26:04,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:26:04,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +20: [2023-05-08 11:26:04,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:26:04,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:26:04,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:26:04,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:26:04,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:26:04,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:26:04,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:26:04,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:26:04,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:26:04,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:26:04,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:26:04,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:26:04,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +20: [2023-05-08 11:26:04,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +20: [2023-05-08 11:26:04,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:26:04,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:26:04,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +29: [2023-05-08 11:26:04,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +29: [2023-05-08 11:26:04,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:26:04,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:26:04,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:26:04,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 2: [2023-05-08 11:26:04,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:26:04,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +18: [2023-05-08 11:26:04,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +18: [2023-05-08 11:26:04,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +18: [2023-05-08 11:26:04,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:26:04,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +18: [2023-05-08 11:26:04,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:26:04,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +18: [2023-05-08 11:26:04,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +18: [2023-05-08 11:26:04,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 4: [2023-05-08 11:26:04,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +31: [2023-05-08 11:26:04,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 4: [2023-05-08 11:26:04,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +31: [2023-05-08 11:26:04,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +31: [2023-05-08 11:26:04,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 7: [2023-05-08 11:26:04,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +31: [2023-05-08 11:26:04,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 7: [2023-05-08 11:26:04,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 7: [2023-05-08 11:26:04,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 7: [2023-05-08 11:26:04,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +11: [2023-05-08 11:26:04,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +11: [2023-05-08 11:26:04,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +11: [2023-05-08 11:26:04,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +20: [2023-05-08 11:26:04,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:26:04,993] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 1: [2023-05-08 11:26:04,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:26:04,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:26:04,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +11: [2023-05-08 11:26:04,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 1: [2023-05-08 11:26:04,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:26:04,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +11: [2023-05-08 11:26:04,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:26:04,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:26:04,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 4: [2023-05-08 11:26:04,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +31: [2023-05-08 11:26:04,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:26:04,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +11: [2023-05-08 11:26:04,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +11: [2023-05-08 11:26:04,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:26:04,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:26:05,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:26:05,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +11: [2023-05-08 11:26:05,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:26:05,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +31: [2023-05-08 11:26:05,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +11: [2023-05-08 11:26:05,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +31: [2023-05-08 11:26:05,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +31: [2023-05-08 11:26:05,004] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:26:05,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,008] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +25: [2023-05-08 11:26:05,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +25: [2023-05-08 11:26:05,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +25: [2023-05-08 11:26:05,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +25: [2023-05-08 11:26:05,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +18: [2023-05-08 11:26:05,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:26:05,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +25: [2023-05-08 11:26:05,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:26:05,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:26:05,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:26:05,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:26:05,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:26:05,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:26:05,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:26:05,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:26:05,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:26:05,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:26:05,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:26:05,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:26:05,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:26:05,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:26:05,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:26:05,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +17: [2023-05-08 11:26:05,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:26:05,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +25: [2023-05-08 11:26:05,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +17: [2023-05-08 11:26:05,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:26:05,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +17: [2023-05-08 11:26:05,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:26:05,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +25: [2023-05-08 11:26:05,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +14: [2023-05-08 11:26:05,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +14: [2023-05-08 11:26:05,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +14: [2023-05-08 11:26:05,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +14: [2023-05-08 11:26:05,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +14: [2023-05-08 11:26:05,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +14: [2023-05-08 11:26:05,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +14: [2023-05-08 11:26:05,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +14: [2023-05-08 11:26:05,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:26:05,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:26:05,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +17: [2023-05-08 11:26:05,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:26:05,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +11: [2023-05-08 11:26:05,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:26:05,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +31: [2023-05-08 11:26:05,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +24: [2023-05-08 11:26:05,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +11: [2023-05-08 11:26:05,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:26:05,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:26:05,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 7: [2023-05-08 11:26:05,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +18: [2023-05-08 11:26:05,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +31: [2023-05-08 11:26:05,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +31: [2023-05-08 11:26:05,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +11: [2023-05-08 11:26:05,018] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +11: [2023-05-08 11:26:05,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +11: [2023-05-08 11:26:05,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 1: [2023-05-08 11:26:05,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 1: [2023-05-08 11:26:05,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:26:05,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:26:05,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:26:05,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +31: [2023-05-08 11:26:05,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +17: [2023-05-08 11:26:05,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:26:05,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:26:05,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:26:05,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +31: [2023-05-08 11:26:05,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +11: [2023-05-08 11:26:05,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +11: [2023-05-08 11:26:05,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +18: [2023-05-08 11:26:05,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:26:05,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +14: [2023-05-08 11:26:05,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +14: [2023-05-08 11:26:05,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:26:05,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:26:05,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +14: [2023-05-08 11:26:05,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:26:05,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +11: [2023-05-08 11:26:05,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:26:05,028] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 4: [2023-05-08 11:26:05,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 1: [2023-05-08 11:26:05,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:26:05,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:26:05,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:26:05,030] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:26:05,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +25: [2023-05-08 11:26:05,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:26:05,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 7: [2023-05-08 11:26:05,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +11: [2023-05-08 11:26:05,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:26:05,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:26:05,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +25: [2023-05-08 11:26:05,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:26:05,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +14: [2023-05-08 11:26:05,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:26:05,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +14: [2023-05-08 11:26:05,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:26:05,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +14: [2023-05-08 11:26:05,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:26:05,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +25: [2023-05-08 11:26:05,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +14: [2023-05-08 11:26:05,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:26:05,041] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:26:05,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:26:05,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:26:05,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +17: [2023-05-08 11:26:05,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +17: [2023-05-08 11:26:05,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +26: [2023-05-08 11:26:05,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +26: [2023-05-08 11:26:05,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +26: [2023-05-08 11:26:05,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +26: [2023-05-08 11:26:05,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +17: [2023-05-08 11:26:05,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:26:05,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +27: [2023-05-08 11:26:05,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +27: [2023-05-08 11:26:05,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +27: [2023-05-08 11:26:05,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +24: [2023-05-08 11:26:05,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +27: [2023-05-08 11:26:05,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +25: [2023-05-08 11:26:05,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:26:05,045] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:26:05,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:26:05,050] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:26:05,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:26:05,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:26:05,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:26:05,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +26: [2023-05-08 11:26:05,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:26:05,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +27: [2023-05-08 11:26:05,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:26:05,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:26:05,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +27: [2023-05-08 11:26:05,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +27: [2023-05-08 11:26:05,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +27: [2023-05-08 11:26:05,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +27: [2023-05-08 11:26:05,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +27: [2023-05-08 11:26:05,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +17: [2023-05-08 11:26:05,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +19: [2023-05-08 11:26:05,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +25: [2023-05-08 11:26:05,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +25: [2023-05-08 11:26:05,055] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +17: [2023-05-08 11:26:05,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:26:05,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +19: [2023-05-08 11:26:05,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +19: [2023-05-08 11:26:05,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +19: [2023-05-08 11:26:05,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +26: [2023-05-08 11:26:05,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:26:05,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:26:05,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +25: [2023-05-08 11:26:05,057] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +27: [2023-05-08 11:26:05,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:26:05,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:26:05,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +25: [2023-05-08 11:26:05,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +25: [2023-05-08 11:26:05,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:26:05,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:26:05,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +25: [2023-05-08 11:26:05,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:26:05,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:26:05,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 6: [2023-05-08 11:26:05,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +15: [2023-05-08 11:26:05,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +15: [2023-05-08 11:26:05,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +25: [2023-05-08 11:26:05,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +26: [2023-05-08 11:26:05,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +26: [2023-05-08 11:26:05,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +27: [2023-05-08 11:26:05,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:26:05,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 5: [2023-05-08 11:26:05,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +19: [2023-05-08 11:26:05,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +15: [2023-05-08 11:26:05,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +19: [2023-05-08 11:26:05,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +19: [2023-05-08 11:26:05,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +25: [2023-05-08 11:26:05,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +10: [2023-05-08 11:26:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +10: [2023-05-08 11:26:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 8: [2023-05-08 11:26:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +30: [2023-05-08 11:26:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 8: [2023-05-08 11:26:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +10: [2023-05-08 11:26:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +30: [2023-05-08 11:26:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +30: [2023-05-08 11:26:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +17: [2023-05-08 11:26:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +17: [2023-05-08 11:26:05,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +26: [2023-05-08 11:26:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:26:05,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +30: [2023-05-08 11:26:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 2: [2023-05-08 11:26:05,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 2: [2023-05-08 11:26:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 2: [2023-05-08 11:26:05,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 2: [2023-05-08 11:26:05,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 8: [2023-05-08 11:26:05,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +10: [2023-05-08 11:26:05,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +19: [2023-05-08 11:26:05,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,070] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 5: [2023-05-08 11:26:05,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +29: [2023-05-08 11:26:05,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +29: [2023-05-08 11:26:05,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +29: [2023-05-08 11:26:05,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +29: [2023-05-08 11:26:05,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +26: [2023-05-08 11:26:05,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +26: [2023-05-08 11:26:05,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +13: [2023-05-08 11:26:05,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +13: [2023-05-08 11:26:05,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +13: [2023-05-08 11:26:05,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +15: [2023-05-08 11:26:05,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:26:05,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:26:05,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +27: [2023-05-08 11:26:05,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +13: [2023-05-08 11:26:05,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:26:05,076] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:26:05,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 2: [2023-05-08 11:26:05,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:26:05,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +21: [2023-05-08 11:26:05,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +22: [2023-05-08 11:26:05,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +22: [2023-05-08 11:26:05,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +21: [2023-05-08 11:26:05,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:26:05,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +20: [2023-05-08 11:26:05,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +20: [2023-05-08 11:26:05,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +20: [2023-05-08 11:26:05,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +21: [2023-05-08 11:26:05,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +20: [2023-05-08 11:26:05,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:26:05,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:26:05,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +17: [2023-05-08 11:26:05,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +27: [2023-05-08 11:26:05,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +28: [2023-05-08 11:26:05,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +28: [2023-05-08 11:26:05,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 8: [2023-05-08 11:26:05,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:26:05,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 2: [2023-05-08 11:26:05,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:26:05,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:26:05,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +17: [2023-05-08 11:26:05,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 2: [2023-05-08 11:26:05,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 8: [2023-05-08 11:26:05,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +30: [2023-05-08 11:26:05,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +10: [2023-05-08 11:26:05,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:26:05,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +17: [2023-05-08 11:26:05,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +14: [2023-05-08 11:26:05,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 8: [2023-05-08 11:26:05,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +14: [2023-05-08 11:26:05,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +14: [2023-05-08 11:26:05,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +29: [2023-05-08 11:26:05,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:26:05,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:26:05,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:26:05,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:26:05,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:26:05,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +14: [2023-05-08 11:26:05,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +29: [2023-05-08 11:26:05,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:26:05,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +12: [2023-05-08 11:26:05,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +10: [2023-05-08 11:26:05,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +26: [2023-05-08 11:26:05,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:26:05,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +12: [2023-05-08 11:26:05,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +13: [2023-05-08 11:26:05,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +12: [2023-05-08 11:26:05,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 5: [2023-05-08 11:26:05,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +25: [2023-05-08 11:26:05,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:26:05,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:26:05,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:26:05,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +25: [2023-05-08 11:26:05,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +25: [2023-05-08 11:26:05,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 9: [2023-05-08 11:26:05,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +10: [2023-05-08 11:26:05,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +26: [2023-05-08 11:26:05,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +25: [2023-05-08 11:26:05,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +25: [2023-05-08 11:26:05,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +20: [2023-05-08 11:26:05,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:26:05,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +20: [2023-05-08 11:26:05,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +27: [2023-05-08 11:26:05,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 6: [2023-05-08 11:26:05,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +13: [2023-05-08 11:26:05,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:26:05,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:26:05,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:26:05,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:26:05,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +13: [2023-05-08 11:26:05,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +20: [2023-05-08 11:26:05,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:26:05,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:26:05,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +16: [2023-05-08 11:26:05,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +16: [2023-05-08 11:26:05,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 5: [2023-05-08 11:26:05,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:26:05,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:26:05,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +27: [2023-05-08 11:26:05,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +27: [2023-05-08 11:26:05,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:26:05,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:26:05,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:26:05,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +21: [2023-05-08 11:26:05,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:26:05,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 1: [2023-05-08 11:26:05,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 1: [2023-05-08 11:26:05,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 1: [2023-05-08 11:26:05,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +27: [2023-05-08 11:26:05,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +27: [2023-05-08 11:26:05,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +27: [2023-05-08 11:26:05,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +27: [2023-05-08 11:26:05,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +28: [2023-05-08 11:26:05,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:26:05,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +28: [2023-05-08 11:26:05,091] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:26:05,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +15: [2023-05-08 11:26:05,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +15: [2023-05-08 11:26:05,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +14: [2023-05-08 11:26:05,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +15: [2023-05-08 11:26:05,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +15: [2023-05-08 11:26:05,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:26:05,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +14: [2023-05-08 11:26:05,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 0: [2023-05-08 11:26:05,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 0: [2023-05-08 11:26:05,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. +15: [2023-05-08 11:26:05,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 2: [2023-05-08 11:26:05,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 2: [2023-05-08 11:26:05,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +12: [2023-05-08 11:26:05,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:26:05,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt. + 2: [2023-05-08 11:26:05,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 6: [2023-05-08 11:26:05,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +29: [2023-05-08 11:26:05,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:26:05,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +29: [2023-05-08 11:26:05,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +15: [2023-05-08 11:26:05,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +15: [2023-05-08 11:26:05,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 9: [2023-05-08 11:26:05,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 3: [2023-05-08 11:26:05,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 3: [2023-05-08 11:26:05,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +12: [2023-05-08 11:26:05,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:26:05,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:26:05,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +13: [2023-05-08 11:26:05,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 3: [2023-05-08 11:26:05,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +12: [2023-05-08 11:26:05,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +10: [2023-05-08 11:26:05,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:26:05,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +10: [2023-05-08 11:26:05,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +18: [2023-05-08 11:26:05,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +16: [2023-05-08 11:26:05,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +18: [2023-05-08 11:26:05,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +18: [2023-05-08 11:26:05,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +16: [2023-05-08 11:26:05,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +18: [2023-05-08 11:26:05,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +27: [2023-05-08 11:26:05,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +16: [2023-05-08 11:26:05,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +16: [2023-05-08 11:26:05,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:26:05,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +27: [2023-05-08 11:26:05,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 1: [2023-05-08 11:26:05,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +30: [2023-05-08 11:26:05,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 1: [2023-05-08 11:26:05,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +20: [2023-05-08 11:26:05,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 1: [2023-05-08 11:26:05,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +20: [2023-05-08 11:26:05,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +20: [2023-05-08 11:26:05,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +20: [2023-05-08 11:26:05,102] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 8: [2023-05-08 11:26:05,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +28: [2023-05-08 11:26:05,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:26:05,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 8: [2023-05-08 11:26:05,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +10: [2023-05-08 11:26:05,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +30: [2023-05-08 11:26:05,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:26:05,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +29: [2023-05-08 11:26:05,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +29: [2023-05-08 11:26:05,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +29: [2023-05-08 11:26:05,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +29: [2023-05-08 11:26:05,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +30: [2023-05-08 11:26:05,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 9: [2023-05-08 11:26:05,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:26:05,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 6: [2023-05-08 11:26:05,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +30: [2023-05-08 11:26:05,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +30: [2023-05-08 11:26:05,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 9: [2023-05-08 11:26:05,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +30: [2023-05-08 11:26:05,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +10: [2023-05-08 11:26:05,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +12: [2023-05-08 11:26:05,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 0: [2023-05-08 11:26:05,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:26:05,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:26:05,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +22: [2023-05-08 11:26:05,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +22: [2023-05-08 11:26:05,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 0: [2023-05-08 11:26:05,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +30: [2023-05-08 11:26:05,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +22: [2023-05-08 11:26:05,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 3: [2023-05-08 11:26:05,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,106] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:26:05,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:26:05,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:26:05,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +20: [2023-05-08 11:26:05,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +18: [2023-05-08 11:26:05,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:26:05,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +20: [2023-05-08 11:26:05,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +20: [2023-05-08 11:26:05,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 2: [2023-05-08 11:26:05,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:26:05,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +16: [2023-05-08 11:26:05,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +16: [2023-05-08 11:26:05,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:26:05,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +29: [2023-05-08 11:26:05,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 9: [2023-05-08 11:26:05,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +16: [2023-05-08 11:26:05,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +16: [2023-05-08 11:26:05,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +16: [2023-05-08 11:26:05,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:26:05,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +10: [2023-05-08 11:26:05,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:26:05,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:26:05,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:26:05,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +14: [2023-05-08 11:26:05,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +16: [2023-05-08 11:26:05,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:26:05,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:26:05,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +28: [2023-05-08 11:26:05,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:26:05,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +28: [2023-05-08 11:26:05,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +28: [2023-05-08 11:26:05,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:26:05,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +28: [2023-05-08 11:26:05,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:26:05,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +13: [2023-05-08 11:26:05,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +13: [2023-05-08 11:26:05,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +13: [2023-05-08 11:26:05,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:26:05,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:26:05,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +14: [2023-05-08 11:26:05,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +14: [2023-05-08 11:26:05,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +28: [2023-05-08 11:26:05,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:26:05,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +22: [2023-05-08 11:26:05,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 9: [2023-05-08 11:26:05,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +12: [2023-05-08 11:26:05,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:26:05,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +13: [2023-05-08 11:26:05,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 5: [2023-05-08 11:26:05,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +13: [2023-05-08 11:26:05,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 5: [2023-05-08 11:26:05,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 0: [2023-05-08 11:26:05,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt... +22: [2023-05-08 11:26:05,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +28: [2023-05-08 11:26:05,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:26:05,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +22: [2023-05-08 11:26:05,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +16: [2023-05-08 11:26:05,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +16: [2023-05-08 11:26:05,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:26:05,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 9: [2023-05-08 11:26:05,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:26:05,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 9: [2023-05-08 11:26:05,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 1: [2023-05-08 11:26:05,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +18: [2023-05-08 11:26:05,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +18: [2023-05-08 11:26:05,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +18: [2023-05-08 11:26:05,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +18: [2023-05-08 11:26:05,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +18: [2023-05-08 11:26:05,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +18: [2023-05-08 11:26:05,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +18: [2023-05-08 11:26:05,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +18: [2023-05-08 11:26:05,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 1: [2023-05-08 11:26:05,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +16: [2023-05-08 11:26:05,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +28: [2023-05-08 11:26:05,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:26:05,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:26:05,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +12: [2023-05-08 11:26:05,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:26:05,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 0: [2023-05-08 11:26:05,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +28: [2023-05-08 11:26:05,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +12: [2023-05-08 11:26:05,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +12: [2023-05-08 11:26:05,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +16: [2023-05-08 11:26:05,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:26:05,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +28: [2023-05-08 11:26:05,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +16: [2023-05-08 11:26:05,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +16: [2023-05-08 11:26:05,129] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +16: [2023-05-08 11:26:05,133] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +28: [2023-05-08 11:26:05,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:26:05,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +16: [2023-05-08 11:26:05,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... +16: [2023-05-08 11:26:05,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:26:05,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +13: [2023-05-08 11:26:05,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +13: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +13: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:26:05,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:26:05,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +12: [2023-05-08 11:26:05,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:26:05,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +12: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +28: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:26:05,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:26:05,139] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:26:05,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:26:05,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,140] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:26:05,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:26:05,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:26:05,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 0: [2023-05-08 11:26:05,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt. +16: [2023-05-08 11:26:05,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt... + 9: [2023-05-08 11:26:05,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:26:05,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +13: [2023-05-08 11:26:05,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +13: [2023-05-08 11:26:05,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +28: [2023-05-08 11:26:05,151] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +16: [2023-05-08 11:26:05,153] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:26:05,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:26:05,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:26:05,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:26:05,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:26:05,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:26:05,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:26:05,157] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:26:05,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:26:05,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:26:05,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +23: [2023-05-08 11:26:05,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +23: [2023-05-08 11:26:05,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +22: [2023-05-08 11:26:05,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:26:05,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:26:05,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:26:05,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +22: [2023-05-08 11:26:05,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:26:05,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:26:05,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:26:05,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +22: [2023-05-08 11:26:05,161] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:26:05,161] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:26:05,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:26:05,162] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,163] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:26:05,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:26:05,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:26:05,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:26:05,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:26:05,167] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +31: [2023-05-08 11:26:05,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:26:05,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:26:05,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:26:05,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +31: [2023-05-08 11:26:05,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +31: [2023-05-08 11:26:05,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:26:05,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:26:05,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:26:05,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:26:05,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +31: [2023-05-08 11:26:05,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:26:05,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +29: [2023-05-08 11:26:05,169] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:26:05,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +21: [2023-05-08 11:26:05,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:26:05,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:26:05,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:26:05,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:26:05,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:26:05,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:26:05,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:26:05,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +21: [2023-05-08 11:26:05,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +21: [2023-05-08 11:26:05,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:26:05,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:26:05,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +21: [2023-05-08 11:26:05,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +21: [2023-05-08 11:26:05,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +21: [2023-05-08 11:26:05,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +21: [2023-05-08 11:26:05,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +21: [2023-05-08 11:26:05,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:26:05,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:26:05,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +21: [2023-05-08 11:26:05,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +21: [2023-05-08 11:26:05,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 5: [2023-05-08 11:26:05,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:26:05,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:26:05,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:26:05,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +29: [2023-05-08 11:26:05,171] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:26:05,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +19: [2023-05-08 11:26:05,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:26:05,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:26:05,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:26:05,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:26:05,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +15: [2023-05-08 11:26:05,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:26:05,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +15: [2023-05-08 11:26:05,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:26:05,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +15: [2023-05-08 11:26:05,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:26:05,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +15: [2023-05-08 11:26:05,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:26:05,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:26:05,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:26:05,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:26:05,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:26:05,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +19: [2023-05-08 11:26:05,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:26:05,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:26:05,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:26:05,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +19: [2023-05-08 11:26:05,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:26:05,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +19: [2023-05-08 11:26:05,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 3: [2023-05-08 11:26:05,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 3: [2023-05-08 11:26:05,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +19: [2023-05-08 11:26:05,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 3: [2023-05-08 11:26:05,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +21: [2023-05-08 11:26:05,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:26:05,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +31: [2023-05-08 11:26:05,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:26:05,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:26:05,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:26:05,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:26:05,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:26:05,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +23: [2023-05-08 11:26:05,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +23: [2023-05-08 11:26:05,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:26:05,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +19: [2023-05-08 11:26:05,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,186] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:26:05,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +31: [2023-05-08 11:26:05,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:26:05,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:26:05,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:26:05,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +30: [2023-05-08 11:26:05,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:26:05,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +23: [2023-05-08 11:26:05,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:26:05,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +21: [2023-05-08 11:26:05,187] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:26:05,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +21: [2023-05-08 11:26:05,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:26:05,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:26:05,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,189] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +29: [2023-05-08 11:26:05,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +30: [2023-05-08 11:26:05,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:26:05,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:26:05,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:26:05,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +30: [2023-05-08 11:26:05,192] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +19: [2023-05-08 11:26:05,192] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:26:05,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +31: [2023-05-08 11:26:05,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:26:05,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:26:05,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:26:05,195] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +22: [2023-05-08 11:26:05,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +22: [2023-05-08 11:26:05,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:26:05,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:26:05,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +31: [2023-05-08 11:26:05,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,196] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +21: [2023-05-08 11:26:05,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +29: [2023-05-08 11:26:05,197] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:26:05,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:26:05,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +21: [2023-05-08 11:26:05,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +19: [2023-05-08 11:26:05,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,199] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:26:05,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:26:05,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:26:05,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:26:05,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:26:05,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:26:05,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +19: [2023-05-08 11:26:05,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:26:05,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:26:05,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +23: [2023-05-08 11:26:05,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:26:05,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:26:05,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +19: [2023-05-08 11:26:05,208] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:26:05,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +19: [2023-05-08 11:26:05,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:26:05,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:26:05,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +29: [2023-05-08 11:26:05,210] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,211] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:26:05,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:26:05,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +24: [2023-05-08 11:26:05,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +29: [2023-05-08 11:26:05,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:26:05,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 2: [2023-05-08 11:26:05,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:26:05,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +23: [2023-05-08 11:26:05,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:26:05,213] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:26:05,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 2: [2023-05-08 11:26:05,215] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +30: [2023-05-08 11:26:05,216] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:26:05,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:26:05,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:26:05,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +19: [2023-05-08 11:26:05,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +30: [2023-05-08 11:26:05,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +20: [2023-05-08 11:26:05,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +20: [2023-05-08 11:26:05,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +20: [2023-05-08 11:26:05,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +20: [2023-05-08 11:26:05,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:26:05,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:26:05,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +20: [2023-05-08 11:26:05,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:26:05,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:26:05,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:26:05,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:26:05,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:26:05,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:26:05,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:26:05,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:26:05,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +19: [2023-05-08 11:26:05,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:26:05,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:26:05,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +30: [2023-05-08 11:26:05,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:26:05,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:26:05,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +24: [2023-05-08 11:26:05,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:26:05,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +18: [2023-05-08 11:26:05,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:26:05,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:26:05,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:26:05,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:26:05,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:26:05,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:26:05,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:26:05,241] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:26:05,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +20: [2023-05-08 11:26:05,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +20: [2023-05-08 11:26:05,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:26:05,245] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:26:05,245] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +24: [2023-05-08 11:26:05,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:26:05,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +18: [2023-05-08 11:26:05,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:26:05,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:26:05,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:26:05,253] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:26:05,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:26:05,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:26:05,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:26:05,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:26:05,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:26:05,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:26:05,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +20: [2023-05-08 11:26:05,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:26:05,256] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:26:05,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:26:05,256] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +20: [2023-05-08 11:26:05,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:26:05,263] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:26:05,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:26:05,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:26:05,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:26:05,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +20: [2023-05-08 11:26:05,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:26:05,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:26:05,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +24: [2023-05-08 11:26:05,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:26:05,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:26:05,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:26:05,272] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:26:05,272] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:26:05,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:26:05,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:26:05,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:26:05,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:26:05,273] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +31: [2023-05-08 11:26:05,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +31: [2023-05-08 11:26:05,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +31: [2023-05-08 11:26:05,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +31: [2023-05-08 11:26:05,276] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 4: [2023-05-08 11:26:05,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 4: [2023-05-08 11:26:05,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 4: [2023-05-08 11:26:05,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +25: [2023-05-08 11:26:05,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +26: [2023-05-08 11:26:05,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:26:05,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:26:05,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:26:05,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:26:05,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:26:05,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:26:05,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:26:05,281] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:26:05,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:26:05,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:26:05,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +31: [2023-05-08 11:26:05,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +31: [2023-05-08 11:26:05,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +31: [2023-05-08 11:26:05,287] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +31: [2023-05-08 11:26:05,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +25: [2023-05-08 11:26:05,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:26:05,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:26:05,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:26:05,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:26:05,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +27: [2023-05-08 11:26:05,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +27: [2023-05-08 11:26:05,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +27: [2023-05-08 11:26:05,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +27: [2023-05-08 11:26:05,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:26:05,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:26:05,296] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:26:05,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:26:05,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +27: [2023-05-08 11:26:05,299] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:26:05,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +31: [2023-05-08 11:26:05,300] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:26:05,300] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +27: [2023-05-08 11:26:05,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +27: [2023-05-08 11:26:05,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +31: [2023-05-08 11:26:05,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:26:05,301] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 4: [2023-05-08 11:26:05,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:26:05,301] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:26:05,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 4: [2023-05-08 11:26:05,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +26: [2023-05-08 11:26:05,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +26: [2023-05-08 11:26:05,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:26:05,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:26:05,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:26:05,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +26: [2023-05-08 11:26:05,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +27: [2023-05-08 11:26:05,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 4: [2023-05-08 11:26:05,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +27: [2023-05-08 11:26:05,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 4: [2023-05-08 11:26:05,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 4: [2023-05-08 11:26:05,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +26: [2023-05-08 11:26:05,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +31: [2023-05-08 11:26:05,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +31: [2023-05-08 11:26:05,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 1: [2023-05-08 11:26:05,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:26:05,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:26:05,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:26:05,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:26:05,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:26:05,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:26:05,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:26:05,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:26:05,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +27: [2023-05-08 11:26:05,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:26:05,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +11: [2023-05-08 11:26:05,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +27: [2023-05-08 11:26:05,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +27: [2023-05-08 11:26:05,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,330] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +11: [2023-05-08 11:26:05,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +11: [2023-05-08 11:26:05,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +27: [2023-05-08 11:26:05,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:26:05,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +11: [2023-05-08 11:26:05,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:26:05,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:26:05,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:26:05,344] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:26:05,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +11: [2023-05-08 11:26:05,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:26:05,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,345] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 1: [2023-05-08 11:26:05,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:26:05,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +14: [2023-05-08 11:26:05,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:26:05,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:26:05,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +14: [2023-05-08 11:26:05,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:26:05,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +11: [2023-05-08 11:26:05,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:26:05,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:26:05,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:26:05,369] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:26:05,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:26:05,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:26:05,379] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:26:05,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +25: [2023-05-08 11:26:05,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +25: [2023-05-08 11:26:05,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +11: [2023-05-08 11:26:05,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +11: [2023-05-08 11:26:05,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +25: [2023-05-08 11:26:05,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +25: [2023-05-08 11:26:05,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +11: [2023-05-08 11:26:05,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +14: [2023-05-08 11:26:05,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:26:05,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:26:05,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +25: [2023-05-08 11:26:05,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:26:05,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +26: [2023-05-08 11:26:05,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +26: [2023-05-08 11:26:05,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +26: [2023-05-08 11:26:05,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +11: [2023-05-08 11:26:05,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:26:05,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +25: [2023-05-08 11:26:05,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:26:05,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +25: [2023-05-08 11:26:05,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:26:05,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:26:05,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +17: [2023-05-08 11:26:05,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +17: [2023-05-08 11:26:05,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +17: [2023-05-08 11:26:05,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +17: [2023-05-08 11:26:05,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:26:05,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +17: [2023-05-08 11:26:05,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +17: [2023-05-08 11:26:05,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:26:05,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:26:05,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +28: [2023-05-08 11:26:05,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:26:05,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:26:05,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +28: [2023-05-08 11:26:05,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +28: [2023-05-08 11:26:05,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:26:05,398] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +28: [2023-05-08 11:26:05,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:26:05,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:26:05,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +17: [2023-05-08 11:26:05,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +17: [2023-05-08 11:26:05,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +25: [2023-05-08 11:26:05,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +25: [2023-05-08 11:26:05,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +26: [2023-05-08 11:26:05,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:26:05,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +26: [2023-05-08 11:26:05,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:26:05,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +11: [2023-05-08 11:26:05,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +23: [2023-05-08 11:26:05,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +11: [2023-05-08 11:26:05,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +25: [2023-05-08 11:26:05,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +11: [2023-05-08 11:26:05,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +25: [2023-05-08 11:26:05,407] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +23: [2023-05-08 11:26:05,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +23: [2023-05-08 11:26:05,407] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 8: [2023-05-08 11:26:05,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +17: [2023-05-08 11:26:05,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:26:05,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:26:05,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 8: [2023-05-08 11:26:05,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:26:05,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 8: [2023-05-08 11:26:05,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:26:05,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +15: [2023-05-08 11:26:05,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +15: [2023-05-08 11:26:05,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +15: [2023-05-08 11:26:05,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +15: [2023-05-08 11:26:05,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +17: [2023-05-08 11:26:05,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:26:05,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +28: [2023-05-08 11:26:05,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +17: [2023-05-08 11:26:05,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +14: [2023-05-08 11:26:05,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +14: [2023-05-08 11:26:05,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +14: [2023-05-08 11:26:05,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +14: [2023-05-08 11:26:05,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +17: [2023-05-08 11:26:05,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:26:05,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +25: [2023-05-08 11:26:05,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +13: [2023-05-08 11:26:05,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +13: [2023-05-08 11:26:05,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +13: [2023-05-08 11:26:05,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +16: [2023-05-08 11:26:05,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +16: [2023-05-08 11:26:05,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +16: [2023-05-08 11:26:05,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +16: [2023-05-08 11:26:05,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +13: [2023-05-08 11:26:05,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +22: [2023-05-08 11:26:05,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +22: [2023-05-08 11:26:05,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +22: [2023-05-08 11:26:05,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +22: [2023-05-08 11:26:05,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +28: [2023-05-08 11:26:05,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:26:05,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:26:05,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:26:05,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:26:05,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +23: [2023-05-08 11:26:05,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:26:05,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:26:05,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:26:05,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +15: [2023-05-08 11:26:05,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +25: [2023-05-08 11:26:05,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:26:05,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +15: [2023-05-08 11:26:05,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:26:05,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:26:05,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +26: [2023-05-08 11:26:05,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +25: [2023-05-08 11:26:05,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 6: [2023-05-08 11:26:05,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +11: [2023-05-08 11:26:05,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +15: [2023-05-08 11:26:05,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:26:05,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:26:05,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,422] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +28: [2023-05-08 11:26:05,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:26:05,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:26:05,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +13: [2023-05-08 11:26:05,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:26:05,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:26:05,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +27: [2023-05-08 11:26:05,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +27: [2023-05-08 11:26:05,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +13: [2023-05-08 11:26:05,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:26:05,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:26:05,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:26:05,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +14: [2023-05-08 11:26:05,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +14: [2023-05-08 11:26:05,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:26:05,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:26:05,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:26:05,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:26:05,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +27: [2023-05-08 11:26:05,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +28: [2023-05-08 11:26:05,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:26:05,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:26:05,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +14: [2023-05-08 11:26:05,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +17: [2023-05-08 11:26:05,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:26:05,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +15: [2023-05-08 11:26:05,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +27: [2023-05-08 11:26:05,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +15: [2023-05-08 11:26:05,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +29: [2023-05-08 11:26:05,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +29: [2023-05-08 11:26:05,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +29: [2023-05-08 11:26:05,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 8: [2023-05-08 11:26:05,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:26:05,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +26: [2023-05-08 11:26:05,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:26:05,432] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:26:05,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 6: [2023-05-08 11:26:05,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +17: [2023-05-08 11:26:05,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:26:05,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 8: [2023-05-08 11:26:05,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +26: [2023-05-08 11:26:05,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 8: [2023-05-08 11:26:05,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:26:05,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:26:05,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +15: [2023-05-08 11:26:05,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +15: [2023-05-08 11:26:05,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +15: [2023-05-08 11:26:05,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +27: [2023-05-08 11:26:05,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:26:05,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +15: [2023-05-08 11:26:05,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +28: [2023-05-08 11:26:05,436] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 6: [2023-05-08 11:26:05,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +17: [2023-05-08 11:26:05,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:26:05,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +27: [2023-05-08 11:26:05,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +13: [2023-05-08 11:26:05,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +14: [2023-05-08 11:26:05,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:26:05,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +17: [2023-05-08 11:26:05,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +13: [2023-05-08 11:26:05,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +27: [2023-05-08 11:26:05,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +17: [2023-05-08 11:26:05,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,440] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:26:05,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +29: [2023-05-08 11:26:05,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:26:05,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:26:05,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +16: [2023-05-08 11:26:05,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +29: [2023-05-08 11:26:05,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +16: [2023-05-08 11:26:05,442] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +16: [2023-05-08 11:26:05,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +16: [2023-05-08 11:26:05,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +16: [2023-05-08 11:26:05,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +23: [2023-05-08 11:26:05,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +23: [2023-05-08 11:26:05,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +23: [2023-05-08 11:26:05,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +23: [2023-05-08 11:26:05,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +13: [2023-05-08 11:26:05,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +13: [2023-05-08 11:26:05,443] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +13: [2023-05-08 11:26:05,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +13: [2023-05-08 11:26:05,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +18: [2023-05-08 11:26:05,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +22: [2023-05-08 11:26:05,443] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +18: [2023-05-08 11:26:05,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +18: [2023-05-08 11:26:05,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +22: [2023-05-08 11:26:05,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +18: [2023-05-08 11:26:05,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +13: [2023-05-08 11:26:05,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +13: [2023-05-08 11:26:05,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +28: [2023-05-08 11:26:05,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +22: [2023-05-08 11:26:05,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:26:05,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:26:05,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +22: [2023-05-08 11:26:05,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +22: [2023-05-08 11:26:05,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +22: [2023-05-08 11:26:05,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +17: [2023-05-08 11:26:05,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:26:05,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +14: [2023-05-08 11:26:05,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +14: [2023-05-08 11:26:05,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +14: [2023-05-08 11:26:05,445] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +29: [2023-05-08 11:26:05,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:26:05,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +21: [2023-05-08 11:26:05,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +21: [2023-05-08 11:26:05,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +21: [2023-05-08 11:26:05,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +27: [2023-05-08 11:26:05,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +21: [2023-05-08 11:26:05,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +16: [2023-05-08 11:26:05,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +23: [2023-05-08 11:26:05,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:26:05,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +27: [2023-05-08 11:26:05,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +23: [2023-05-08 11:26:05,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 6: [2023-05-08 11:26:05,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +23: [2023-05-08 11:26:05,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 5: [2023-05-08 11:26:05,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 5: [2023-05-08 11:26:05,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 5: [2023-05-08 11:26:05,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 5: [2023-05-08 11:26:05,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +14: [2023-05-08 11:26:05,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:26:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +28: [2023-05-08 11:26:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +30: [2023-05-08 11:26:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +30: [2023-05-08 11:26:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +28: [2023-05-08 11:26:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +28: [2023-05-08 11:26:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +30: [2023-05-08 11:26:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 0: [2023-05-08 11:26:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 0: [2023-05-08 11:26:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +28: [2023-05-08 11:26:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +19: [2023-05-08 11:26:05,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +19: [2023-05-08 11:26:05,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 0: [2023-05-08 11:26:05,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +18: [2023-05-08 11:26:05,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +27: [2023-05-08 11:26:05,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +27: [2023-05-08 11:26:05,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +27: [2023-05-08 11:26:05,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +27: [2023-05-08 11:26:05,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +19: [2023-05-08 11:26:05,456] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +19: [2023-05-08 11:26:05,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +14: [2023-05-08 11:26:05,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 3: [2023-05-08 11:26:05,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 3: [2023-05-08 11:26:05,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 3: [2023-05-08 11:26:05,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 2: [2023-05-08 11:26:05,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 3: [2023-05-08 11:26:05,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 2: [2023-05-08 11:26:05,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 2: [2023-05-08 11:26:05,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 2: [2023-05-08 11:26:05,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +20: [2023-05-08 11:26:05,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +20: [2023-05-08 11:26:05,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +20: [2023-05-08 11:26:05,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +20: [2023-05-08 11:26:05,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +18: [2023-05-08 11:26:05,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:26:05,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:26:05,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:26:05,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 1: [2023-05-08 11:26:05,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 1: [2023-05-08 11:26:05,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 1: [2023-05-08 11:26:05,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +12: [2023-05-08 11:26:05,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 1: [2023-05-08 11:26:05,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +12: [2023-05-08 11:26:05,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +12: [2023-05-08 11:26:05,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +21: [2023-05-08 11:26:05,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:26:05,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:26:05,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:26:05,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +29: [2023-05-08 11:26:05,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +29: [2023-05-08 11:26:05,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 8: [2023-05-08 11:26:05,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 8: [2023-05-08 11:26:05,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 8: [2023-05-08 11:26:05,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +29: [2023-05-08 11:26:05,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +29: [2023-05-08 11:26:05,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +13: [2023-05-08 11:26:05,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +13: [2023-05-08 11:26:05,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:26:05,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +30: [2023-05-08 11:26:05,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:26:05,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:26:05,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:26:05,465] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:26:05,465] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +19: [2023-05-08 11:26:05,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:26:05,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:26:05,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:26:05,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +28: [2023-05-08 11:26:05,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +19: [2023-05-08 11:26:05,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +19: [2023-05-08 11:26:05,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,466] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +30: [2023-05-08 11:26:05,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:26:05,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +13: [2023-05-08 11:26:05,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +20: [2023-05-08 11:26:05,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +19: [2023-05-08 11:26:05,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +27: [2023-05-08 11:26:05,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +13: [2023-05-08 11:26:05,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:26:05,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +13: [2023-05-08 11:26:05,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:26:05,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +27: [2023-05-08 11:26:05,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 1: [2023-05-08 11:26:05,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:26:05,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +12: [2023-05-08 11:26:05,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:26:05,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:26:05,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +12: [2023-05-08 11:26:05,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:26:05,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:26:05,471] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:26:05,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 0: [2023-05-08 11:26:05,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +18: [2023-05-08 11:26:05,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +20: [2023-05-08 11:26:05,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:26:05,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:26:05,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 2: [2023-05-08 11:26:05,477] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,479] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +13: [2023-05-08 11:26:05,478] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +12: [2023-05-08 11:26:05,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:26:05,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +30: [2023-05-08 11:26:05,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:26:05,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +30: [2023-05-08 11:26:05,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 5: [2023-05-08 11:26:05,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 5: [2023-05-08 11:26:05,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +18: [2023-05-08 11:26:05,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:26:05,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +30: [2023-05-08 11:26:05,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +30: [2023-05-08 11:26:05,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +30: [2023-05-08 11:26:05,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +18: [2023-05-08 11:26:05,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +20: [2023-05-08 11:26:05,485] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:26:05,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:26:05,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:26:05,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:26:05,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +29: [2023-05-08 11:26:05,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +20: [2023-05-08 11:26:05,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +29: [2023-05-08 11:26:05,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:26:05,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:26:05,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +29: [2023-05-08 11:26:05,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +28: [2023-05-08 11:26:05,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +28: [2023-05-08 11:26:05,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +28: [2023-05-08 11:26:05,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +28: [2023-05-08 11:26:05,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 0: [2023-05-08 11:26:05,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 1: [2023-05-08 11:26:05,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +13: [2023-05-08 11:26:05,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +24: [2023-05-08 11:26:05,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 1: [2023-05-08 11:26:05,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 1: [2023-05-08 11:26:05,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +12: [2023-05-08 11:26:05,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:26:05,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +13: [2023-05-08 11:26:05,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +12: [2023-05-08 11:26:05,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +12: [2023-05-08 11:26:05,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +12: [2023-05-08 11:26:05,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +12: [2023-05-08 11:26:05,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +24: [2023-05-08 11:26:05,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +24: [2023-05-08 11:26:05,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +13: [2023-05-08 11:26:05,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +24: [2023-05-08 11:26:05,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 8: [2023-05-08 11:26:05,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 8: [2023-05-08 11:26:05,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 1: [2023-05-08 11:26:05,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 0: [2023-05-08 11:26:05,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 8: [2023-05-08 11:26:05,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +28: [2023-05-08 11:26:05,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +28: [2023-05-08 11:26:05,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 5: [2023-05-08 11:26:05,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +20: [2023-05-08 11:26:05,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +28: [2023-05-08 11:26:05,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +28: [2023-05-08 11:26:05,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 5: [2023-05-08 11:26:05,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +30: [2023-05-08 11:26:05,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +20: [2023-05-08 11:26:05,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +20: [2023-05-08 11:26:05,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 2: [2023-05-08 11:26:05,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:26:05,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 2: [2023-05-08 11:26:05,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 2: [2023-05-08 11:26:05,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +20: [2023-05-08 11:26:05,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +13: [2023-05-08 11:26:05,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +13: [2023-05-08 11:26:05,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +13: [2023-05-08 11:26:05,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +20: [2023-05-08 11:26:05,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:26:05,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:26:05,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:26:05,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +20: [2023-05-08 11:26:05,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 9: [2023-05-08 11:26:05,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 9: [2023-05-08 11:26:05,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. + 9: [2023-05-08 11:26:05,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +10: [2023-05-08 11:26:05,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +10: [2023-05-08 11:26:05,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +10: [2023-05-08 11:26:05,504] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +10: [2023-05-08 11:26:05,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt. +29: [2023-05-08 11:26:05,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:26:05,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:26:05,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +13: [2023-05-08 11:26:05,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:26:05,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:26:05,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:26:05,506] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:26:05,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:26:05,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:26:05,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:26:05,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:26:05,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +25: [2023-05-08 11:26:05,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 2: [2023-05-08 11:26:05,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:26:05,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:26:05,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:26:05,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +23: [2023-05-08 11:26:05,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:26:05,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:26:05,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +23: [2023-05-08 11:26:05,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +10: [2023-05-08 11:26:05,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:26:05,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:26:05,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:26:05,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:26:05,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:26:05,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +29: [2023-05-08 11:26:05,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +29: [2023-05-08 11:26:05,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +10: [2023-05-08 11:26:05,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt... +22: [2023-05-08 11:26:05,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +23: [2023-05-08 11:26:05,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +29: [2023-05-08 11:26:05,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:26:05,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:26:05,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:26:05,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +22: [2023-05-08 11:26:05,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +22: [2023-05-08 11:26:05,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:26:05,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:26:05,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:26:05,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +24: [2023-05-08 11:26:05,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +29: [2023-05-08 11:26:05,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +24: [2023-05-08 11:26:05,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:26:05,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +31: [2023-05-08 11:26:05,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 7: [2023-05-08 11:26:05,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +31: [2023-05-08 11:26:05,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +31: [2023-05-08 11:26:05,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +31: [2023-05-08 11:26:05,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 7: [2023-05-08 11:26:05,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:26:05,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +21: [2023-05-08 11:26:05,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +21: [2023-05-08 11:26:05,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +21: [2023-05-08 11:26:05,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +21: [2023-05-08 11:26:05,524] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +21: [2023-05-08 11:26:05,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:26:05,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +21: [2023-05-08 11:26:05,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +21: [2023-05-08 11:26:05,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +21: [2023-05-08 11:26:05,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +23: [2023-05-08 11:26:05,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:26:05,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +21: [2023-05-08 11:26:05,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:26:05,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 7: [2023-05-08 11:26:05,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:26:05,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:26:05,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:26:05,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 2: [2023-05-08 11:26:05,526] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:26:05,526] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 2: [2023-05-08 11:26:05,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:26:05,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:26:05,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +10: [2023-05-08 11:26:05,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:26:05,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:26:05,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:26:05,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +10: [2023-05-08 11:26:05,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +10: [2023-05-08 11:26:05,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +23: [2023-05-08 11:26:05,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +10: [2023-05-08 11:26:05,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:26:05,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:26:05,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +10: [2023-05-08 11:26:05,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:26:05,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:26:05,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 9: [2023-05-08 11:26:05,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 9: [2023-05-08 11:26:05,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 9: [2023-05-08 11:26:05,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... + 9: [2023-05-08 11:26:05,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +10: [2023-05-08 11:26:05,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:26:05,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +31: [2023-05-08 11:26:05,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:26:05,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:26:05,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +10: [2023-05-08 11:26:05,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +10: [2023-05-08 11:26:05,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt. +25: [2023-05-08 11:26:05,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +31: [2023-05-08 11:26:05,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +31: [2023-05-08 11:26:05,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:26:05,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +10: [2023-05-08 11:26:05,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt... +31: [2023-05-08 11:26:05,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:26:05,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +24: [2023-05-08 11:26:05,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:26:05,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +24: [2023-05-08 11:26:05,536] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 7: [2023-05-08 11:26:05,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:26:05,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +23: [2023-05-08 11:26:05,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 4: [2023-05-08 11:26:05,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 4: [2023-05-08 11:26:05,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 4: [2023-05-08 11:26:05,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +24: [2023-05-08 11:26:05,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:26:05,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +23: [2023-05-08 11:26:05,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:26:05,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +25: [2023-05-08 11:26:05,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:26:05,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:26:05,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:26:05,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:26:05,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:26:05,542] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:26:05,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:26:05,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +10: [2023-05-08 11:26:05,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:26:05,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +24: [2023-05-08 11:26:05,543] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +16: [2023-05-08 11:26:05,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +21: [2023-05-08 11:26:05,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:26:05,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:26:05,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +16: [2023-05-08 11:26:05,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:26:05,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +16: [2023-05-08 11:26:05,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +16: [2023-05-08 11:26:05,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +22: [2023-05-08 11:26:05,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:26:05,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +23: [2023-05-08 11:26:05,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +23: [2023-05-08 11:26:05,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:26:05,548] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:26:05,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +10: [2023-05-08 11:26:05,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +22: [2023-05-08 11:26:05,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +10: [2023-05-08 11:26:05,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:26:05,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:26:05,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:26:05,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 3: [2023-05-08 11:26:05,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 3: [2023-05-08 11:26:05,556] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +10: [2023-05-08 11:26:05,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:26:05,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:26:05,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:26:05,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:26:05,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:26:05,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:26:05,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +20: [2023-05-08 11:26:05,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +20: [2023-05-08 11:26:05,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +20: [2023-05-08 11:26:05,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:26:05,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +20: [2023-05-08 11:26:05,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:26:05,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +23: [2023-05-08 11:26:05,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 3: [2023-05-08 11:26:05,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:26:05,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:26:05,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:26:05,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +15: [2023-05-08 11:26:05,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +15: [2023-05-08 11:26:05,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:26:05,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:26:05,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +22: [2023-05-08 11:26:05,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:26:05,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +15: [2023-05-08 11:26:05,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +15: [2023-05-08 11:26:05,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +16: [2023-05-08 11:26:05,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:26:05,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:26:05,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:26:05,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:26:05,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:26:05,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +26: [2023-05-08 11:26:05,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +26: [2023-05-08 11:26:05,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +16: [2023-05-08 11:26:05,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +24: [2023-05-08 11:26:05,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:26:05,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:26:05,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +26: [2023-05-08 11:26:05,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 3: [2023-05-08 11:26:05,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +12: [2023-05-08 11:26:05,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:26:05,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +12: [2023-05-08 11:26:05,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +12: [2023-05-08 11:26:05,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:26:05,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +16: [2023-05-08 11:26:05,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +15: [2023-05-08 11:26:05,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +15: [2023-05-08 11:26:05,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +12: [2023-05-08 11:26:05,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +12: [2023-05-08 11:26:05,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:26:05,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:26:05,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:26:05,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +24: [2023-05-08 11:26:05,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:26:05,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +15: [2023-05-08 11:26:05,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +26: [2023-05-08 11:26:05,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 7: [2023-05-08 11:26:05,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 7: [2023-05-08 11:26:05,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +20: [2023-05-08 11:26:05,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:26:05,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:26:05,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:26:05,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +26: [2023-05-08 11:26:05,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:26:05,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:26:05,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +26: [2023-05-08 11:26:05,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:26:05,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +26: [2023-05-08 11:26:05,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:26:05,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:26:05,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +26: [2023-05-08 11:26:05,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,588] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:26:05,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +16: [2023-05-08 11:26:05,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +26: [2023-05-08 11:26:05,591] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:26:05,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,592] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:26:05,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:26:05,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:26:05,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:26:05,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:26:05,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:26:05,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:26:05,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:26:05,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:26:05,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:26:05,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +30: [2023-05-08 11:26:05,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +30: [2023-05-08 11:26:05,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +30: [2023-05-08 11:26:05,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +30: [2023-05-08 11:26:05,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +30: [2023-05-08 11:26:05,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +30: [2023-05-08 11:26:05,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +30: [2023-05-08 11:26:05,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +30: [2023-05-08 11:26:05,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:26:05,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:26:05,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +20: [2023-05-08 11:26:05,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:26:05,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +25: [2023-05-08 11:26:05,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +25: [2023-05-08 11:26:05,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:26:05,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +15: [2023-05-08 11:26:05,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +12: [2023-05-08 11:26:05,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:26:05,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:26:05,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:26:05,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +12: [2023-05-08 11:26:05,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:26:05,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 4: [2023-05-08 11:26:05,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 4: [2023-05-08 11:26:05,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 9: [2023-05-08 11:26:05,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:26:05,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:26:05,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 5: [2023-05-08 11:26:05,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 4: [2023-05-08 11:26:05,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 1: [2023-05-08 11:26:05,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +26: [2023-05-08 11:26:05,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:26:05,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:26:05,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +25: [2023-05-08 11:26:05,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +26: [2023-05-08 11:26:05,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +12: [2023-05-08 11:26:05,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +30: [2023-05-08 11:26:05,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:26:05,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +30: [2023-05-08 11:26:05,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:26:05,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:26:05,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +27: [2023-05-08 11:26:05,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:26:05,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:26:05,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +30: [2023-05-08 11:26:05,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 7: [2023-05-08 11:26:05,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +25: [2023-05-08 11:26:05,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +27: [2023-05-08 11:26:05,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:26:05,611] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +12: [2023-05-08 11:26:05,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +12: [2023-05-08 11:26:05,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +12: [2023-05-08 11:26:05,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,612] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +30: [2023-05-08 11:26:05,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +26: [2023-05-08 11:26:05,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:26:05,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +27: [2023-05-08 11:26:05,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +27: [2023-05-08 11:26:05,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +26: [2023-05-08 11:26:05,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +30: [2023-05-08 11:26:05,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +12: [2023-05-08 11:26:05,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +12: [2023-05-08 11:26:05,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,622] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,622] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:26:05,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:26:05,623] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +26: [2023-05-08 11:26:05,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,624] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +30: [2023-05-08 11:26:05,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +30: [2023-05-08 11:26:05,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,625] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:26:05,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +27: [2023-05-08 11:26:05,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +27: [2023-05-08 11:26:05,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +27: [2023-05-08 11:26:05,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:26:05,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:26:05,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 0: [2023-05-08 11:26:05,628] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 6: [2023-05-08 11:26:05,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:26:05,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:26:05,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +26: [2023-05-08 11:26:05,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:26:05,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:26:05,629] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +25: [2023-05-08 11:26:05,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +25: [2023-05-08 11:26:05,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 1: [2023-05-08 11:26:05,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:26:05,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:26:05,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:26:05,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +27: [2023-05-08 11:26:05,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +26: [2023-05-08 11:26:05,640] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +18: [2023-05-08 11:26:05,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:26:05,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:26:05,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:26:05,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:26:05,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:26:05,643] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:26:05,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:26:05,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +19: [2023-05-08 11:26:05,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +19: [2023-05-08 11:26:05,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +19: [2023-05-08 11:26:05,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +19: [2023-05-08 11:26:05,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +19: [2023-05-08 11:26:05,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +19: [2023-05-08 11:26:05,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +19: [2023-05-08 11:26:05,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +19: [2023-05-08 11:26:05,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +19: [2023-05-08 11:26:05,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +19: [2023-05-08 11:26:05,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +19: [2023-05-08 11:26:05,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +19: [2023-05-08 11:26:05,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +19: [2023-05-08 11:26:05,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +19: [2023-05-08 11:26:05,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +19: [2023-05-08 11:26:05,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +19: [2023-05-08 11:26:05,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +31: [2023-05-08 11:26:05,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +31: [2023-05-08 11:26:05,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +31: [2023-05-08 11:26:05,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +31: [2023-05-08 11:26:05,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +31: [2023-05-08 11:26:05,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:26:05,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +11: [2023-05-08 11:26:05,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +11: [2023-05-08 11:26:05,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +31: [2023-05-08 11:26:05,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +31: [2023-05-08 11:26:05,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +31: [2023-05-08 11:26:05,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +31: [2023-05-08 11:26:05,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:26:05,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +18: [2023-05-08 11:26:05,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +31: [2023-05-08 11:26:05,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +31: [2023-05-08 11:26:05,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +31: [2023-05-08 11:26:05,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +31: [2023-05-08 11:26:05,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +31: [2023-05-08 11:26:05,655] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +31: [2023-05-08 11:26:05,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +31: [2023-05-08 11:26:05,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +27: [2023-05-08 11:26:05,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:26:05,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:26:05,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:26:05,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:26:05,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:26:05,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +27: [2023-05-08 11:26:05,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +19: [2023-05-08 11:26:05,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +27: [2023-05-08 11:26:05,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +16: [2023-05-08 11:26:05,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +16: [2023-05-08 11:26:05,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +16: [2023-05-08 11:26:05,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +19: [2023-05-08 11:26:05,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:26:05,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:26:05,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +27: [2023-05-08 11:26:05,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:26:05,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +27: [2023-05-08 11:26:05,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +11: [2023-05-08 11:26:05,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:26:05,666] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +19: [2023-05-08 11:26:05,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:26:05,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +19: [2023-05-08 11:26:05,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:26:05,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:26:05,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +31: [2023-05-08 11:26:05,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +18: [2023-05-08 11:26:05,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:26:05,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +31: [2023-05-08 11:26:05,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +31: [2023-05-08 11:26:05,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 1: [2023-05-08 11:26:05,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 1: [2023-05-08 11:26:05,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +16: [2023-05-08 11:26:05,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +16: [2023-05-08 11:26:05,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +19: [2023-05-08 11:26:05,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +19: [2023-05-08 11:26:05,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +11: [2023-05-08 11:26:05,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:26:05,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +16: [2023-05-08 11:26:05,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +16: [2023-05-08 11:26:05,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +19: [2023-05-08 11:26:05,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +11: [2023-05-08 11:26:05,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:26:05,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:26:05,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +11: [2023-05-08 11:26:05,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +20: [2023-05-08 11:26:05,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +20: [2023-05-08 11:26:05,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +19: [2023-05-08 11:26:05,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +20: [2023-05-08 11:26:05,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +20: [2023-05-08 11:26:05,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +31: [2023-05-08 11:26:05,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +31: [2023-05-08 11:26:05,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +31: [2023-05-08 11:26:05,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,682] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:26:05,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +18: [2023-05-08 11:26:05,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +18: [2023-05-08 11:26:05,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +18: [2023-05-08 11:26:05,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +16: [2023-05-08 11:26:05,686] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:26:05,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +20: [2023-05-08 11:26:05,687] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +20: [2023-05-08 11:26:05,688] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +31: [2023-05-08 11:26:05,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:26:05,689] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:26:05,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +20: [2023-05-08 11:26:05,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +20: [2023-05-08 11:26:05,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:26:05,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:26:05,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:26:05,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +18: [2023-05-08 11:26:05,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +11: [2023-05-08 11:26:05,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +11: [2023-05-08 11:26:05,696] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 1: [2023-05-08 11:26:05,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +31: [2023-05-08 11:26:05,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 1: [2023-05-08 11:26:05,698] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +13: [2023-05-08 11:26:05,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +13: [2023-05-08 11:26:05,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +13: [2023-05-08 11:26:05,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +13: [2023-05-08 11:26:05,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +14: [2023-05-08 11:26:05,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +14: [2023-05-08 11:26:05,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +11: [2023-05-08 11:26:05,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +11: [2023-05-08 11:26:05,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +11: [2023-05-08 11:26:05,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +11: [2023-05-08 11:26:05,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +11: [2023-05-08 11:26:05,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +11: [2023-05-08 11:26:05,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +14: [2023-05-08 11:26:05,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +11: [2023-05-08 11:26:05,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:26:05,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +20: [2023-05-08 11:26:05,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:26:05,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +22: [2023-05-08 11:26:05,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +22: [2023-05-08 11:26:05,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +20: [2023-05-08 11:26:05,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:26:05,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +20: [2023-05-08 11:26:05,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +20: [2023-05-08 11:26:05,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 8: [2023-05-08 11:26:05,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,705] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +29: [2023-05-08 11:26:05,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +29: [2023-05-08 11:26:05,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +29: [2023-05-08 11:26:05,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +11: [2023-05-08 11:26:05,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +29: [2023-05-08 11:26:05,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 8: [2023-05-08 11:26:05,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +16: [2023-05-08 11:26:05,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:26:05,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:26:05,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +17: [2023-05-08 11:26:05,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +17: [2023-05-08 11:26:05,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +12: [2023-05-08 11:26:05,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +17: [2023-05-08 11:26:05,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +12: [2023-05-08 11:26:05,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +12: [2023-05-08 11:26:05,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +18: [2023-05-08 11:26:05,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:26:05,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +18: [2023-05-08 11:26:05,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +13: [2023-05-08 11:26:05,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +30: [2023-05-08 11:26:05,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +30: [2023-05-08 11:26:05,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +30: [2023-05-08 11:26:05,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +30: [2023-05-08 11:26:05,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +13: [2023-05-08 11:26:05,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:26:05,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:26:05,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +23: [2023-05-08 11:26:05,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +23: [2023-05-08 11:26:05,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +16: [2023-05-08 11:26:05,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +16: [2023-05-08 11:26:05,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +23: [2023-05-08 11:26:05,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +13: [2023-05-08 11:26:05,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +20: [2023-05-08 11:26:05,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:26:05,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +23: [2023-05-08 11:26:05,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 9: [2023-05-08 11:26:05,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 9: [2023-05-08 11:26:05,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 9: [2023-05-08 11:26:05,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:26:05,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +14: [2023-05-08 11:26:05,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:26:05,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +28: [2023-05-08 11:26:05,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +14: [2023-05-08 11:26:05,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:26:05,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +13: [2023-05-08 11:26:05,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,712] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +18: [2023-05-08 11:26:05,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:26:05,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +22: [2023-05-08 11:26:05,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +22: [2023-05-08 11:26:05,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:26:05,713] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:26:05,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:26:05,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +14: [2023-05-08 11:26:05,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:26:05,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +15: [2023-05-08 11:26:05,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +15: [2023-05-08 11:26:05,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +14: [2023-05-08 11:26:05,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:26:05,714] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +29: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +28: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +28: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +28: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +28: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +11: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +14: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +14: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +17: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +14: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +28: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 8: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 8: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 8: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +28: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:26:05,715] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 5: [2023-05-08 11:26:05,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +12: [2023-05-08 11:26:05,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +29: [2023-05-08 11:26:05,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +29: [2023-05-08 11:26:05,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:26:05,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:26:05,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:26:05,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:26:05,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:26:05,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:26:05,716] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:26:05,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:26:05,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +13: [2023-05-08 11:26:05,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:26:05,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:26:05,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +13: [2023-05-08 11:26:05,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:26:05,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +13: [2023-05-08 11:26:05,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:26:05,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:26:05,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +13: [2023-05-08 11:26:05,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:26:05,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:26:05,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +13: [2023-05-08 11:26:05,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:26:05,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:26:05,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +13: [2023-05-08 11:26:05,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:26:05,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:26:05,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +10: [2023-05-08 11:26:05,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +21: [2023-05-08 11:26:05,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:26:05,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:26:05,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +10: [2023-05-08 11:26:05,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +22: [2023-05-08 11:26:05,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +22: [2023-05-08 11:26:05,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:26:05,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +30: [2023-05-08 11:26:05,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +21: [2023-05-08 11:26:05,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +23: [2023-05-08 11:26:05,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 2: [2023-05-08 11:26:05,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +17: [2023-05-08 11:26:05,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +12: [2023-05-08 11:26:05,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:26:05,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +30: [2023-05-08 11:26:05,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +30: [2023-05-08 11:26:05,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +30: [2023-05-08 11:26:05,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +23: [2023-05-08 11:26:05,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +23: [2023-05-08 11:26:05,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +10: [2023-05-08 11:26:05,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,722] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +28: [2023-05-08 11:26:05,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 9: [2023-05-08 11:26:05,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. + 0: [2023-05-08 11:26:05,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +28: [2023-05-08 11:26:05,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:26:05,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +22: [2023-05-08 11:26:05,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:26:05,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:26:05,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:26:05,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +22: [2023-05-08 11:26:05,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +22: [2023-05-08 11:26:05,723] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:26:05,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:26:05,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +15: [2023-05-08 11:26:05,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:26:05,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +28: [2023-05-08 11:26:05,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +22: [2023-05-08 11:26:05,724] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:26:05,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +28: [2023-05-08 11:26:05,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:26:05,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt. +11: [2023-05-08 11:26:05,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:26:05,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,726] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +29: [2023-05-08 11:26:05,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +13: [2023-05-08 11:26:05,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:26:05,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +22: [2023-05-08 11:26:05,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +13: [2023-05-08 11:26:05,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 5: [2023-05-08 11:26:05,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +12: [2023-05-08 11:26:05,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +29: [2023-05-08 11:26:05,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +12: [2023-05-08 11:26:05,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 5: [2023-05-08 11:26:05,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:26:05,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +13: [2023-05-08 11:26:05,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 6: [2023-05-08 11:26:05,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 8: [2023-05-08 11:26:05,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:26:05,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +10: [2023-05-08 11:26:05,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +14: [2023-05-08 11:26:05,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:26:05,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +13: [2023-05-08 11:26:05,730] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,730] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +13: [2023-05-08 11:26:05,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +14: [2023-05-08 11:26:05,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:26:05,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +30: [2023-05-08 11:26:05,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 8: [2023-05-08 11:26:05,731] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:26:05,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +17: [2023-05-08 11:26:05,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:26:05,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 8: [2023-05-08 11:26:05,732] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:26:05,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:26:05,733] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +17: [2023-05-08 11:26:05,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +14: [2023-05-08 11:26:05,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:26:05,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +28: [2023-05-08 11:26:05,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:26:05,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 2: [2023-05-08 11:26:05,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:26:05,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 2: [2023-05-08 11:26:05,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +28: [2023-05-08 11:26:05,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:26:05,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:26:05,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:26:05,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:26:05,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +21: [2023-05-08 11:26:05,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:26:05,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +29: [2023-05-08 11:26:05,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +14: [2023-05-08 11:26:05,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:26:05,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +12: [2023-05-08 11:26:05,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +23: [2023-05-08 11:26:05,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +14: [2023-05-08 11:26:05,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +13: [2023-05-08 11:26:05,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:26:05,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +13: [2023-05-08 11:26:05,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +17: [2023-05-08 11:26:05,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:26:05,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +17: [2023-05-08 11:26:05,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:26:05,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 0: [2023-05-08 11:26:05,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +11: [2023-05-08 11:26:05,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:26:05,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:26:05,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:26:05,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:26:05,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +23: [2023-05-08 11:26:05,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +13: [2023-05-08 11:26:05,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... +30: [2023-05-08 11:26:05,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:26:05,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +15: [2023-05-08 11:26:05,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +30: [2023-05-08 11:26:05,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +15: [2023-05-08 11:26:05,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +15: [2023-05-08 11:26:05,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +15: [2023-05-08 11:26:05,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +13: [2023-05-08 11:26:05,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:26:05,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:26:05,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 6: [2023-05-08 11:26:05,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 8: [2023-05-08 11:26:05,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:26:05,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +13: [2023-05-08 11:26:05,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +13: [2023-05-08 11:26:05,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:26:05,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +17: [2023-05-08 11:26:05,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +15: [2023-05-08 11:26:05,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +15: [2023-05-08 11:26:05,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +14: [2023-05-08 11:26:05,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:26:05,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +15: [2023-05-08 11:26:05,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +30: [2023-05-08 11:26:05,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +10: [2023-05-08 11:26:05,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +12: [2023-05-08 11:26:05,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +10: [2023-05-08 11:26:05,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +10: [2023-05-08 11:26:05,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 5: [2023-05-08 11:26:05,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +30: [2023-05-08 11:26:05,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +14: [2023-05-08 11:26:05,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +13: [2023-05-08 11:26:05,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +12: [2023-05-08 11:26:05,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +22: [2023-05-08 11:26:05,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:26:05,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 5: [2023-05-08 11:26:05,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +28: [2023-05-08 11:26:05,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:26:05,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +28: [2023-05-08 11:26:05,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +13: [2023-05-08 11:26:05,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:26:05,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +11: [2023-05-08 11:26:05,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:26:05,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:26:05,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:26:05,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +29: [2023-05-08 11:26:05,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +29: [2023-05-08 11:26:05,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +29: [2023-05-08 11:26:05,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +29: [2023-05-08 11:26:05,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:26:05,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:26:05,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +29: [2023-05-08 11:26:05,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:26:05,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:26:05,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:26:05,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +12: [2023-05-08 11:26:05,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:26:05,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +17: [2023-05-08 11:26:05,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +28: [2023-05-08 11:26:05,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:26:05,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:26:05,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +10: [2023-05-08 11:26:05,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +14: [2023-05-08 11:26:05,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:26:05,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +10: [2023-05-08 11:26:05,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +10: [2023-05-08 11:26:05,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +10: [2023-05-08 11:26:05,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +10: [2023-05-08 11:26:05,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +12: [2023-05-08 11:26:05,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 8: [2023-05-08 11:26:05,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +13: [2023-05-08 11:26:05,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +10: [2023-05-08 11:26:05,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:26:05,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +10: [2023-05-08 11:26:05,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +28: [2023-05-08 11:26:05,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:26:05,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +14: [2023-05-08 11:26:05,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +28: [2023-05-08 11:26:05,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 0: [2023-05-08 11:26:05,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 9: [2023-05-08 11:26:05,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +23: [2023-05-08 11:26:05,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +17: [2023-05-08 11:26:05,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +13: [2023-05-08 11:26:05,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:26:05,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:26:05,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 9: [2023-05-08 11:26:05,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +23: [2023-05-08 11:26:05,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 8: [2023-05-08 11:26:05,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 9: [2023-05-08 11:26:05,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +14: [2023-05-08 11:26:05,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +17: [2023-05-08 11:26:05,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,753] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:26:05,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +28: [2023-05-08 11:26:05,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +10: [2023-05-08 11:26:05,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +17: [2023-05-08 11:26:05,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 8: [2023-05-08 11:26:05,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 5: [2023-05-08 11:26:05,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 2: [2023-05-08 11:26:05,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +17: [2023-05-08 11:26:05,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +10: [2023-05-08 11:26:05,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +21: [2023-05-08 11:26:05,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +17: [2023-05-08 11:26:05,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +22: [2023-05-08 11:26:05,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:26:05,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:26:05,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:26:05,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:26:05,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +16: [2023-05-08 11:26:05,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +16: [2023-05-08 11:26:05,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +16: [2023-05-08 11:26:05,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:26:05,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:26:05,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +28: [2023-05-08 11:26:05,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:26:05,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +28: [2023-05-08 11:26:05,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +16: [2023-05-08 11:26:05,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +29: [2023-05-08 11:26:05,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:26:05,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... + 2: [2023-05-08 11:26:05,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:26:05,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:26:05,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:26:05,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:26:05,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:26:05,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:26:05,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:26:05,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:26:05,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +10: [2023-05-08 11:26:05,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +28: [2023-05-08 11:26:05,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:26:05,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:26:05,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +10: [2023-05-08 11:26:05,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +29: [2023-05-08 11:26:05,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +29: [2023-05-08 11:26:05,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +28: [2023-05-08 11:26:05,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt. +22: [2023-05-08 11:26:05,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +28: [2023-05-08 11:26:05,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +29: [2023-05-08 11:26:05,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +28: [2023-05-08 11:26:05,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt... +10: [2023-05-08 11:26:05,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,773] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +10: [2023-05-08 11:26:05,773] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +29: [2023-05-08 11:26:05,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +29: [2023-05-08 11:26:05,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:26:05,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:26:05,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:26:05,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:26:05,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:26:05,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:26:05,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +10: [2023-05-08 11:26:05,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:26:05,779] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +29: [2023-05-08 11:26:05,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +23: [2023-05-08 11:26:05,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:26:05,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:26:05,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,787] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +23: [2023-05-08 11:26:05,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:26:05,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:26:05,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:26:05,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:26:05,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +23: [2023-05-08 11:26:05,790] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:26:05,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +24: [2023-05-08 11:26:05,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 3: [2023-05-08 11:26:05,794] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:26:05,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:26:05,796] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 3: [2023-05-08 11:26:05,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +24: [2023-05-08 11:26:05,797] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +16: [2023-05-08 11:26:05,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +24: [2023-05-08 11:26:05,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:26:05,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +25: [2023-05-08 11:26:05,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +25: [2023-05-08 11:26:05,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +25: [2023-05-08 11:26:05,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +24: [2023-05-08 11:26:05,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:26:05,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +21: [2023-05-08 11:26:05,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +21: [2023-05-08 11:26:05,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +21: [2023-05-08 11:26:05,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +24: [2023-05-08 11:26:05,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 7: [2023-05-08 11:26:05,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 7: [2023-05-08 11:26:05,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 3: [2023-05-08 11:26:05,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 3: [2023-05-08 11:26:05,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +24: [2023-05-08 11:26:05,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 7: [2023-05-08 11:26:05,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 7: [2023-05-08 11:26:05,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 4: [2023-05-08 11:26:05,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 4: [2023-05-08 11:26:05,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +26: [2023-05-08 11:26:05,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +26: [2023-05-08 11:26:05,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +26: [2023-05-08 11:26:05,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 4: [2023-05-08 11:26:05,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +26: [2023-05-08 11:26:05,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 4: [2023-05-08 11:26:05,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +21: [2023-05-08 11:26:05,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:26:05,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:26:05,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:26:05,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:26:05,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:26:05,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:26:05,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,820] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:26:05,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +31: [2023-05-08 11:26:05,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +31: [2023-05-08 11:26:05,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 7: [2023-05-08 11:26:05,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +31: [2023-05-08 11:26:05,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 7: [2023-05-08 11:26:05,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +19: [2023-05-08 11:26:05,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +19: [2023-05-08 11:26:05,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +19: [2023-05-08 11:26:05,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 3: [2023-05-08 11:26:05,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:26:05,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +11: [2023-05-08 11:26:05,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +11: [2023-05-08 11:26:05,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +11: [2023-05-08 11:26:05,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +26: [2023-05-08 11:26:05,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:26:05,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +19: [2023-05-08 11:26:05,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +11: [2023-05-08 11:26:05,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +26: [2023-05-08 11:26:05,825] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +31: [2023-05-08 11:26:05,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +26: [2023-05-08 11:26:05,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +26: [2023-05-08 11:26:05,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:26:05,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +27: [2023-05-08 11:26:05,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +27: [2023-05-08 11:26:05,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +27: [2023-05-08 11:26:05,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +24: [2023-05-08 11:26:05,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:26:05,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +21: [2023-05-08 11:26:05,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:26:05,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +12: [2023-05-08 11:26:05,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:26:05,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +12: [2023-05-08 11:26:05,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:26:05,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:26:05,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:26:05,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:26:05,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +12: [2023-05-08 11:26:05,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:26:05,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:26:05,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:26:05,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +12: [2023-05-08 11:26:05,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:26:05,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:26:05,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +31: [2023-05-08 11:26:05,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:26:05,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:26:05,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +15: [2023-05-08 11:26:05,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +31: [2023-05-08 11:26:05,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:26:05,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +19: [2023-05-08 11:26:05,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:26:05,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:26:05,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +16: [2023-05-08 11:26:05,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +16: [2023-05-08 11:26:05,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +16: [2023-05-08 11:26:05,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +25: [2023-05-08 11:26:05,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +25: [2023-05-08 11:26:05,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +25: [2023-05-08 11:26:05,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +25: [2023-05-08 11:26:05,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +19: [2023-05-08 11:26:05,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +16: [2023-05-08 11:26:05,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +19: [2023-05-08 11:26:05,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +11: [2023-05-08 11:26:05,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:26:05,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:26:05,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +11: [2023-05-08 11:26:05,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:26:05,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +31: [2023-05-08 11:26:05,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +21: [2023-05-08 11:26:05,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:26:05,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 7: [2023-05-08 11:26:05,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +31: [2023-05-08 11:26:05,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:26:05,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:26:05,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 7: [2023-05-08 11:26:05,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 7: [2023-05-08 11:26:05,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +25: [2023-05-08 11:26:05,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:26:05,836] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +26: [2023-05-08 11:26:05,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:26:05,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +26: [2023-05-08 11:26:05,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +26: [2023-05-08 11:26:05,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +26: [2023-05-08 11:26:05,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +25: [2023-05-08 11:26:05,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:26:05,838] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:26:05,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +25: [2023-05-08 11:26:05,839] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +27: [2023-05-08 11:26:05,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:26:05,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:26:05,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,841] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:26:05,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:26:05,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:26:05,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 3: [2023-05-08 11:26:05,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 3: [2023-05-08 11:26:05,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 3: [2023-05-08 11:26:05,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 7: [2023-05-08 11:26:05,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +25: [2023-05-08 11:26:05,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +12: [2023-05-08 11:26:05,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +26: [2023-05-08 11:26:05,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +26: [2023-05-08 11:26:05,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +16: [2023-05-08 11:26:05,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:26:05,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:26:05,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +16: [2023-05-08 11:26:05,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +16: [2023-05-08 11:26:05,846] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:26:05,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 7: [2023-05-08 11:26:05,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +11: [2023-05-08 11:26:05,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +11: [2023-05-08 11:26:05,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +24: [2023-05-08 11:26:05,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 7: [2023-05-08 11:26:05,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +15: [2023-05-08 11:26:05,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:26:05,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,849] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +25: [2023-05-08 11:26:05,849] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +11: [2023-05-08 11:26:05,850] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +11: [2023-05-08 11:26:05,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +21: [2023-05-08 11:26:05,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +25: [2023-05-08 11:26:05,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +12: [2023-05-08 11:26:05,853] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:26:05,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:26:05,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:26:05,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +26: [2023-05-08 11:26:05,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:26:05,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:26:05,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +12: [2023-05-08 11:26:05,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:26:05,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:26:05,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +26: [2023-05-08 11:26:05,856] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 3: [2023-05-08 11:26:05,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 1: [2023-05-08 11:26:05,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,857] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +25: [2023-05-08 11:26:05,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:26:05,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 1: [2023-05-08 11:26:05,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:26:05,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:26:05,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:26:05,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:26:05,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +27: [2023-05-08 11:26:05,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +16: [2023-05-08 11:26:05,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +27: [2023-05-08 11:26:05,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:26:05,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 6: [2023-05-08 11:26:05,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +25: [2023-05-08 11:26:05,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +11: [2023-05-08 11:26:05,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:26:05,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:26:05,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +24: [2023-05-08 11:26:05,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:26:05,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:26:05,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +24: [2023-05-08 11:26:05,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:26:05,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +24: [2023-05-08 11:26:05,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:26:05,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +12: [2023-05-08 11:26:05,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:26:05,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +16: [2023-05-08 11:26:05,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:26:05,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +24: [2023-05-08 11:26:05,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +12: [2023-05-08 11:26:05,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +12: [2023-05-08 11:26:05,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:26:05,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:26:05,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +25: [2023-05-08 11:26:05,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:26:05,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 4: [2023-05-08 11:26:05,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 4: [2023-05-08 11:26:05,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 4: [2023-05-08 11:26:05,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 4: [2023-05-08 11:26:05,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 4: [2023-05-08 11:26:05,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +16: [2023-05-08 11:26:05,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +16: [2023-05-08 11:26:05,870] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 6: [2023-05-08 11:26:05,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,872] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +19: [2023-05-08 11:26:05,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +19: [2023-05-08 11:26:05,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +19: [2023-05-08 11:26:05,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +19: [2023-05-08 11:26:05,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +19: [2023-05-08 11:26:05,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +30: [2023-05-08 11:26:05,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:26:05,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +19: [2023-05-08 11:26:05,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +30: [2023-05-08 11:26:05,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:26:05,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +30: [2023-05-08 11:26:05,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +19: [2023-05-08 11:26:05,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:26:05,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +19: [2023-05-08 11:26:05,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +19: [2023-05-08 11:26:05,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +19: [2023-05-08 11:26:05,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +19: [2023-05-08 11:26:05,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:26:05,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +19: [2023-05-08 11:26:05,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +19: [2023-05-08 11:26:05,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +19: [2023-05-08 11:26:05,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +19: [2023-05-08 11:26:05,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +24: [2023-05-08 11:26:05,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:26:05,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +25: [2023-05-08 11:26:05,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:26:05,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:26:05,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:26:05,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +19: [2023-05-08 11:26:05,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,882] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +24: [2023-05-08 11:26:05,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:26:05,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:26:05,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +30: [2023-05-08 11:26:05,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +30: [2023-05-08 11:26:05,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +30: [2023-05-08 11:26:05,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +29: [2023-05-08 11:26:05,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +29: [2023-05-08 11:26:05,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +29: [2023-05-08 11:26:05,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +29: [2023-05-08 11:26:05,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 4: [2023-05-08 11:26:05,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,884] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:26:05,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +19: [2023-05-08 11:26:05,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +19: [2023-05-08 11:26:05,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:26:05,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:26:05,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:26:05,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:26:05,887] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:26:05,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:26:05,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:26:05,888] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +19: [2023-05-08 11:26:05,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:26:05,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +18: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +27: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +27: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +18: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +18: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +18: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +27: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +18: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +18: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +18: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 1: [2023-05-08 11:26:05,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +18: [2023-05-08 11:26:05,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:26:05,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:26:05,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:26:05,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:26:05,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,891] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:26:05,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:26:05,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +24: [2023-05-08 11:26:05,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +30: [2023-05-08 11:26:05,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,893] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +29: [2023-05-08 11:26:05,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +12: [2023-05-08 11:26:05,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 1: [2023-05-08 11:26:05,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:26:05,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +29: [2023-05-08 11:26:05,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +24: [2023-05-08 11:26:05,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +12: [2023-05-08 11:26:05,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +12: [2023-05-08 11:26:05,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +29: [2023-05-08 11:26:05,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +30: [2023-05-08 11:26:05,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,896] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:26:05,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +19: [2023-05-08 11:26:05,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:26:05,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +24: [2023-05-08 11:26:05,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:26:05,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +19: [2023-05-08 11:26:05,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +19: [2023-05-08 11:26:05,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 1: [2023-05-08 11:26:05,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +18: [2023-05-08 11:26:05,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:26:05,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 6: [2023-05-08 11:26:05,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 6: [2023-05-08 11:26:05,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +30: [2023-05-08 11:26:05,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:26:05,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 6: [2023-05-08 11:26:05,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:26:05,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:26:05,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +12: [2023-05-08 11:26:05,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +18: [2023-05-08 11:26:05,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +18: [2023-05-08 11:26:05,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:26:05,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +31: [2023-05-08 11:26:05,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +31: [2023-05-08 11:26:05,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +31: [2023-05-08 11:26:05,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +31: [2023-05-08 11:26:05,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +31: [2023-05-08 11:26:05,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +31: [2023-05-08 11:26:05,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +11: [2023-05-08 11:26:05,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:26:05,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:26:05,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +31: [2023-05-08 11:26:05,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +31: [2023-05-08 11:26:05,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +31: [2023-05-08 11:26:05,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +31: [2023-05-08 11:26:05,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +31: [2023-05-08 11:26:05,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 8: [2023-05-08 11:26:05,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:26:05,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +12: [2023-05-08 11:26:05,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +31: [2023-05-08 11:26:05,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +22: [2023-05-08 11:26:05,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 5: [2023-05-08 11:26:05,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +31: [2023-05-08 11:26:05,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +22: [2023-05-08 11:26:05,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +17: [2023-05-08 11:26:05,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +17: [2023-05-08 11:26:05,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +17: [2023-05-08 11:26:05,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +17: [2023-05-08 11:26:05,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +22: [2023-05-08 11:26:05,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 5: [2023-05-08 11:26:05,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +12: [2023-05-08 11:26:05,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +22: [2023-05-08 11:26:05,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +24: [2023-05-08 11:26:05,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:26:05,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 4: [2023-05-08 11:26:05,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +19: [2023-05-08 11:26:05,908] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:26:05,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +14: [2023-05-08 11:26:05,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +14: [2023-05-08 11:26:05,909] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +30: [2023-05-08 11:26:05,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:26:05,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:26:05,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:26:05,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:26:05,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +30: [2023-05-08 11:26:05,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +27: [2023-05-08 11:26:05,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:26:05,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +29: [2023-05-08 11:26:05,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:26:05,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +14: [2023-05-08 11:26:05,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +11: [2023-05-08 11:26:05,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +18: [2023-05-08 11:26:05,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +18: [2023-05-08 11:26:05,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +18: [2023-05-08 11:26:05,914] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +18: [2023-05-08 11:26:05,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +29: [2023-05-08 11:26:05,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +29: [2023-05-08 11:26:05,915] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +11: [2023-05-08 11:26:05,915] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:26:05,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:26:05,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +17: [2023-05-08 11:26:05,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +27: [2023-05-08 11:26:05,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:26:05,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +19: [2023-05-08 11:26:05,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +17: [2023-05-08 11:26:05,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +22: [2023-05-08 11:26:05,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:26:05,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:26:05,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 6: [2023-05-08 11:26:05,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:26:05,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +22: [2023-05-08 11:26:05,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +22: [2023-05-08 11:26:05,918] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +11: [2023-05-08 11:26:05,918] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:26:05,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +14: [2023-05-08 11:26:05,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:26:05,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +30: [2023-05-08 11:26:05,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +30: [2023-05-08 11:26:05,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +20: [2023-05-08 11:26:05,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:26:05,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +31: [2023-05-08 11:26:05,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +20: [2023-05-08 11:26:05,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +30: [2023-05-08 11:26:05,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:26:05,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:26:05,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +17: [2023-05-08 11:26:05,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +17: [2023-05-08 11:26:05,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:26:05,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +17: [2023-05-08 11:26:05,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:26:05,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +17: [2023-05-08 11:26:05,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +17: [2023-05-08 11:26:05,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +17: [2023-05-08 11:26:05,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:26:05,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:26:05,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +18: [2023-05-08 11:26:05,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:26:05,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +29: [2023-05-08 11:26:05,924] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +11: [2023-05-08 11:26:05,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +20: [2023-05-08 11:26:05,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +27: [2023-05-08 11:26:05,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +29: [2023-05-08 11:26:05,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +28: [2023-05-08 11:26:05,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +28: [2023-05-08 11:26:05,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +28: [2023-05-08 11:26:05,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +10: [2023-05-08 11:26:05,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:26:05,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +10: [2023-05-08 11:26:05,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +28: [2023-05-08 11:26:05,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 5: [2023-05-08 11:26:05,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 5: [2023-05-08 11:26:05,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 6: [2023-05-08 11:26:05,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:26:05,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +29: [2023-05-08 11:26:05,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 6: [2023-05-08 11:26:05,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +12: [2023-05-08 11:26:05,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:26:05,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:26:05,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +12: [2023-05-08 11:26:05,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +10: [2023-05-08 11:26:05,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +30: [2023-05-08 11:26:05,927] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 2: [2023-05-08 11:26:05,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 2: [2023-05-08 11:26:05,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +22: [2023-05-08 11:26:05,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:26:05,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +30: [2023-05-08 11:26:05,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 2: [2023-05-08 11:26:05,929] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +29: [2023-05-08 11:26:05,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +31: [2023-05-08 11:26:05,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +13: [2023-05-08 11:26:05,930] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 8: [2023-05-08 11:26:05,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:26:05,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +13: [2023-05-08 11:26:05,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +31: [2023-05-08 11:26:05,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:26:05,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:26:05,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +15: [2023-05-08 11:26:05,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +13: [2023-05-08 11:26:05,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +15: [2023-05-08 11:26:05,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 9: [2023-05-08 11:26:05,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +15: [2023-05-08 11:26:05,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 9: [2023-05-08 11:26:05,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +17: [2023-05-08 11:26:05,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:26:05,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +12: [2023-05-08 11:26:05,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +31: [2023-05-08 11:26:05,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:26:05,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 9: [2023-05-08 11:26:05,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +17: [2023-05-08 11:26:05,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +17: [2023-05-08 11:26:05,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +20: [2023-05-08 11:26:05,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:26:05,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:26:05,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +22: [2023-05-08 11:26:05,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:26:05,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +14: [2023-05-08 11:26:05,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +14: [2023-05-08 11:26:05,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +14: [2023-05-08 11:26:05,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +14: [2023-05-08 11:26:05,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +14: [2023-05-08 11:26:05,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +14: [2023-05-08 11:26:05,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +28: [2023-05-08 11:26:05,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +13: [2023-05-08 11:26:05,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +13: [2023-05-08 11:26:05,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +13: [2023-05-08 11:26:05,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +13: [2023-05-08 11:26:05,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +31: [2023-05-08 11:26:05,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +13: [2023-05-08 11:26:05,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +17: [2023-05-08 11:26:05,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:26:05,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:26:05,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +28: [2023-05-08 11:26:05,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +13: [2023-05-08 11:26:05,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +28: [2023-05-08 11:26:05,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 8: [2023-05-08 11:26:05,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 5: [2023-05-08 11:26:05,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +10: [2023-05-08 11:26:05,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +13: [2023-05-08 11:26:05,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +13: [2023-05-08 11:26:05,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:26:05,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:26:05,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:26:05,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:26:05,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +17: [2023-05-08 11:26:05,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 2: [2023-05-08 11:26:05,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +22: [2023-05-08 11:26:05,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +22: [2023-05-08 11:26:05,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +13: [2023-05-08 11:26:05,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:26:05,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 2: [2023-05-08 11:26:05,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:26:05,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +10: [2023-05-08 11:26:05,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +31: [2023-05-08 11:26:05,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:26:05,941] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +13: [2023-05-08 11:26:05,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +13: [2023-05-08 11:26:05,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +13: [2023-05-08 11:26:05,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:26:05,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:26:05,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +17: [2023-05-08 11:26:05,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:26:05,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:26:05,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +21: [2023-05-08 11:26:05,943] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:26:05,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:26:05,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:26:05,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:26:05,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:26:05,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +14: [2023-05-08 11:26:05,944] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 0: [2023-05-08 11:26:05,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 0: [2023-05-08 11:26:05,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. +21: [2023-05-08 11:26:05,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +31: [2023-05-08 11:26:05,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt. + 9: [2023-05-08 11:26:05,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +17: [2023-05-08 11:26:05,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +15: [2023-05-08 11:26:05,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +28: [2023-05-08 11:26:05,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +28: [2023-05-08 11:26:05,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 2: [2023-05-08 11:26:05,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +17: [2023-05-08 11:26:05,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:26:05,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 2: [2023-05-08 11:26:05,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +10: [2023-05-08 11:26:05,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +26: [2023-05-08 11:26:05,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +10: [2023-05-08 11:26:05,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +26: [2023-05-08 11:26:05,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +26: [2023-05-08 11:26:05,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:26:05,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +26: [2023-05-08 11:26:05,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +26: [2023-05-08 11:26:05,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +26: [2023-05-08 11:26:05,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +13: [2023-05-08 11:26:05,953] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +13: [2023-05-08 11:26:05,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 2: [2023-05-08 11:26:05,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,954] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 2: [2023-05-08 11:26:05,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +26: [2023-05-08 11:26:05,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +26: [2023-05-08 11:26:05,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +17: [2023-05-08 11:26:05,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +17: [2023-05-08 11:26:05,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:26:05,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 5: [2023-05-08 11:26:05,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:26:05,955] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:26:05,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 2: [2023-05-08 11:26:05,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 5: [2023-05-08 11:26:05,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 5: [2023-05-08 11:26:05,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 5: [2023-05-08 11:26:05,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 5: [2023-05-08 11:26:05,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +21: [2023-05-08 11:26:05,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:26:05,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:26:05,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +14: [2023-05-08 11:26:05,956] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:26:05,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:26:05,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:26:05,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:26:05,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:26:05,957] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +28: [2023-05-08 11:26:05,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 0: [2023-05-08 11:26:05,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +14: [2023-05-08 11:26:05,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:26:05,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +28: [2023-05-08 11:26:05,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +28: [2023-05-08 11:26:05,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +10: [2023-05-08 11:26:05,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +10: [2023-05-08 11:26:05,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 9: [2023-05-08 11:26:05,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +14: [2023-05-08 11:26:05,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:26:05,959] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 9: [2023-05-08 11:26:05,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +21: [2023-05-08 11:26:05,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +13: [2023-05-08 11:26:05,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:26:05,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +13: [2023-05-08 11:26:05,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +13: [2023-05-08 11:26:05,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +28: [2023-05-08 11:26:05,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +28: [2023-05-08 11:26:05,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 9: [2023-05-08 11:26:05,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 9: [2023-05-08 11:26:05,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 9: [2023-05-08 11:26:05,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 0: [2023-05-08 11:26:05,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt... +13: [2023-05-08 11:26:05,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +13: [2023-05-08 11:26:05,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 8: [2023-05-08 11:26:05,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +10: [2023-05-08 11:26:05,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 8: [2023-05-08 11:26:05,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +15: [2023-05-08 11:26:05,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:26:05,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 8: [2023-05-08 11:26:05,966] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +10: [2023-05-08 11:26:05,966] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +13: [2023-05-08 11:26:05,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:26:05,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 8: [2023-05-08 11:26:05,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +26: [2023-05-08 11:26:05,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:26:05,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +21: [2023-05-08 11:26:05,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:26:05,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +15: [2023-05-08 11:26:05,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +13: [2023-05-08 11:26:05,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +26: [2023-05-08 11:26:05,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:26:05,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:26:05,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:26:05,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. +13: [2023-05-08 11:26:05,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:26:05,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +28: [2023-05-08 11:26:05,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:26:05,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +28: [2023-05-08 11:26:05,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:26:05,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 0: [2023-05-08 11:26:05,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... + 0: [2023-05-08 11:26:05,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +13: [2023-05-08 11:26:05,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:26:05,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:26:05,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +26: [2023-05-08 11:26:05,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:26:05,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +28: [2023-05-08 11:26:05,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:26:05,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:26:05,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +28: [2023-05-08 11:26:05,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:26:05,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:26:05,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:26:05,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +26: [2023-05-08 11:26:05,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +26: [2023-05-08 11:26:05,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +26: [2023-05-08 11:26:05,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:26:05,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:26:05,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:26:05,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:26:05,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:26:05,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:26:05,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:26:05,988] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:26:05,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:26:05,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:26:05,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +28: [2023-05-08 11:26:05,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:26:05,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:26:05,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:26:05,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +26: [2023-05-08 11:26:05,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:26:05,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +26: [2023-05-08 11:26:05,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +28: [2023-05-08 11:26:05,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,995] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,996] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt. + 3: [2023-05-08 11:26:05,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 3: [2023-05-08 11:26:05,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:26:05,998] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:26:05,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt... +14: [2023-05-08 11:26:06,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +28: [2023-05-08 11:26:06,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:26:06,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +28: [2023-05-08 11:26:06,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:26:06,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +28: [2023-05-08 11:26:06,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:26:06,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:26:06,020] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,022] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:26:06,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +10: [2023-05-08 11:26:06,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +10: [2023-05-08 11:26:06,022] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +10: [2023-05-08 11:26:06,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:26:06,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:26:06,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:26:06,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +10: [2023-05-08 11:26:06,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:26:06,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:26:06,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +16: [2023-05-08 11:26:06,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +16: [2023-05-08 11:26:06,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +16: [2023-05-08 11:26:06,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:26:06,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +10: [2023-05-08 11:26:06,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:26:06,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:26:06,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:26:06,026] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:26:06,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +20: [2023-05-08 11:26:06,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +20: [2023-05-08 11:26:06,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 3: [2023-05-08 11:26:06,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:26:06,027] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +22: [2023-05-08 11:26:06,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:26:06,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:26:06,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:26:06,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:26:06,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:26:06,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +15: [2023-05-08 11:26:06,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +15: [2023-05-08 11:26:06,033] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:26:06,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:26:06,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:26:06,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:26:06,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:26:06,035] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,037] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +20: [2023-05-08 11:26:06,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:26:06,038] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:26:06,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +10: [2023-05-08 11:26:06,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:26:06,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +10: [2023-05-08 11:26:06,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:26:06,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +10: [2023-05-08 11:26:06,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +10: [2023-05-08 11:26:06,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +10: [2023-05-08 11:26:06,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,050] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:26:06,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:26:06,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:26:06,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:26:06,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:26:06,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:26:06,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:26:06,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +15: [2023-05-08 11:26:06,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:26:06,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:26:06,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:26:06,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +20: [2023-05-08 11:26:06,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +12: [2023-05-08 11:26:06,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +12: [2023-05-08 11:26:06,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:26:06,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,054] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:26:06,054] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +12: [2023-05-08 11:26:06,055] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:26:06,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:26:06,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +10: [2023-05-08 11:26:06,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:26:06,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:26:06,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:26:06,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:26:06,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:26:06,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:26:06,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:26:06,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,060] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +10: [2023-05-08 11:26:06,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:26:06,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:26:06,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:26:06,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:26:06,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +20: [2023-05-08 11:26:06,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,063] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +29: [2023-05-08 11:26:06,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +29: [2023-05-08 11:26:06,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +29: [2023-05-08 11:26:06,064] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +29: [2023-05-08 11:26:06,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +29: [2023-05-08 11:26:06,064] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +29: [2023-05-08 11:26:06,065] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:26:06,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +29: [2023-05-08 11:26:06,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:26:06,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:26:06,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 2: [2023-05-08 11:26:06,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +16: [2023-05-08 11:26:06,065] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +29: [2023-05-08 11:26:06,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +23: [2023-05-08 11:26:06,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:26:06,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +23: [2023-05-08 11:26:06,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:26:06,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:26:06,066] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:26:06,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +23: [2023-05-08 11:26:06,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +23: [2023-05-08 11:26:06,066] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:26:06,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:26:06,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:26:06,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:26:06,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +30: [2023-05-08 11:26:06,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:26:06,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:26:06,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:26:06,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:26:06,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:26:06,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:26:06,072] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:26:06,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:26:06,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:26:06,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:26:06,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +23: [2023-05-08 11:26:06,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:26:06,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:26:06,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +29: [2023-05-08 11:26:06,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +29: [2023-05-08 11:26:06,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +29: [2023-05-08 11:26:06,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:26:06,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:26:06,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:26:06,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:26:06,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:26:06,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:26:06,085] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:26:06,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:26:06,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +12: [2023-05-08 11:26:06,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:26:06,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +12: [2023-05-08 11:26:06,088] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +19: [2023-05-08 11:26:06,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:26:06,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:26:06,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +19: [2023-05-08 11:26:06,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:26:06,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +19: [2023-05-08 11:26:06,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +24: [2023-05-08 11:26:06,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:26:06,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:26:06,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +29: [2023-05-08 11:26:06,089] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,090] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +31: [2023-05-08 11:26:06,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:26:06,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:26:06,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +31: [2023-05-08 11:26:06,092] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +29: [2023-05-08 11:26:06,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +29: [2023-05-08 11:26:06,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +23: [2023-05-08 11:26:06,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +29: [2023-05-08 11:26:06,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +31: [2023-05-08 11:26:06,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:26:06,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:26:06,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +12: [2023-05-08 11:26:06,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:26:06,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:26:06,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:26:06,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:26:06,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:26:06,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +23: [2023-05-08 11:26:06,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +31: [2023-05-08 11:26:06,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:26:06,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:26:06,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:26:06,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,098] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +29: [2023-05-08 11:26:06,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:26:06,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:26:06,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:26:06,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:26:06,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:26:06,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:26:06,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:26:06,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:26:06,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:26:06,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +19: [2023-05-08 11:26:06,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:26:06,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +19: [2023-05-08 11:26:06,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +19: [2023-05-08 11:26:06,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:26:06,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:26:06,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:26:06,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +27: [2023-05-08 11:26:06,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:26:06,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:26:06,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:26:06,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:26:06,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:26:06,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:26:06,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:26:06,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:26:06,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:26:06,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:26:06,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:26:06,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:26:06,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:26:06,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:26:06,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:26:06,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:26:06,147] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:26:06,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +25: [2023-05-08 11:26:06,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:26:06,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:26:06,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:26:06,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:26:06,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:26:06,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:26:06,172] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:26:06,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:26:06,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:26:06,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:26:06,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:26:06,194] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:26:06,195] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +21: [2023-05-08 11:26:06,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +21: [2023-05-08 11:26:06,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +21: [2023-05-08 11:26:06,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +25: [2023-05-08 11:26:06,201] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +25: [2023-05-08 11:26:06,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +25: [2023-05-08 11:26:06,205] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +21: [2023-05-08 11:26:06,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:26:06,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:26:06,207] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:26:06,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +25: [2023-05-08 11:26:06,214] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 1: [2023-05-08 11:26:06,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 1: [2023-05-08 11:26:06,214] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 1: [2023-05-08 11:26:06,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +23: [2023-05-08 11:26:06,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +23: [2023-05-08 11:26:06,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +20: [2023-05-08 11:26:06,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:26:06,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:26:06,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:26:06,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +25: [2023-05-08 11:26:06,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +20: [2023-05-08 11:26:06,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +20: [2023-05-08 11:26:06,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +20: [2023-05-08 11:26:06,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:26:06,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +25: [2023-05-08 11:26:06,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +23: [2023-05-08 11:26:06,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +25: [2023-05-08 11:26:06,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +20: [2023-05-08 11:26:06,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:26:06,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +21: [2023-05-08 11:26:06,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:26:06,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:26:06,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 1: [2023-05-08 11:26:06,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:26:06,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +21: [2023-05-08 11:26:06,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:26:06,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +25: [2023-05-08 11:26:06,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:26:06,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:26:06,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:26:06,229] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:26:06,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:26:06,231] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +25: [2023-05-08 11:26:06,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +25: [2023-05-08 11:26:06,231] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +25: [2023-05-08 11:26:06,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +21: [2023-05-08 11:26:06,232] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +25: [2023-05-08 11:26:06,242] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +25: [2023-05-08 11:26:06,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 1: [2023-05-08 11:26:06,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 1: [2023-05-08 11:26:06,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 1: [2023-05-08 11:26:06,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +20: [2023-05-08 11:26:06,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:26:06,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:26:06,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:26:06,246] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:26:06,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +25: [2023-05-08 11:26:06,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +25: [2023-05-08 11:26:06,248] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +25: [2023-05-08 11:26:06,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +25: [2023-05-08 11:26:06,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +25: [2023-05-08 11:26:06,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +25: [2023-05-08 11:26:06,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 1: [2023-05-08 11:26:06,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:26:06,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:26:06,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:26:06,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 1: [2023-05-08 11:26:06,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +23: [2023-05-08 11:26:06,255] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +18: [2023-05-08 11:26:06,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +18: [2023-05-08 11:26:06,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +18: [2023-05-08 11:26:06,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +20: [2023-05-08 11:26:06,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:26:06,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:26:06,259] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:26:06,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +23: [2023-05-08 11:26:06,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +23: [2023-05-08 11:26:06,260] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +20: [2023-05-08 11:26:06,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 6: [2023-05-08 11:26:06,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 6: [2023-05-08 11:26:06,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 6: [2023-05-08 11:26:06,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +18: [2023-05-08 11:26:06,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +18: [2023-05-08 11:26:06,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +18: [2023-05-08 11:26:06,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +18: [2023-05-08 11:26:06,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:26:06,271] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,276] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +18: [2023-05-08 11:26:06,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +18: [2023-05-08 11:26:06,282] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +18: [2023-05-08 11:26:06,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +18: [2023-05-08 11:26:06,282] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +26: [2023-05-08 11:26:06,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +18: [2023-05-08 11:26:06,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +18: [2023-05-08 11:26:06,284] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +27: [2023-05-08 11:26:06,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +27: [2023-05-08 11:26:06,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +27: [2023-05-08 11:26:06,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +27: [2023-05-08 11:26:06,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +31: [2023-05-08 11:26:06,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +31: [2023-05-08 11:26:06,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +31: [2023-05-08 11:26:06,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +26: [2023-05-08 11:26:06,284] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +31: [2023-05-08 11:26:06,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 6: [2023-05-08 11:26:06,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:26:06,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +11: [2023-05-08 11:26:06,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +11: [2023-05-08 11:26:06,285] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 6: [2023-05-08 11:26:06,286] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +11: [2023-05-08 11:26:06,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +26: [2023-05-08 11:26:06,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 6: [2023-05-08 11:26:06,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,288] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +26: [2023-05-08 11:26:06,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +18: [2023-05-08 11:26:06,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +18: [2023-05-08 11:26:06,289] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +14: [2023-05-08 11:26:06,291] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +14: [2023-05-08 11:26:06,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:26:06,293] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +14: [2023-05-08 11:26:06,293] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:26:06,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:26:06,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:26:06,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +26: [2023-05-08 11:26:06,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +26: [2023-05-08 11:26:06,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:26:06,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:26:06,294] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:26:06,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +14: [2023-05-08 11:26:06,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +11: [2023-05-08 11:26:06,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:26:06,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:26:06,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:26:06,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:26:06,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:26:06,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +26: [2023-05-08 11:26:06,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +31: [2023-05-08 11:26:06,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +31: [2023-05-08 11:26:06,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:26:06,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +26: [2023-05-08 11:26:06,298] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:26:06,303] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,306] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 6: [2023-05-08 11:26:06,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +11: [2023-05-08 11:26:06,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:26:06,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +11: [2023-05-08 11:26:06,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +14: [2023-05-08 11:26:06,307] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +11: [2023-05-08 11:26:06,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:26:06,308] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +31: [2023-05-08 11:26:06,308] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:26:06,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +24: [2023-05-08 11:26:06,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +24: [2023-05-08 11:26:06,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 8: [2023-05-08 11:26:06,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,309] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +26: [2023-05-08 11:26:06,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:26:06,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:26:06,310] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:26:06,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +26: [2023-05-08 11:26:06,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +26: [2023-05-08 11:26:06,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +11: [2023-05-08 11:26:06,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +26: [2023-05-08 11:26:06,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:26:06,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 8: [2023-05-08 11:26:06,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:26:06,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +26: [2023-05-08 11:26:06,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +26: [2023-05-08 11:26:06,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +24: [2023-05-08 11:26:06,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +17: [2023-05-08 11:26:06,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:26:06,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +17: [2023-05-08 11:26:06,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:26:06,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +17: [2023-05-08 11:26:06,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +17: [2023-05-08 11:26:06,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:26:06,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +17: [2023-05-08 11:26:06,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:26:06,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +17: [2023-05-08 11:26:06,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,312] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:26:06,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +27: [2023-05-08 11:26:06,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +31: [2023-05-08 11:26:06,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:26:06,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +14: [2023-05-08 11:26:06,314] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:26:06,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:26:06,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 3: [2023-05-08 11:26:06,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 3: [2023-05-08 11:26:06,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 3: [2023-05-08 11:26:06,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +22: [2023-05-08 11:26:06,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +22: [2023-05-08 11:26:06,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +22: [2023-05-08 11:26:06,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +22: [2023-05-08 11:26:06,318] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +27: [2023-05-08 11:26:06,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +14: [2023-05-08 11:26:06,318] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:26:06,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +24: [2023-05-08 11:26:06,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:26:06,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +11: [2023-05-08 11:26:06,320] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +11: [2023-05-08 11:26:06,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +11: [2023-05-08 11:26:06,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:26:06,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:26:06,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +24: [2023-05-08 11:26:06,321] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +19: [2023-05-08 11:26:06,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +11: [2023-05-08 11:26:06,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +11: [2023-05-08 11:26:06,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +19: [2023-05-08 11:26:06,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +19: [2023-05-08 11:26:06,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +27: [2023-05-08 11:26:06,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:26:06,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +11: [2023-05-08 11:26:06,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:26:06,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:26:06,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +11: [2023-05-08 11:26:06,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:26:06,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 8: [2023-05-08 11:26:06,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,325] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:26:06,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 8: [2023-05-08 11:26:06,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:26:06,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:26:06,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:26:06,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +22: [2023-05-08 11:26:06,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:26:06,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:26:06,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +17: [2023-05-08 11:26:06,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +17: [2023-05-08 11:26:06,328] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,328] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:26:06,329] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 3: [2023-05-08 11:26:06,330] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,331] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:26:06,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:26:06,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:26:06,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +24: [2023-05-08 11:26:06,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +27: [2023-05-08 11:26:06,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +17: [2023-05-08 11:26:06,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:26:06,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +19: [2023-05-08 11:26:06,333] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +11: [2023-05-08 11:26:06,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,332] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:26:06,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +11: [2023-05-08 11:26:06,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,335] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +17: [2023-05-08 11:26:06,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +17: [2023-05-08 11:26:06,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +11: [2023-05-08 11:26:06,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:26:06,338] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:26:06,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:26:06,339] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 2: [2023-05-08 11:26:06,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,340] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +22: [2023-05-08 11:26:06,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:26:06,342] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +11: [2023-05-08 11:26:06,343] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:26:06,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:26:06,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:26:06,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +24: [2023-05-08 11:26:06,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +11: [2023-05-08 11:26:06,346] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:26:06,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:26:06,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:26:06,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:26:06,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:26:06,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +22: [2023-05-08 11:26:06,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +26: [2023-05-08 11:26:06,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:26:06,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,347] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:26:06,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +11: [2023-05-08 11:26:06,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:26:06,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +20: [2023-05-08 11:26:06,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +20: [2023-05-08 11:26:06,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +26: [2023-05-08 11:26:06,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:26:06,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +26: [2023-05-08 11:26:06,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:26:06,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,350] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:26:06,351] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:26:06,352] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +19: [2023-05-08 11:26:06,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:26:06,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:26:06,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +19: [2023-05-08 11:26:06,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +18: [2023-05-08 11:26:06,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +18: [2023-05-08 11:26:06,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +18: [2023-05-08 11:26:06,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +18: [2023-05-08 11:26:06,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +18: [2023-05-08 11:26:06,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +18: [2023-05-08 11:26:06,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +11: [2023-05-08 11:26:06,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +18: [2023-05-08 11:26:06,355] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,355] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:26:06,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +19: [2023-05-08 11:26:06,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +19: [2023-05-08 11:26:06,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:26:06,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 1: [2023-05-08 11:26:06,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:26:06,359] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +20: [2023-05-08 11:26:06,360] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:26:06,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +26: [2023-05-08 11:26:06,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:26:06,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:26:06,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:26:06,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 3: [2023-05-08 11:26:06,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +11: [2023-05-08 11:26:06,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:26:06,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:26:06,366] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,370] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +18: [2023-05-08 11:26:06,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +18: [2023-05-08 11:26:06,370] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:26:06,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,371] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:26:06,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +26: [2023-05-08 11:26:06,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,372] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:26:06,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +30: [2023-05-08 11:26:06,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +30: [2023-05-08 11:26:06,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +30: [2023-05-08 11:26:06,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 3: [2023-05-08 11:26:06,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +18: [2023-05-08 11:26:06,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:26:06,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:26:06,379] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:26:06,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:26:06,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:26:06,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:26:06,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +23: [2023-05-08 11:26:06,381] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +23: [2023-05-08 11:26:06,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +23: [2023-05-08 11:26:06,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +23: [2023-05-08 11:26:06,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +23: [2023-05-08 11:26:06,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +23: [2023-05-08 11:26:06,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:26:06,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:26:06,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:26:06,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:26:06,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:26:06,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:26:06,382] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:26:06,383] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:26:06,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +23: [2023-05-08 11:26:06,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:26:06,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:26:06,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:26:06,384] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +30: [2023-05-08 11:26:06,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +20: [2023-05-08 11:26:06,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:26:06,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +30: [2023-05-08 11:26:06,386] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:26:06,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +10: [2023-05-08 11:26:06,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +10: [2023-05-08 11:26:06,386] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +30: [2023-05-08 11:26:06,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:26:06,387] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +10: [2023-05-08 11:26:06,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +16: [2023-05-08 11:26:06,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 4: [2023-05-08 11:26:06,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +16: [2023-05-08 11:26:06,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 4: [2023-05-08 11:26:06,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 4: [2023-05-08 11:26:06,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 7: [2023-05-08 11:26:06,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 7: [2023-05-08 11:26:06,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +29: [2023-05-08 11:26:06,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +29: [2023-05-08 11:26:06,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 7: [2023-05-08 11:26:06,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +16: [2023-05-08 11:26:06,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +29: [2023-05-08 11:26:06,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +13: [2023-05-08 11:26:06,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +13: [2023-05-08 11:26:06,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +13: [2023-05-08 11:26:06,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 9: [2023-05-08 11:26:06,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 9: [2023-05-08 11:26:06,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 7: [2023-05-08 11:26:06,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 4: [2023-05-08 11:26:06,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +16: [2023-05-08 11:26:06,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +29: [2023-05-08 11:26:06,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +13: [2023-05-08 11:26:06,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 9: [2023-05-08 11:26:06,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +28: [2023-05-08 11:26:06,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +28: [2023-05-08 11:26:06,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +28: [2023-05-08 11:26:06,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +17: [2023-05-08 11:26:06,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +17: [2023-05-08 11:26:06,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +17: [2023-05-08 11:26:06,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +28: [2023-05-08 11:26:06,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +17: [2023-05-08 11:26:06,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +18: [2023-05-08 11:26:06,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +29: [2023-05-08 11:26:06,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:26:06,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +29: [2023-05-08 11:26:06,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:26:06,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:26:06,390] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:26:06,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +29: [2023-05-08 11:26:06,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:26:06,391] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:26:06,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +29: [2023-05-08 11:26:06,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 2: [2023-05-08 11:26:06,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 2: [2023-05-08 11:26:06,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +20: [2023-05-08 11:26:06,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 7: [2023-05-08 11:26:06,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 8: [2023-05-08 11:26:06,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 8: [2023-05-08 11:26:06,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 8: [2023-05-08 11:26:06,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 7: [2023-05-08 11:26:06,395] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:26:06,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +24: [2023-05-08 11:26:06,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +10: [2023-05-08 11:26:06,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:26:06,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:26:06,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:26:06,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +13: [2023-05-08 11:26:06,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:26:06,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:26:06,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +28: [2023-05-08 11:26:06,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +28: [2023-05-08 11:26:06,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +17: [2023-05-08 11:26:06,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +29: [2023-05-08 11:26:06,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +29: [2023-05-08 11:26:06,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +17: [2023-05-08 11:26:06,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +16: [2023-05-08 11:26:06,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +17: [2023-05-08 11:26:06,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +13: [2023-05-08 11:26:06,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +13: [2023-05-08 11:26:06,399] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +17: [2023-05-08 11:26:06,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +29: [2023-05-08 11:26:06,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +13: [2023-05-08 11:26:06,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +29: [2023-05-08 11:26:06,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +28: [2023-05-08 11:26:06,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:26:06,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +23: [2023-05-08 11:26:06,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +23: [2023-05-08 11:26:06,401] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:26:06,401] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,402] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +30: [2023-05-08 11:26:06,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +24: [2023-05-08 11:26:06,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:26:06,402] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:26:06,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 2: [2023-05-08 11:26:06,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:26:06,403] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:26:06,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 8: [2023-05-08 11:26:06,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,403] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:26:06,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:26:06,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 2: [2023-05-08 11:26:06,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +24: [2023-05-08 11:26:06,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:26:06,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +17: [2023-05-08 11:26:06,410] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +13: [2023-05-08 11:26:06,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +17: [2023-05-08 11:26:06,410] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +21: [2023-05-08 11:26:06,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +21: [2023-05-08 11:26:06,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +23: [2023-05-08 11:26:06,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +23: [2023-05-08 11:26:06,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +23: [2023-05-08 11:26:06,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,411] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:26:06,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:26:06,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +17: [2023-05-08 11:26:06,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +17: [2023-05-08 11:26:06,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +21: [2023-05-08 11:26:06,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +28: [2023-05-08 11:26:06,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +21: [2023-05-08 11:26:06,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +24: [2023-05-08 11:26:06,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +23: [2023-05-08 11:26:06,412] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +13: [2023-05-08 11:26:06,412] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +13: [2023-05-08 11:26:06,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 2: [2023-05-08 11:26:06,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +10: [2023-05-08 11:26:06,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:26:06,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +24: [2023-05-08 11:26:06,413] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:26:06,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +25: [2023-05-08 11:26:06,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:26:06,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:26:06,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +13: [2023-05-08 11:26:06,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +13: [2023-05-08 11:26:06,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +25: [2023-05-08 11:26:06,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:26:06,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:26:06,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +10: [2023-05-08 11:26:06,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +25: [2023-05-08 11:26:06,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +25: [2023-05-08 11:26:06,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +10: [2023-05-08 11:26:06,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +10: [2023-05-08 11:26:06,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 2: [2023-05-08 11:26:06,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 9: [2023-05-08 11:26:06,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +10: [2023-05-08 11:26:06,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +10: [2023-05-08 11:26:06,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +10: [2023-05-08 11:26:06,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +10: [2023-05-08 11:26:06,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +29: [2023-05-08 11:26:06,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:26:06,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +25: [2023-05-08 11:26:06,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +16: [2023-05-08 11:26:06,416] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 4: [2023-05-08 11:26:06,416] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 4: [2023-05-08 11:26:06,417] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +28: [2023-05-08 11:26:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:26:06,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +12: [2023-05-08 11:26:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +24: [2023-05-08 11:26:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:26:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:26:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +12: [2023-05-08 11:26:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +17: [2023-05-08 11:26:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:26:06,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +29: [2023-05-08 11:26:06,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 8: [2023-05-08 11:26:06,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +23: [2023-05-08 11:26:06,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +17: [2023-05-08 11:26:06,419] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 8: [2023-05-08 11:26:06,420] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +16: [2023-05-08 11:26:06,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +16: [2023-05-08 11:26:06,420] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 7: [2023-05-08 11:26:06,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 8: [2023-05-08 11:26:06,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:26:06,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +21: [2023-05-08 11:26:06,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +21: [2023-05-08 11:26:06,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,422] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +21: [2023-05-08 11:26:06,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +17: [2023-05-08 11:26:06,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +16: [2023-05-08 11:26:06,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +16: [2023-05-08 11:26:06,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 4: [2023-05-08 11:26:06,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:26:06,424] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:26:06,424] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +17: [2023-05-08 11:26:06,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 4: [2023-05-08 11:26:06,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 9: [2023-05-08 11:26:06,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +29: [2023-05-08 11:26:06,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:26:06,425] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:26:06,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:26:06,426] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 9: [2023-05-08 11:26:06,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +16: [2023-05-08 11:26:06,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:26:06,427] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 9: [2023-05-08 11:26:06,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +16: [2023-05-08 11:26:06,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +29: [2023-05-08 11:26:06,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +12: [2023-05-08 11:26:06,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +29: [2023-05-08 11:26:06,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:26:06,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +29: [2023-05-08 11:26:06,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +29: [2023-05-08 11:26:06,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,430] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +28: [2023-05-08 11:26:06,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +29: [2023-05-08 11:26:06,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +29: [2023-05-08 11:26:06,431] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +29: [2023-05-08 11:26:06,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +29: [2023-05-08 11:26:06,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 2: [2023-05-08 11:26:06,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +28: [2023-05-08 11:26:06,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 7: [2023-05-08 11:26:06,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +25: [2023-05-08 11:26:06,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:26:06,433] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:26:06,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +25: [2023-05-08 11:26:06,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +29: [2023-05-08 11:26:06,437] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +29: [2023-05-08 11:26:06,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +29: [2023-05-08 11:26:06,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +12: [2023-05-08 11:26:06,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:26:06,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +25: [2023-05-08 11:26:06,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:26:06,444] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 0: [2023-05-08 11:26:06,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 0: [2023-05-08 11:26:06,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 0: [2023-05-08 11:26:06,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +25: [2023-05-08 11:26:06,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:26:06,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:26:06,452] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:26:06,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +12: [2023-05-08 11:26:06,452] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 5: [2023-05-08 11:26:06,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:26:06,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,459] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,462] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:26:06,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +12: [2023-05-08 11:26:06,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 5: [2023-05-08 11:26:06,469] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:26:06,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +15: [2023-05-08 11:26:06,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +15: [2023-05-08 11:26:06,472] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +15: [2023-05-08 11:26:06,473] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +27: [2023-05-08 11:26:06,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:26:06,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:26:06,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:26:06,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +21: [2023-05-08 11:26:06,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +21: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +21: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +13: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +13: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +21: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +21: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +21: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +21: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +13: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +21: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +21: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +21: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +21: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +21: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +21: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +13: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +22: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +21: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +13: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +21: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +22: [2023-05-08 11:26:06,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:26:06,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:26:06,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:26:06,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:26:06,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:26:06,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:26:06,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:26:06,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:26:06,476] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:26:06,476] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:26:06,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +22: [2023-05-08 11:26:06,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:26:06,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:26:06,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:26:06,477] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 0: [2023-05-08 11:26:06,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,480] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +15: [2023-05-08 11:26:06,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:26:06,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:26:06,480] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:26:06,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:26:06,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +15: [2023-05-08 11:26:06,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:26:06,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:26:06,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:26:06,481] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:26:06,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:26:06,481] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +10: [2023-05-08 11:26:06,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:26:06,482] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:26:06,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:26:06,482] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +30: [2023-05-08 11:26:06,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +30: [2023-05-08 11:26:06,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:26:06,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +30: [2023-05-08 11:26:06,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +30: [2023-05-08 11:26:06,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +30: [2023-05-08 11:26:06,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +30: [2023-05-08 11:26:06,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +30: [2023-05-08 11:26:06,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:26:06,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +15: [2023-05-08 11:26:06,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:26:06,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +30: [2023-05-08 11:26:06,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:26:06,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +27: [2023-05-08 11:26:06,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:26:06,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:26:06,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:26:06,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:26:06,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:26:06,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:26:06,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:26:06,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:26:06,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:26:06,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:26:06,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:26:06,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:26:06,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:26:06,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:26:06,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:26:06,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:26:06,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:26:06,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:26:06,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 1: [2023-05-08 11:26:06,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 1: [2023-05-08 11:26:06,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 1: [2023-05-08 11:26:06,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +21: [2023-05-08 11:26:06,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +21: [2023-05-08 11:26:06,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +21: [2023-05-08 11:26:06,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +13: [2023-05-08 11:26:06,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:26:06,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:26:06,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +10: [2023-05-08 11:26:06,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:26:06,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:26:06,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:26:06,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:26:06,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,493] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +22: [2023-05-08 11:26:06,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:26:06,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:26:06,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:26:06,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:26:06,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:26:06,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +19: [2023-05-08 11:26:06,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +30: [2023-05-08 11:26:06,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:26:06,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:26:06,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:26:06,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:26:06,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:26:06,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:26:06,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 9: [2023-05-08 11:26:06,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,497] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:26:06,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:26:06,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 4: [2023-05-08 11:26:06,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +30: [2023-05-08 11:26:06,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +30: [2023-05-08 11:26:06,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,498] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:26:06,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +23: [2023-05-08 11:26:06,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +23: [2023-05-08 11:26:06,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +23: [2023-05-08 11:26:06,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +23: [2023-05-08 11:26:06,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 0: [2023-05-08 11:26:06,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:26:06,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +21: [2023-05-08 11:26:06,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:26:06,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +21: [2023-05-08 11:26:06,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,501] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:26:06,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +22: [2023-05-08 11:26:06,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +13: [2023-05-08 11:26:06,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:26:06,502] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +15: [2023-05-08 11:26:06,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:26:06,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:26:06,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +15: [2023-05-08 11:26:06,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:26:06,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:26:06,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:26:06,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +10: [2023-05-08 11:26:06,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +31: [2023-05-08 11:26:06,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +13: [2023-05-08 11:26:06,504] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:26:06,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +22: [2023-05-08 11:26:06,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,505] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:26:06,505] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +16: [2023-05-08 11:26:06,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +16: [2023-05-08 11:26:06,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +31: [2023-05-08 11:26:06,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:26:06,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:26:06,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:26:06,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +15: [2023-05-08 11:26:06,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +20: [2023-05-08 11:26:06,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:26:06,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:26:06,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:26:06,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +22: [2023-05-08 11:26:06,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:26:06,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:26:06,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:26:06,508] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +10: [2023-05-08 11:26:06,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +12: [2023-05-08 11:26:06,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:26:06,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +12: [2023-05-08 11:26:06,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:26:06,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:26:06,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:26:06,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +22: [2023-05-08 11:26:06,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:26:06,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +31: [2023-05-08 11:26:06,509] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +19: [2023-05-08 11:26:06,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:26:06,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +31: [2023-05-08 11:26:06,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:26:06,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +20: [2023-05-08 11:26:06,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:26:06,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +23: [2023-05-08 11:26:06,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +30: [2023-05-08 11:26:06,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:26:06,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +26: [2023-05-08 11:26:06,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +26: [2023-05-08 11:26:06,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +31: [2023-05-08 11:26:06,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:26:06,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:26:06,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +27: [2023-05-08 11:26:06,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +27: [2023-05-08 11:26:06,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +26: [2023-05-08 11:26:06,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +23: [2023-05-08 11:26:06,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:26:06,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +23: [2023-05-08 11:26:06,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +23: [2023-05-08 11:26:06,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:26:06,512] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +21: [2023-05-08 11:26:06,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:26:06,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +15: [2023-05-08 11:26:06,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:26:06,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:26:06,513] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:26:06,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +15: [2023-05-08 11:26:06,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:26:06,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,514] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +15: [2023-05-08 11:26:06,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +15: [2023-05-08 11:26:06,515] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 1: [2023-05-08 11:26:06,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 4: [2023-05-08 11:26:06,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,516] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,516] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 1: [2023-05-08 11:26:06,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 6: [2023-05-08 11:26:06,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:26:06,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +12: [2023-05-08 11:26:06,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:26:06,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:26:06,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +30: [2023-05-08 11:26:06,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +15: [2023-05-08 11:26:06,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:26:06,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +14: [2023-05-08 11:26:06,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +14: [2023-05-08 11:26:06,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +14: [2023-05-08 11:26:06,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:26:06,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +14: [2023-05-08 11:26:06,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +12: [2023-05-08 11:26:06,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:26:06,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +14: [2023-05-08 11:26:06,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +12: [2023-05-08 11:26:06,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:26:06,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +25: [2023-05-08 11:26:06,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +25: [2023-05-08 11:26:06,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +25: [2023-05-08 11:26:06,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +25: [2023-05-08 11:26:06,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +26: [2023-05-08 11:26:06,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:26:06,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:26:06,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:26:06,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +31: [2023-05-08 11:26:06,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:26:06,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:26:06,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:26:06,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +19: [2023-05-08 11:26:06,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:26:06,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +19: [2023-05-08 11:26:06,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +31: [2023-05-08 11:26:06,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:26:06,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +26: [2023-05-08 11:26:06,523] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +19: [2023-05-08 11:26:06,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:26:06,524] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:26:06,525] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +31: [2023-05-08 11:26:06,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 6: [2023-05-08 11:26:06,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,527] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:26:06,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +19: [2023-05-08 11:26:06,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +19: [2023-05-08 11:26:06,527] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +19: [2023-05-08 11:26:06,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 3: [2023-05-08 11:26:06,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,528] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +15: [2023-05-08 11:26:06,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +15: [2023-05-08 11:26:06,529] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:26:06,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:26:06,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +23: [2023-05-08 11:26:06,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:26:06,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +31: [2023-05-08 11:26:06,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +23: [2023-05-08 11:26:06,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +31: [2023-05-08 11:26:06,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +31: [2023-05-08 11:26:06,531] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +23: [2023-05-08 11:26:06,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:26:06,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 9: [2023-05-08 11:26:06,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:26:06,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:26:06,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +28: [2023-05-08 11:26:06,532] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:26:06,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +25: [2023-05-08 11:26:06,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:26:06,532] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:26:06,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +23: [2023-05-08 11:26:06,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +31: [2023-05-08 11:26:06,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +23: [2023-05-08 11:26:06,533] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +23: [2023-05-08 11:26:06,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:26:06,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:26:06,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:26:06,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +28: [2023-05-08 11:26:06,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:26:06,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +14: [2023-05-08 11:26:06,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:26:06,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:26:06,535] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:26:06,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:26:06,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:26:06,535] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +26: [2023-05-08 11:26:06,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:26:06,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:26:06,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +26: [2023-05-08 11:26:06,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +31: [2023-05-08 11:26:06,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +14: [2023-05-08 11:26:06,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:26:06,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:26:06,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +19: [2023-05-08 11:26:06,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +31: [2023-05-08 11:26:06,539] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +26: [2023-05-08 11:26:06,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:26:06,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +18: [2023-05-08 11:26:06,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +18: [2023-05-08 11:26:06,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +20: [2023-05-08 11:26:06,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,540] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:26:06,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +19: [2023-05-08 11:26:06,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:26:06,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +27: [2023-05-08 11:26:06,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:26:06,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +27: [2023-05-08 11:26:06,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +27: [2023-05-08 11:26:06,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +19: [2023-05-08 11:26:06,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,543] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:26:06,545] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +28: [2023-05-08 11:26:06,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +14: [2023-05-08 11:26:06,546] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +20: [2023-05-08 11:26:06,546] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +14: [2023-05-08 11:26:06,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +20: [2023-05-08 11:26:06,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:26:06,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:26:06,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:26:06,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:26:06,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +14: [2023-05-08 11:26:06,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:26:06,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +18: [2023-05-08 11:26:06,549] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +26: [2023-05-08 11:26:06,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +18: [2023-05-08 11:26:06,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,550] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:26:06,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:26:06,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +18: [2023-05-08 11:26:06,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +18: [2023-05-08 11:26:06,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:26:06,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:26:06,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +19: [2023-05-08 11:26:06,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +27: [2023-05-08 11:26:06,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:26:06,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:26:06,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +27: [2023-05-08 11:26:06,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +20: [2023-05-08 11:26:06,555] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +20: [2023-05-08 11:26:06,557] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 9: [2023-05-08 11:26:06,557] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +28: [2023-05-08 11:26:06,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:26:06,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +28: [2023-05-08 11:26:06,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 9: [2023-05-08 11:26:06,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 8: [2023-05-08 11:26:06,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:26:06,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:26:06,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:26:06,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +19: [2023-05-08 11:26:06,560] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +25: [2023-05-08 11:26:06,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:26:06,562] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:26:06,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +18: [2023-05-08 11:26:06,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +18: [2023-05-08 11:26:06,563] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +18: [2023-05-08 11:26:06,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +18: [2023-05-08 11:26:06,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 3: [2023-05-08 11:26:06,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:26:06,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +18: [2023-05-08 11:26:06,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +19: [2023-05-08 11:26:06,567] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +19: [2023-05-08 11:26:06,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 9: [2023-05-08 11:26:06,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,568] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 5: [2023-05-08 11:26:06,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 5: [2023-05-08 11:26:06,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 5: [2023-05-08 11:26:06,569] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 8: [2023-05-08 11:26:06,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,570] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:26:06,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +18: [2023-05-08 11:26:06,571] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +14: [2023-05-08 11:26:06,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +14: [2023-05-08 11:26:06,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +14: [2023-05-08 11:26:06,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. +28: [2023-05-08 11:26:06,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:26:06,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt. + 8: [2023-05-08 11:26:06,573] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,574] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:26:06,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 5: [2023-05-08 11:26:06,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:26:06,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... +14: [2023-05-08 11:26:06,582] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +14: [2023-05-08 11:26:06,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +18: [2023-05-08 11:26:06,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:26:06,586] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:26:06,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:26:06,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:26:06,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +30: [2023-05-08 11:26:06,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +30: [2023-05-08 11:26:06,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +30: [2023-05-08 11:26:06,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +18: [2023-05-08 11:26:06,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:26:06,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:26:06,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:26:06,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,590] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +14: [2023-05-08 11:26:06,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +14: [2023-05-08 11:26:06,594] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +14: [2023-05-08 11:26:06,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:26:06,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:26:06,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +30: [2023-05-08 11:26:06,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +30: [2023-05-08 11:26:06,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +30: [2023-05-08 11:26:06,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 5: [2023-05-08 11:26:06,599] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... + 5: [2023-05-08 11:26:06,599] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +18: [2023-05-08 11:26:06,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:26:06,600] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:26:06,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +14: [2023-05-08 11:26:06,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +26: [2023-05-08 11:26:06,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:26:06,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +14: [2023-05-08 11:26:06,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +26: [2023-05-08 11:26:06,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:26:06,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:26:06,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:26:06,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:26:06,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:26:06,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:26:06,607] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:26:06,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +11: [2023-05-08 11:26:06,607] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. +22: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +22: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +22: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +11: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +24: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +24: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +24: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +29: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +24: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +29: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +24: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +29: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:26:06,608] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,609] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +29: [2023-05-08 11:26:06,609] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:26:06,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +29: [2023-05-08 11:26:06,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +18: [2023-05-08 11:26:06,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:26:06,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:26:06,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +14: [2023-05-08 11:26:06,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,610] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:26:06,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:26:06,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:26:06,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt... +30: [2023-05-08 11:26:06,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +30: [2023-05-08 11:26:06,613] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +30: [2023-05-08 11:26:06,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +30: [2023-05-08 11:26:06,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +30: [2023-05-08 11:26:06,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +30: [2023-05-08 11:26:06,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +26: [2023-05-08 11:26:06,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:26:06,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +22: [2023-05-08 11:26:06,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:26:06,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +22: [2023-05-08 11:26:06,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +11: [2023-05-08 11:26:06,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +22: [2023-05-08 11:26:06,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +26: [2023-05-08 11:26:06,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:26:06,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:26:06,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:26:06,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +30: [2023-05-08 11:26:06,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +30: [2023-05-08 11:26:06,621] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 2: [2023-05-08 11:26:06,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +24: [2023-05-08 11:26:06,623] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:26:06,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:26:06,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:26:06,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +26: [2023-05-08 11:26:06,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:26:06,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:26:06,629] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:26:06,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 6: [2023-05-08 11:26:06,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 6: [2023-05-08 11:26:06,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 6: [2023-05-08 11:26:06,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 6: [2023-05-08 11:26:06,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +24: [2023-05-08 11:26:06,631] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:26:06,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:26:06,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,632] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +24: [2023-05-08 11:26:06,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +24: [2023-05-08 11:26:06,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +11: [2023-05-08 11:26:06,633] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:26:06,633] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 2: [2023-05-08 11:26:06,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,634] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:26:06,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 3: [2023-05-08 11:26:06,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 3: [2023-05-08 11:26:06,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 3: [2023-05-08 11:26:06,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +29: [2023-05-08 11:26:06,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:26:06,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:26:06,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +29: [2023-05-08 11:26:06,635] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:26:06,635] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 7: [2023-05-08 11:26:06,636] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:26:06,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +29: [2023-05-08 11:26:06,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +11: [2023-05-08 11:26:06,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +29: [2023-05-08 11:26:06,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +29: [2023-05-08 11:26:06,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +29: [2023-05-08 11:26:06,636] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 7: [2023-05-08 11:26:06,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:26:06,637] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:26:06,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +22: [2023-05-08 11:26:06,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +26: [2023-05-08 11:26:06,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 2: [2023-05-08 11:26:06,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 7: [2023-05-08 11:26:06,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 7: [2023-05-08 11:26:06,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 2: [2023-05-08 11:26:06,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +22: [2023-05-08 11:26:06,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 2: [2023-05-08 11:26:06,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,638] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +22: [2023-05-08 11:26:06,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +22: [2023-05-08 11:26:06,639] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +22: [2023-05-08 11:26:06,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:26:06,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +17: [2023-05-08 11:26:06,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +28: [2023-05-08 11:26:06,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +17: [2023-05-08 11:26:06,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +28: [2023-05-08 11:26:06,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +28: [2023-05-08 11:26:06,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +17: [2023-05-08 11:26:06,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +28: [2023-05-08 11:26:06,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +13: [2023-05-08 11:26:06,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +13: [2023-05-08 11:26:06,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +13: [2023-05-08 11:26:06,640] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +22: [2023-05-08 11:26:06,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +13: [2023-05-08 11:26:06,641] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +24: [2023-05-08 11:26:06,641] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:26:06,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,642] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,642] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:26:06,643] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:26:06,644] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +24: [2023-05-08 11:26:06,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +11: [2023-05-08 11:26:06,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:26:06,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +29: [2023-05-08 11:26:06,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +29: [2023-05-08 11:26:06,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:26:06,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +26: [2023-05-08 11:26:06,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +29: [2023-05-08 11:26:06,647] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:26:06,645] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:26:06,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +11: [2023-05-08 11:26:06,646] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +11: [2023-05-08 11:26:06,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:26:06,648] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +24: [2023-05-08 11:26:06,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +17: [2023-05-08 11:26:06,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +17: [2023-05-08 11:26:06,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:26:06,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 4: [2023-05-08 11:26:06,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +29: [2023-05-08 11:26:06,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +29: [2023-05-08 11:26:06,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 4: [2023-05-08 11:26:06,651] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 2: [2023-05-08 11:26:06,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +17: [2023-05-08 11:26:06,651] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +17: [2023-05-08 11:26:06,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:26:06,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +28: [2023-05-08 11:26:06,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:26:06,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +28: [2023-05-08 11:26:06,652] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:26:06,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +12: [2023-05-08 11:26:06,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +12: [2023-05-08 11:26:06,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +13: [2023-05-08 11:26:06,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +29: [2023-05-08 11:26:06,653] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:26:06,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +28: [2023-05-08 11:26:06,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,654] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:26:06,654] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,655] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 6: [2023-05-08 11:26:06,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +29: [2023-05-08 11:26:06,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:26:06,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +17: [2023-05-08 11:26:06,657] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +17: [2023-05-08 11:26:06,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +17: [2023-05-08 11:26:06,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +17: [2023-05-08 11:26:06,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +17: [2023-05-08 11:26:06,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +17: [2023-05-08 11:26:06,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +17: [2023-05-08 11:26:06,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 6: [2023-05-08 11:26:06,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 4: [2023-05-08 11:26:06,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:26:06,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +11: [2023-05-08 11:26:06,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:26:06,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,660] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +11: [2023-05-08 11:26:06,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 4: [2023-05-08 11:26:06,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +11: [2023-05-08 11:26:06,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +29: [2023-05-08 11:26:06,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +24: [2023-05-08 11:26:06,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:26:06,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +12: [2023-05-08 11:26:06,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +12: [2023-05-08 11:26:06,662] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +13: [2023-05-08 11:26:06,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +13: [2023-05-08 11:26:06,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 2: [2023-05-08 11:26:06,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,663] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +10: [2023-05-08 11:26:06,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +10: [2023-05-08 11:26:06,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +10: [2023-05-08 11:26:06,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +12: [2023-05-08 11:26:06,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:26:06,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 2: [2023-05-08 11:26:06,664] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 7: [2023-05-08 11:26:06,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 7: [2023-05-08 11:26:06,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 3: [2023-05-08 11:26:06,665] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +28: [2023-05-08 11:26:06,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:26:06,667] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +11: [2023-05-08 11:26:06,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +11: [2023-05-08 11:26:06,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 3: [2023-05-08 11:26:06,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,668] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 7: [2023-05-08 11:26:06,669] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:26:06,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +24: [2023-05-08 11:26:06,669] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +24: [2023-05-08 11:26:06,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:26:06,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +24: [2023-05-08 11:26:06,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:26:06,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +29: [2023-05-08 11:26:06,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +17: [2023-05-08 11:26:06,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,670] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:26:06,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +20: [2023-05-08 11:26:06,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +20: [2023-05-08 11:26:06,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +20: [2023-05-08 11:26:06,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +20: [2023-05-08 11:26:06,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +13: [2023-05-08 11:26:06,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +24: [2023-05-08 11:26:06,671] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +13: [2023-05-08 11:26:06,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +17: [2023-05-08 11:26:06,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +17: [2023-05-08 11:26:06,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,672] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:26:06,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +23: [2023-05-08 11:26:06,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,672] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +23: [2023-05-08 11:26:06,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:26:06,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:26:06,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +23: [2023-05-08 11:26:06,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,673] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +10: [2023-05-08 11:26:06,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:26:06,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:26:06,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +23: [2023-05-08 11:26:06,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:26:06,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:26:06,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +23: [2023-05-08 11:26:06,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:26:06,673] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:26:06,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +12: [2023-05-08 11:26:06,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +29: [2023-05-08 11:26:06,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +28: [2023-05-08 11:26:06,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:26:06,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +10: [2023-05-08 11:26:06,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:26:06,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:26:06,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +11: [2023-05-08 11:26:06,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 7: [2023-05-08 11:26:06,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +23: [2023-05-08 11:26:06,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,675] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:26:06,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:26:06,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:26:06,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +29: [2023-05-08 11:26:06,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +17: [2023-05-08 11:26:06,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +17: [2023-05-08 11:26:06,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:26:06,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +17: [2023-05-08 11:26:06,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +15: [2023-05-08 11:26:06,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 4: [2023-05-08 11:26:06,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 2: [2023-05-08 11:26:06,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:26:06,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:26:06,676] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +17: [2023-05-08 11:26:06,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:26:06,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +13: [2023-05-08 11:26:06,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:26:06,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:26:06,678] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +17: [2023-05-08 11:26:06,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +13: [2023-05-08 11:26:06,679] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +17: [2023-05-08 11:26:06,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +13: [2023-05-08 11:26:06,679] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:26:06,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:26:06,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +28: [2023-05-08 11:26:06,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +28: [2023-05-08 11:26:06,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +24: [2023-05-08 11:26:06,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +20: [2023-05-08 11:26:06,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:26:06,680] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:26:06,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +20: [2023-05-08 11:26:06,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:26:06,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +10: [2023-05-08 11:26:06,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:26:06,681] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 4: [2023-05-08 11:26:06,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +12: [2023-05-08 11:26:06,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +12: [2023-05-08 11:26:06,683] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +12: [2023-05-08 11:26:06,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +12: [2023-05-08 11:26:06,683] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +12: [2023-05-08 11:26:06,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:26:06,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:26:06,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 3: [2023-05-08 11:26:06,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +17: [2023-05-08 11:26:06,684] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +15: [2023-05-08 11:26:06,685] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +23: [2023-05-08 11:26:06,685] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +15: [2023-05-08 11:26:06,686] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:26:06,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +15: [2023-05-08 11:26:06,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +17: [2023-05-08 11:26:06,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +17: [2023-05-08 11:26:06,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +23: [2023-05-08 11:26:06,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:26:06,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:26:06,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +21: [2023-05-08 11:26:06,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +21: [2023-05-08 11:26:06,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:26:06,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +21: [2023-05-08 11:26:06,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +21: [2023-05-08 11:26:06,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +10: [2023-05-08 11:26:06,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +21: [2023-05-08 11:26:06,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +21: [2023-05-08 11:26:06,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:26:06,693] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:26:06,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:26:06,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +10: [2023-05-08 11:26:06,694] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,694] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +10: [2023-05-08 11:26:06,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:26:06,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +23: [2023-05-08 11:26:06,695] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:26:06,696] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:26:06,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:26:06,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +20: [2023-05-08 11:26:06,697] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 5: [2023-05-08 11:26:06,697] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:26:06,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:26:06,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +10: [2023-05-08 11:26:06,699] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +23: [2023-05-08 11:26:06,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:26:06,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +16: [2023-05-08 11:26:06,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +10: [2023-05-08 11:26:06,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +10: [2023-05-08 11:26:06,700] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +15: [2023-05-08 11:26:06,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:26:06,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:26:06,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +15: [2023-05-08 11:26:06,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +23: [2023-05-08 11:26:06,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:26:06,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:26:06,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:26:06,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +15: [2023-05-08 11:26:06,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +15: [2023-05-08 11:26:06,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +21: [2023-05-08 11:26:06,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:26:06,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:26:06,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:26:06,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:26:06,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +10: [2023-05-08 11:26:06,706] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +16: [2023-05-08 11:26:06,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 0: [2023-05-08 11:26:06,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 0: [2023-05-08 11:26:06,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +16: [2023-05-08 11:26:06,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 5: [2023-05-08 11:26:06,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 0: [2023-05-08 11:26:06,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +25: [2023-05-08 11:26:06,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +25: [2023-05-08 11:26:06,707] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 8: [2023-05-08 11:26:06,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +25: [2023-05-08 11:26:06,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +25: [2023-05-08 11:26:06,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,707] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +25: [2023-05-08 11:26:06,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +21: [2023-05-08 11:26:06,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:26:06,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +25: [2023-05-08 11:26:06,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +21: [2023-05-08 11:26:06,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:26:06,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:26:06,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:26:06,708] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:26:06,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +27: [2023-05-08 11:26:06,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +27: [2023-05-08 11:26:06,709] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:26:06,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +23: [2023-05-08 11:26:06,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +25: [2023-05-08 11:26:06,709] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +27: [2023-05-08 11:26:06,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +14: [2023-05-08 11:26:06,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +16: [2023-05-08 11:26:06,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:26:06,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +14: [2023-05-08 11:26:06,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +16: [2023-05-08 11:26:06,710] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:26:06,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:26:06,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 5: [2023-05-08 11:26:06,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +14: [2023-05-08 11:26:06,710] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +10: [2023-05-08 11:26:06,711] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. + 5: [2023-05-08 11:26:06,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt. +23: [2023-05-08 11:26:06,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:26:06,712] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:26:06,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +16: [2023-05-08 11:26:06,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:26:06,713] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +15: [2023-05-08 11:26:06,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +21: [2023-05-08 11:26:06,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +21: [2023-05-08 11:26:06,716] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 8: [2023-05-08 11:26:06,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +21: [2023-05-08 11:26:06,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 3: [2023-05-08 11:26:06,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,717] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,717] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +21: [2023-05-08 11:26:06,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +21: [2023-05-08 11:26:06,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +16: [2023-05-08 11:26:06,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +10: [2023-05-08 11:26:06,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,718] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +14: [2023-05-08 11:26:06,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:26:06,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +23: [2023-05-08 11:26:06,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:26:06,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,719] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +25: [2023-05-08 11:26:06,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:26:06,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +21: [2023-05-08 11:26:06,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +14: [2023-05-08 11:26:06,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:26:06,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:26:06,720] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:26:06,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:26:06,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,721] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +27: [2023-05-08 11:26:06,721] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:26:06,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +14: [2023-05-08 11:26:06,722] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +14: [2023-05-08 11:26:06,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +14: [2023-05-08 11:26:06,723] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt... +16: [2023-05-08 11:26:06,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:26:06,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:26:06,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:26:06,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:26:06,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:26:06,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:26:06,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:26:06,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +20: [2023-05-08 11:26:06,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +20: [2023-05-08 11:26:06,725] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +20: [2023-05-08 11:26:06,725] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,726] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +21: [2023-05-08 11:26:06,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:26:06,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:26:06,727] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:26:06,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +25: [2023-05-08 11:26:06,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:26:06,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +21: [2023-05-08 11:26:06,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:26:06,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:26:06,728] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:26:06,729] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,729] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:26:06,733] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:26:06,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:26:06,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +16: [2023-05-08 11:26:06,734] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,734] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,735] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +27: [2023-05-08 11:26:06,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:26:06,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +14: [2023-05-08 11:26:06,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:26:06,735] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +14: [2023-05-08 11:26:06,736] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +14: [2023-05-08 11:26:06,736] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 0: [2023-05-08 11:26:06,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,737] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:26:06,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 1: [2023-05-08 11:26:06,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 1: [2023-05-08 11:26:06,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +25: [2023-05-08 11:26:06,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,738] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 0: [2023-05-08 11:26:06,738] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 5: [2023-05-08 11:26:06,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 8: [2023-05-08 11:26:06,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +21: [2023-05-08 11:26:06,739] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:26:06,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +16: [2023-05-08 11:26:06,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:26:06,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +26: [2023-05-08 11:26:06,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +26: [2023-05-08 11:26:06,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 0: [2023-05-08 11:26:06,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:26:06,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +16: [2023-05-08 11:26:06,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:26:06,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +16: [2023-05-08 11:26:06,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +16: [2023-05-08 11:26:06,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +16: [2023-05-08 11:26:06,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +16: [2023-05-08 11:26:06,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +27: [2023-05-08 11:26:06,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +23: [2023-05-08 11:26:06,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +23: [2023-05-08 11:26:06,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +23: [2023-05-08 11:26:06,740] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +23: [2023-05-08 11:26:06,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +25: [2023-05-08 11:26:06,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:26:06,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +27: [2023-05-08 11:26:06,741] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +27: [2023-05-08 11:26:06,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +27: [2023-05-08 11:26:06,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:26:06,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +22: [2023-05-08 11:26:06,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +22: [2023-05-08 11:26:06,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +22: [2023-05-08 11:26:06,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,742] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 5: [2023-05-08 11:26:06,742] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +22: [2023-05-08 11:26:06,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +22: [2023-05-08 11:26:06,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +22: [2023-05-08 11:26:06,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,743] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:26:06,743] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +14: [2023-05-08 11:26:06,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 3: [2023-05-08 11:26:06,744] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:26:06,744] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:26:06,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,745] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +22: [2023-05-08 11:26:06,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:26:06,745] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 3: [2023-05-08 11:26:06,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:26:06,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +21: [2023-05-08 11:26:06,746] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +16: [2023-05-08 11:26:06,746] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +14: [2023-05-08 11:26:06,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +14: [2023-05-08 11:26:06,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +14: [2023-05-08 11:26:06,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,747] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 0: [2023-05-08 11:26:06,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 0: [2023-05-08 11:26:06,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +20: [2023-05-08 11:26:06,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:26:06,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:26:06,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:26:06,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:26:06,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. +25: [2023-05-08 11:26:06,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:26:06,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:26:06,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:26:06,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:26:06,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:26:06,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... + 4: [2023-05-08 11:26:06,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:26:06,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +23: [2023-05-08 11:26:06,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +23: [2023-05-08 11:26:06,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:26:06,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +13: [2023-05-08 11:26:06,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +21: [2023-05-08 11:26:06,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +13: [2023-05-08 11:26:06,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:26:06,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +18: [2023-05-08 11:26:06,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +18: [2023-05-08 11:26:06,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +18: [2023-05-08 11:26:06,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +18: [2023-05-08 11:26:06,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:26:06,751] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +14: [2023-05-08 11:26:06,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +18: [2023-05-08 11:26:06,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:26:06,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +18: [2023-05-08 11:26:06,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +23: [2023-05-08 11:26:06,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:26:06,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:26:06,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +16: [2023-05-08 11:26:06,752] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +13: [2023-05-08 11:26:06,752] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:26:06,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:26:06,753] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,754] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt... +31: [2023-05-08 11:26:06,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +31: [2023-05-08 11:26:06,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +31: [2023-05-08 11:26:06,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +13: [2023-05-08 11:26:06,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +13: [2023-05-08 11:26:06,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +31: [2023-05-08 11:26:06,754] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 9: [2023-05-08 11:26:06,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,755] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +12: [2023-05-08 11:26:06,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:26:06,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:26:06,755] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:26:06,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:26:06,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:26:06,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +12: [2023-05-08 11:26:06,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:26:06,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:26:06,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:26:06,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:26:06,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +21: [2023-05-08 11:26:06,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +28: [2023-05-08 11:26:06,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:26:06,756] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:26:06,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:26:06,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:26:06,756] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,757] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:26:06,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +12: [2023-05-08 11:26:06,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:26:06,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:26:06,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:26:06,758] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:26:06,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:26:06,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:26:06,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:26:06,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:26:06,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:26:06,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:26:06,759] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:26:06,759] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +20: [2023-05-08 11:26:06,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:26:06,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +20: [2023-05-08 11:26:06,760] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:26:06,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:26:06,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:26:06,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:26:06,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:26:06,761] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +14: [2023-05-08 11:26:06,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:26:06,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:26:06,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:26:06,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:26:06,761] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +16: [2023-05-08 11:26:06,762] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:26:06,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +22: [2023-05-08 11:26:06,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +22: [2023-05-08 11:26:06,762] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +31: [2023-05-08 11:26:06,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +31: [2023-05-08 11:26:06,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +31: [2023-05-08 11:26:06,763] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +31: [2023-05-08 11:26:06,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +31: [2023-05-08 11:26:06,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:26:06,763] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,764] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +18: [2023-05-08 11:26:06,764] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +31: [2023-05-08 11:26:06,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:26:06,765] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:26:06,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 4: [2023-05-08 11:26:06,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +13: [2023-05-08 11:26:06,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:26:06,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +31: [2023-05-08 11:26:06,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:26:06,766] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +31: [2023-05-08 11:26:06,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +12: [2023-05-08 11:26:06,766] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:26:06,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:26:06,767] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +23: [2023-05-08 11:26:06,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:26:06,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +18: [2023-05-08 11:26:06,767] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +23: [2023-05-08 11:26:06,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 1: [2023-05-08 11:26:06,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 1: [2023-05-08 11:26:06,768] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +20: [2023-05-08 11:26:06,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +23: [2023-05-08 11:26:06,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +23: [2023-05-08 11:26:06,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +23: [2023-05-08 11:26:06,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +23: [2023-05-08 11:26:06,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +31: [2023-05-08 11:26:06,769] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:26:06,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:26:06,769] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:26:06,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +22: [2023-05-08 11:26:06,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:26:06,770] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:26:06,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:26:06,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:26:06,771] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:26:06,771] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +13: [2023-05-08 11:26:06,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:26:06,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:26:06,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +31: [2023-05-08 11:26:06,772] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:26:06,772] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:26:06,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:26:06,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:26:06,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:26:06,774] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 9: [2023-05-08 11:26:06,774] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +13: [2023-05-08 11:26:06,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +30: [2023-05-08 11:26:06,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:26:06,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,775] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +30: [2023-05-08 11:26:06,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:26:06,775] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:26:06,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:26:06,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:26:06,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 1: [2023-05-08 11:26:06,776] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 4: [2023-05-08 11:26:06,776] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:26:06,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:26:06,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:26:06,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:26:06,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:26:06,777] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +23: [2023-05-08 11:26:06,779] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:26:06,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:26:06,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +27: [2023-05-08 11:26:06,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +23: [2023-05-08 11:26:06,780] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +13: [2023-05-08 11:26:06,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:26:06,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +31: [2023-05-08 11:26:06,780] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:26:06,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +19: [2023-05-08 11:26:06,781] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:26:06,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:26:06,782] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:26:06,782] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:26:06,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:26:06,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:26:06,783] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 4: [2023-05-08 11:26:06,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +13: [2023-05-08 11:26:06,783] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:26:06,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:26:06,784] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:26:06,784] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +19: [2023-05-08 11:26:06,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:26:06,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +30: [2023-05-08 11:26:06,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:26:06,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:26:06,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +27: [2023-05-08 11:26:06,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:26:06,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +13: [2023-05-08 11:26:06,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:26:06,786] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +30: [2023-05-08 11:26:06,786] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:26:06,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +27: [2023-05-08 11:26:06,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +30: [2023-05-08 11:26:06,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:26:06,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +31: [2023-05-08 11:26:06,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +28: [2023-05-08 11:26:06,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:26:06,791] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +13: [2023-05-08 11:26:06,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,792] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:26:06,793] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:26:06,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:26:06,793] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +31: [2023-05-08 11:26:06,795] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:26:06,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:26:06,796] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:26:06,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:26:06,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:26:06,797] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +25: [2023-05-08 11:26:06,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +25: [2023-05-08 11:26:06,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +25: [2023-05-08 11:26:06,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +25: [2023-05-08 11:26:06,798] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +30: [2023-05-08 11:26:06,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 9: [2023-05-08 11:26:06,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +30: [2023-05-08 11:26:06,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +31: [2023-05-08 11:26:06,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:26:06,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +30: [2023-05-08 11:26:06,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +12: [2023-05-08 11:26:06,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:26:06,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:26:06,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +31: [2023-05-08 11:26:06,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +11: [2023-05-08 11:26:06,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +11: [2023-05-08 11:26:06,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:26:06,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +11: [2023-05-08 11:26:06,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +11: [2023-05-08 11:26:06,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:26:06,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +25: [2023-05-08 11:26:06,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +25: [2023-05-08 11:26:06,808] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +31: [2023-05-08 11:26:06,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:26:06,809] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:26:06,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:26:06,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:26:06,810] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +11: [2023-05-08 11:26:06,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +11: [2023-05-08 11:26:06,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +30: [2023-05-08 11:26:06,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +31: [2023-05-08 11:26:06,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +11: [2023-05-08 11:26:06,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:26:06,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:26:06,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:26:06,814] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:26:06,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +19: [2023-05-08 11:26:06,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +19: [2023-05-08 11:26:06,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:26:06,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +19: [2023-05-08 11:26:06,822] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +30: [2023-05-08 11:26:06,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:26:06,824] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +19: [2023-05-08 11:26:06,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:26:06,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +25: [2023-05-08 11:26:06,825] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:26:06,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 9: [2023-05-08 11:26:06,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +25: [2023-05-08 11:26:06,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +25: [2023-05-08 11:26:06,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:26:06,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 9: [2023-05-08 11:26:06,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +25: [2023-05-08 11:26:06,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +25: [2023-05-08 11:26:06,826] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 9: [2023-05-08 11:26:06,826] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +30: [2023-05-08 11:26:06,827] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:26:06,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 8: [2023-05-08 11:26:06,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +30: [2023-05-08 11:26:06,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:26:06,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 8: [2023-05-08 11:26:06,828] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:26:06,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:26:06,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +11: [2023-05-08 11:26:06,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:26:06,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:26:06,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +11: [2023-05-08 11:26:06,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:26:06,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,837] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +11: [2023-05-08 11:26:06,839] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:26:06,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:26:06,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +17: [2023-05-08 11:26:06,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:26:06,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:26:06,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +17: [2023-05-08 11:26:06,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +17: [2023-05-08 11:26:06,840] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:26:06,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:26:06,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,846] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:26:06,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +29: [2023-05-08 11:26:06,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +29: [2023-05-08 11:26:06,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +29: [2023-05-08 11:26:06,848] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +11: [2023-05-08 11:26:06,848] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:26:06,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,851] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +22: [2023-05-08 11:26:06,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +22: [2023-05-08 11:26:06,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +22: [2023-05-08 11:26:06,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +22: [2023-05-08 11:26:06,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +10: [2023-05-08 11:26:06,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:26:06,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:26:06,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:26:06,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +10: [2023-05-08 11:26:06,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +10: [2023-05-08 11:26:06,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +10: [2023-05-08 11:26:06,854] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:26:06,854] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:26:06,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:26:06,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +18: [2023-05-08 11:26:06,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +18: [2023-05-08 11:26:06,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +18: [2023-05-08 11:26:06,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +18: [2023-05-08 11:26:06,856] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +29: [2023-05-08 11:26:06,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:26:06,858] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +17: [2023-05-08 11:26:06,858] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:26:06,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +15: [2023-05-08 11:26:06,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:26:06,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:26:06,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:26:06,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +15: [2023-05-08 11:26:06,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +15: [2023-05-08 11:26:06,860] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:26:06,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +29: [2023-05-08 11:26:06,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +29: [2023-05-08 11:26:06,860] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +15: [2023-05-08 11:26:06,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +22: [2023-05-08 11:26:06,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +22: [2023-05-08 11:26:06,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +22: [2023-05-08 11:26:06,862] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 7: [2023-05-08 11:26:06,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:26:06,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:26:06,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +17: [2023-05-08 11:26:06,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:26:06,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:26:06,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +17: [2023-05-08 11:26:06,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +18: [2023-05-08 11:26:06,868] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +10: [2023-05-08 11:26:06,868] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:26:06,869] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:26:06,869] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:26:06,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,870] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 6: [2023-05-08 11:26:06,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 6: [2023-05-08 11:26:06,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +10: [2023-05-08 11:26:06,871] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +29: [2023-05-08 11:26:06,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:26:06,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +22: [2023-05-08 11:26:06,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +22: [2023-05-08 11:26:06,876] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 7: [2023-05-08 11:26:06,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:26:06,877] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:26:06,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +10: [2023-05-08 11:26:06,877] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +18: [2023-05-08 11:26:06,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +10: [2023-05-08 11:26:06,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +15: [2023-05-08 11:26:06,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:26:06,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:26:06,879] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:26:06,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +18: [2023-05-08 11:26:06,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:26:06,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 6: [2023-05-08 11:26:06,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +22: [2023-05-08 11:26:06,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +22: [2023-05-08 11:26:06,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +22: [2023-05-08 11:26:06,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +22: [2023-05-08 11:26:06,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +10: [2023-05-08 11:26:06,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:26:06,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +18: [2023-05-08 11:26:06,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +18: [2023-05-08 11:26:06,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 7: [2023-05-08 11:26:06,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +15: [2023-05-08 11:26:06,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:26:06,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +15: [2023-05-08 11:26:06,889] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +29: [2023-05-08 11:26:06,889] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:26:06,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:26:06,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,890] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +18: [2023-05-08 11:26:06,890] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:26:06,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +29: [2023-05-08 11:26:06,891] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +18: [2023-05-08 11:26:06,892] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +26: [2023-05-08 11:26:06,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +26: [2023-05-08 11:26:06,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +26: [2023-05-08 11:26:06,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:26:06,894] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:26:06,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +26: [2023-05-08 11:26:06,894] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,895] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:26:06,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,895] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:26:06,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:26:06,896] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:26:06,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +15: [2023-05-08 11:26:06,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +21: [2023-05-08 11:26:06,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:26:06,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:26:06,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +21: [2023-05-08 11:26:06,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:26:06,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +11: [2023-05-08 11:26:06,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +11: [2023-05-08 11:26:06,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +21: [2023-05-08 11:26:06,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:26:06,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,899] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:26:06,899] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 0: [2023-05-08 11:26:06,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +21: [2023-05-08 11:26:06,900] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +21: [2023-05-08 11:26:06,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 6: [2023-05-08 11:26:06,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 6: [2023-05-08 11:26:06,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 6: [2023-05-08 11:26:06,903] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,903] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,905] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:26:06,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,906] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:26:06,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +11: [2023-05-08 11:26:06,908] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +11: [2023-05-08 11:26:06,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +11: [2023-05-08 11:26:06,909] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,910] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +26: [2023-05-08 11:26:06,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 2: [2023-05-08 11:26:06,912] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 5: [2023-05-08 11:26:06,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +21: [2023-05-08 11:26:06,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,916] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:26:06,916] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,917] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +21: [2023-05-08 11:26:06,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,919] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 6: [2023-05-08 11:26:06,919] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 6: [2023-05-08 11:26:06,920] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +26: [2023-05-08 11:26:06,920] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:26:06,921] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,921] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:26:06,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:26:06,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:26:06,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:26:06,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +16: [2023-05-08 11:26:06,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +16: [2023-05-08 11:26:06,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:26:06,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,922] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +21: [2023-05-08 11:26:06,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:26:06,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,923] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +21: [2023-05-08 11:26:06,924] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:26:06,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +11: [2023-05-08 11:26:06,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +11: [2023-05-08 11:26:06,925] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:26:06,926] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +11: [2023-05-08 11:26:06,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +11: [2023-05-08 11:26:06,926] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +11: [2023-05-08 11:26:06,927] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 6: [2023-05-08 11:26:06,928] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +21: [2023-05-08 11:26:06,928] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,929] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,930] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +26: [2023-05-08 11:26:06,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,931] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +28: [2023-05-08 11:26:06,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +28: [2023-05-08 11:26:06,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +28: [2023-05-08 11:26:06,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +28: [2023-05-08 11:26:06,931] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +21: [2023-05-08 11:26:06,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:26:06,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +27: [2023-05-08 11:26:06,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +27: [2023-05-08 11:26:06,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +27: [2023-05-08 11:26:06,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:26:06,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:26:06,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:26:06,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:26:06,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,934] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,935] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:26:06,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +27: [2023-05-08 11:26:06,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 6: [2023-05-08 11:26:06,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:26:06,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +26: [2023-05-08 11:26:06,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:26:06,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +16: [2023-05-08 11:26:06,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,938] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +10: [2023-05-08 11:26:06,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +10: [2023-05-08 11:26:06,938] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 0: [2023-05-08 11:26:06,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:26:06,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 4: [2023-05-08 11:26:06,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 4: [2023-05-08 11:26:06,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +10: [2023-05-08 11:26:06,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 2: [2023-05-08 11:26:06,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 2: [2023-05-08 11:26:06,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 3: [2023-05-08 11:26:06,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 2: [2023-05-08 11:26:06,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,939] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,939] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 2: [2023-05-08 11:26:06,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,940] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:26:06,940] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,941] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 0: [2023-05-08 11:26:06,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:26:06,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +28: [2023-05-08 11:26:06,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,942] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +28: [2023-05-08 11:26:06,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,943] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 0: [2023-05-08 11:26:06,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 0: [2023-05-08 11:26:06,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +16: [2023-05-08 11:26:06,944] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:26:06,945] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,945] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 7: [2023-05-08 11:26:06,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 2: [2023-05-08 11:26:06,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 7: [2023-05-08 11:26:06,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +21: [2023-05-08 11:26:06,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +13: [2023-05-08 11:26:06,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 9: [2023-05-08 11:26:06,946] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +13: [2023-05-08 11:26:06,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +16: [2023-05-08 11:26:06,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:26:06,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,947] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +13: [2023-05-08 11:26:06,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +17: [2023-05-08 11:26:06,947] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +17: [2023-05-08 11:26:06,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +17: [2023-05-08 11:26:06,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 0: [2023-05-08 11:26:06,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:26:06,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +16: [2023-05-08 11:26:06,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,948] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +13: [2023-05-08 11:26:06,948] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +24: [2023-05-08 11:26:06,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +24: [2023-05-08 11:26:06,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +10: [2023-05-08 11:26:06,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +10: [2023-05-08 11:26:06,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,949] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:26:06,949] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +24: [2023-05-08 11:26:06,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 4: [2023-05-08 11:26:06,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +12: [2023-05-08 11:26:06,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +10: [2023-05-08 11:26:06,950] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +12: [2023-05-08 11:26:06,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +12: [2023-05-08 11:26:06,950] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 2: [2023-05-08 11:26:06,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,951] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +12: [2023-05-08 11:26:06,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +16: [2023-05-08 11:26:06,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:26:06,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 5: [2023-05-08 11:26:06,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 5: [2023-05-08 11:26:06,951] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 5: [2023-05-08 11:26:06,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 0: [2023-05-08 11:26:06,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:26:06,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,952] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:26:06,952] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,953] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +16: [2023-05-08 11:26:06,954] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,955] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,956] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +17: [2023-05-08 11:26:06,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:26:06,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:26:06,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +17: [2023-05-08 11:26:06,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:26:06,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:26:06,957] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:26:06,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:26:06,958] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,958] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:26:06,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +17: [2023-05-08 11:26:06,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:26:06,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,959] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 2: [2023-05-08 11:26:06,960] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +24: [2023-05-08 11:26:06,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +16: [2023-05-08 11:26:06,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:26:06,960] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +10: [2023-05-08 11:26:06,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:26:06,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:26:06,961] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:26:06,961] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +27: [2023-05-08 11:26:06,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:26:06,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:26:06,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +22: [2023-05-08 11:26:06,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +22: [2023-05-08 11:26:06,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +22: [2023-05-08 11:26:06,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +22: [2023-05-08 11:26:06,962] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +22: [2023-05-08 11:26:06,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:26:06,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +27: [2023-05-08 11:26:06,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:26:06,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +10: [2023-05-08 11:26:06,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +24: [2023-05-08 11:26:06,962] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +12: [2023-05-08 11:26:06,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +12: [2023-05-08 11:26:06,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +13: [2023-05-08 11:26:06,963] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +13: [2023-05-08 11:26:06,963] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:26:06,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +13: [2023-05-08 11:26:06,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:26:06,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +27: [2023-05-08 11:26:06,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +27: [2023-05-08 11:26:06,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:26:06,964] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:26:06,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +27: [2023-05-08 11:26:06,964] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +27: [2023-05-08 11:26:06,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:26:06,965] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +27: [2023-05-08 11:26:06,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:26:06,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:26:06,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:26:06,965] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:26:06,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +13: [2023-05-08 11:26:06,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +13: [2023-05-08 11:26:06,967] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +15: [2023-05-08 11:26:06,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +31: [2023-05-08 11:26:06,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +31: [2023-05-08 11:26:06,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:26:06,967] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +15: [2023-05-08 11:26:06,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +31: [2023-05-08 11:26:06,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +31: [2023-05-08 11:26:06,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:26:06,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:26:06,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:26:06,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:26:06,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +10: [2023-05-08 11:26:06,968] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +15: [2023-05-08 11:26:06,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +13: [2023-05-08 11:26:06,968] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +13: [2023-05-08 11:26:06,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,969] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +31: [2023-05-08 11:26:06,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,969] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 3: [2023-05-08 11:26:06,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 9: [2023-05-08 11:26:06,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:26:06,970] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +28: [2023-05-08 11:26:06,970] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:26:06,971] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 0: [2023-05-08 11:26:06,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:26:06,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 4: [2023-05-08 11:26:06,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +13: [2023-05-08 11:26:06,971] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 7: [2023-05-08 11:26:06,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +12: [2023-05-08 11:26:06,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +13: [2023-05-08 11:26:06,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +12: [2023-05-08 11:26:06,972] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +30: [2023-05-08 11:26:06,972] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +30: [2023-05-08 11:26:06,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +28: [2023-05-08 11:26:06,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +24: [2023-05-08 11:26:06,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:26:06,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +27: [2023-05-08 11:26:06,973] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:26:06,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +16: [2023-05-08 11:26:06,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +17: [2023-05-08 11:26:06,973] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +17: [2023-05-08 11:26:06,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +28: [2023-05-08 11:26:06,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +24: [2023-05-08 11:26:06,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +28: [2023-05-08 11:26:06,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:26:06,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +17: [2023-05-08 11:26:06,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:26:06,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +28: [2023-05-08 11:26:06,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:26:06,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +27: [2023-05-08 11:26:06,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:26:06,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:26:06,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +27: [2023-05-08 11:26:06,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:26:06,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:26:06,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:26:06,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +28: [2023-05-08 11:26:06,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +30: [2023-05-08 11:26:06,974] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +30: [2023-05-08 11:26:06,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +28: [2023-05-08 11:26:06,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +10: [2023-05-08 11:26:06,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:26:06,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +19: [2023-05-08 11:26:06,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +19: [2023-05-08 11:26:06,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:26:06,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:26:06,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +19: [2023-05-08 11:26:06,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +19: [2023-05-08 11:26:06,975] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:26:06,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 0: [2023-05-08 11:26:06,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +17: [2023-05-08 11:26:06,975] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +17: [2023-05-08 11:26:06,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +30: [2023-05-08 11:26:06,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +30: [2023-05-08 11:26:06,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +14: [2023-05-08 11:26:06,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +14: [2023-05-08 11:26:06,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +19: [2023-05-08 11:26:06,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +30: [2023-05-08 11:26:06,976] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:26:06,976] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +10: [2023-05-08 11:26:06,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +14: [2023-05-08 11:26:06,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +28: [2023-05-08 11:26:06,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:26:06,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:26:06,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +13: [2023-05-08 11:26:06,977] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +13: [2023-05-08 11:26:06,977] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +30: [2023-05-08 11:26:06,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 8: [2023-05-08 11:26:06,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +13: [2023-05-08 11:26:06,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +28: [2023-05-08 11:26:06,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. +15: [2023-05-08 11:26:06,978] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +22: [2023-05-08 11:26:06,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:26:06,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt. + 5: [2023-05-08 11:26:06,978] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:26:06,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +28: [2023-05-08 11:26:06,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 2: [2023-05-08 11:26:06,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt... + 2: [2023-05-08 11:26:06,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt... + 2: [2023-05-08 11:26:06,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt... + 2: [2023-05-08 11:26:06,979] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt... +13: [2023-05-08 11:26:06,979] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +13: [2023-05-08 11:26:06,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 5: [2023-05-08 11:26:06,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 0: [2023-05-08 11:26:06,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 0: [2023-05-08 11:26:06,980] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 3: [2023-05-08 11:26:06,980] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +24: [2023-05-08 11:26:06,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:26:06,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +24: [2023-05-08 11:26:06,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:26:06,981] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 3: [2023-05-08 11:26:06,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:26:06,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:26:06,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +13: [2023-05-08 11:26:06,981] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +13: [2023-05-08 11:26:06,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +28: [2023-05-08 11:26:06,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:26:06,982] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +31: [2023-05-08 11:26:06,982] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +22: [2023-05-08 11:26:06,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +22: [2023-05-08 11:26:06,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:26:06,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:26:06,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:26:06,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:26:06,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:26:06,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:26:06,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +12: [2023-05-08 11:26:06,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:26:06,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:26:06,983] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +31: [2023-05-08 11:26:06,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +31: [2023-05-08 11:26:06,983] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +23: [2023-05-08 11:26:06,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:26:06,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:26:06,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:26:06,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:26:06,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:26:06,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:26:06,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:26:06,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +16: [2023-05-08 11:26:06,984] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,984] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 5: [2023-05-08 11:26:06,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:26:06,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +24: [2023-05-08 11:26:06,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +30: [2023-05-08 11:26:06,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:26:06,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:26:06,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:26:06,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:26:06,985] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:26:06,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:26:06,985] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +14: [2023-05-08 11:26:06,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 4: [2023-05-08 11:26:06,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +19: [2023-05-08 11:26:06,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:26:06,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +14: [2023-05-08 11:26:06,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:26:06,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,986] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +13: [2023-05-08 11:26:06,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +14: [2023-05-08 11:26:06,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,987] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +19: [2023-05-08 11:26:06,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 3: [2023-05-08 11:26:06,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 8: [2023-05-08 11:26:06,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +30: [2023-05-08 11:26:06,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:26:06,987] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:26:06,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +22: [2023-05-08 11:26:06,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,988] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:26:06,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:26:06,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +13: [2023-05-08 11:26:06,989] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:26:06,989] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +19: [2023-05-08 11:26:06,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +19: [2023-05-08 11:26:06,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 8: [2023-05-08 11:26:06,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt... +15: [2023-05-08 11:26:06,990] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +28: [2023-05-08 11:26:06,990] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:26:06,991] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +15: [2023-05-08 11:26:06,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:26:06,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +25: [2023-05-08 11:26:06,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,991] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:26:06,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +25: [2023-05-08 11:26:06,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:26:06,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +25: [2023-05-08 11:26:06,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +13: [2023-05-08 11:26:06,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +25: [2023-05-08 11:26:06,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +13: [2023-05-08 11:26:06,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:26:06,992] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +15: [2023-05-08 11:26:06,992] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +31: [2023-05-08 11:26:06,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +25: [2023-05-08 11:26:06,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:26:06,993] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +31: [2023-05-08 11:26:06,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,994] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +25: [2023-05-08 11:26:06,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:26:06,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:26:06,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:26:06,994] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +12: [2023-05-08 11:26:06,995] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +31: [2023-05-08 11:26:06,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:26:06,996] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +13: [2023-05-08 11:26:06,997] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:26:06,997] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:26:06,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:26:06,998] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:26:06,999] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +28: [2023-05-08 11:26:06,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +19: [2023-05-08 11:26:06,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +19: [2023-05-08 11:26:06,999] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +19: [2023-05-08 11:26:07,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +19: [2023-05-08 11:26:07,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +28: [2023-05-08 11:26:07,000] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:26:07,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:26:07,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt... +27: [2023-05-08 11:26:07,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt... +27: [2023-05-08 11:26:07,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt... +27: [2023-05-08 11:26:07,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt... +16: [2023-05-08 11:26:07,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +23: [2023-05-08 11:26:07,000] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:26:07,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:26:07,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:26:07,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +13: [2023-05-08 11:26:07,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +31: [2023-05-08 11:26:07,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:26:07,001] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:26:07,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +14: [2023-05-08 11:26:07,001] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 8: [2023-05-08 11:26:07,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:26:07,002] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:26:07,002] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +14: [2023-05-08 11:26:07,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +14: [2023-05-08 11:26:07,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +30: [2023-05-08 11:26:07,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:26:07,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:26:07,003] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 8: [2023-05-08 11:26:07,003] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +25: [2023-05-08 11:26:07,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:26:07,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +29: [2023-05-08 11:26:07,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:26:07,004] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:26:07,005] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 8: [2023-05-08 11:26:07,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +15: [2023-05-08 11:26:07,005] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 8: [2023-05-08 11:26:07,006] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +29: [2023-05-08 11:26:07,006] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +14: [2023-05-08 11:26:07,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +14: [2023-05-08 11:26:07,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 3: [2023-05-08 11:26:07,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:26:07,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,007] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 3: [2023-05-08 11:26:07,007] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +16: [2023-05-08 11:26:07,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:26:07,008] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:26:07,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... + 3: [2023-05-08 11:26:07,009] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:26:07,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:26:07,009] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:26:07,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:26:07,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +14: [2023-05-08 11:26:07,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +16: [2023-05-08 11:26:07,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +12: [2023-05-08 11:26:07,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:26:07,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +24: [2023-05-08 11:26:07,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:26:07,010] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:26:07,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +24: [2023-05-08 11:26:07,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +24: [2023-05-08 11:26:07,010] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:26:07,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:26:07,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +24: [2023-05-08 11:26:07,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:26:07,011] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:26:07,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +24: [2023-05-08 11:26:07,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:26:07,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +12: [2023-05-08 11:26:07,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +21: [2023-05-08 11:26:07,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +21: [2023-05-08 11:26:07,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +21: [2023-05-08 11:26:07,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +21: [2023-05-08 11:26:07,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +16: [2023-05-08 11:26:07,011] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +23: [2023-05-08 11:26:07,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:26:07,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:26:07,012] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:26:07,012] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:26:07,013] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +29: [2023-05-08 11:26:07,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:26:07,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:26:07,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:26:07,014] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:26:07,014] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +29: [2023-05-08 11:26:07,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 3: [2023-05-08 11:26:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +25: [2023-05-08 11:26:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:26:07,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +29: [2023-05-08 11:26:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:26:07,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:26:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt. +23: [2023-05-08 11:26:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:26:07,015] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +29: [2023-05-08 11:26:07,015] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:26:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:26:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:26:07,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:26:07,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +29: [2023-05-08 11:26:07,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:26:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:26:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:26:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:26:07,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:26:07,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:26:07,016] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 8: [2023-05-08 11:26:07,017] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt... +29: [2023-05-08 11:26:07,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:26:07,017] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:26:07,018] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +28: [2023-05-08 11:26:07,019] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:26:07,019] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:26:07,020] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:26:07,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:26:07,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:26:07,021] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:26:07,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:26:07,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:26:07,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:26:07,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:26:07,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:26:07,023] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:26:07,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:26:07,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +24: [2023-05-08 11:26:07,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:26:07,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +25: [2023-05-08 11:26:07,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +21: [2023-05-08 11:26:07,024] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:26:07,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:26:07,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +11: [2023-05-08 11:26:07,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:26:07,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:26:07,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:26:07,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:26:07,025] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:26:07,025] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:26:07,026] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:26:07,028] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:26:07,029] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +24: [2023-05-08 11:26:07,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:26:07,029] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:26:07,030] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 3: [2023-05-08 11:26:07,031] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,031] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 3: [2023-05-08 11:26:07,032] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +24: [2023-05-08 11:26:07,032] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:26:07,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:26:07,033] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:26:07,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:26:07,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:26:07,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:26:07,034] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:26:07,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:26:07,034] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:26:07,035] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:26:07,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:26:07,036] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:26:07,036] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +24: [2023-05-08 11:26:07,037] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:26:07,038] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:26:07,039] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +24: [2023-05-08 11:26:07,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +24: [2023-05-08 11:26:07,040] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +24: [2023-05-08 11:26:07,040] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +24: [2023-05-08 11:26:07,042] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +24: [2023-05-08 11:26:07,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +24: [2023-05-08 11:26:07,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +24: [2023-05-08 11:26:07,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +24: [2023-05-08 11:26:07,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:26:07,042] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +24: [2023-05-08 11:26:07,043] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +24: [2023-05-08 11:26:07,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +21: [2023-05-08 11:26:07,043] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +24: [2023-05-08 11:26:07,044] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt... + 3: [2023-05-08 11:26:07,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt... + 3: [2023-05-08 11:26:07,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt... + 3: [2023-05-08 11:26:07,044] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt... +20: [2023-05-08 11:26:07,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +20: [2023-05-08 11:26:07,045] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +21: [2023-05-08 11:26:07,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:26:07,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:26:07,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +21: [2023-05-08 11:26:07,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +20: [2023-05-08 11:26:07,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 7: [2023-05-08 11:26:07,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +21: [2023-05-08 11:26:07,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +21: [2023-05-08 11:26:07,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 7: [2023-05-08 11:26:07,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 7: [2023-05-08 11:26:07,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 7: [2023-05-08 11:26:07,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,046] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:26:07,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:26:07,046] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:26:07,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:26:07,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:26:07,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:26:07,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:26:07,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:26:07,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:26:07,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:26:07,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:26:07,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +18: [2023-05-08 11:26:07,047] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +18: [2023-05-08 11:26:07,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:26:07,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:26:07,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:26:07,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:26:07,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +17: [2023-05-08 11:26:07,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:26:07,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:26:07,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:26:07,048] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:26:07,048] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:26:07,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:26:07,049] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +18: [2023-05-08 11:26:07,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:26:07,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:26:07,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:26:07,049] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:26:07,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:26:07,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:26:07,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +11: [2023-05-08 11:26:07,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +11: [2023-05-08 11:26:07,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:26:07,051] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +11: [2023-05-08 11:26:07,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:26:07,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:26:07,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,052] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:26:07,052] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,053] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:26:07,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt... +29: [2023-05-08 11:26:07,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt... +29: [2023-05-08 11:26:07,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt... +29: [2023-05-08 11:26:07,053] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt... +20: [2023-05-08 11:26:07,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 1: [2023-05-08 11:26:07,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,056] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,056] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:26:07,057] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +23: [2023-05-08 11:26:07,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt... +23: [2023-05-08 11:26:07,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt... +23: [2023-05-08 11:26:07,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt... +10: [2023-05-08 11:26:07,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:26:07,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt... + 1: [2023-05-08 11:26:07,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:26:07,058] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +10: [2023-05-08 11:26:07,058] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +20: [2023-05-08 11:26:07,059] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +17: [2023-05-08 11:26:07,059] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:26:07,060] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +25: [2023-05-08 11:26:07,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt... +25: [2023-05-08 11:26:07,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt... +25: [2023-05-08 11:26:07,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt... +25: [2023-05-08 11:26:07,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt... +10: [2023-05-08 11:26:07,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:26:07,061] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +11: [2023-05-08 11:26:07,061] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:26:07,062] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 7: [2023-05-08 11:26:07,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:26:07,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:26:07,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +18: [2023-05-08 11:26:07,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:26:07,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:26:07,067] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 1: [2023-05-08 11:26:07,067] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:26:07,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +20: [2023-05-08 11:26:07,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +20: [2023-05-08 11:26:07,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:26:07,068] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:26:07,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:26:07,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +20: [2023-05-08 11:26:07,069] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:26:07,069] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +17: [2023-05-08 11:26:07,070] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:26:07,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:26:07,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:26:07,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +18: [2023-05-08 11:26:07,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:26:07,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:26:07,071] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 1: [2023-05-08 11:26:07,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:26:07,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:26:07,071] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +17: [2023-05-08 11:26:07,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:26:07,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 7: [2023-05-08 11:26:07,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:26:07,072] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:26:07,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +17: [2023-05-08 11:26:07,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +17: [2023-05-08 11:26:07,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:26:07,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:26:07,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +17: [2023-05-08 11:26:07,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +17: [2023-05-08 11:26:07,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 7: [2023-05-08 11:26:07,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 7: [2023-05-08 11:26:07,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +17: [2023-05-08 11:26:07,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +17: [2023-05-08 11:26:07,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +17: [2023-05-08 11:26:07,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +18: [2023-05-08 11:26:07,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +18: [2023-05-08 11:26:07,073] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:26:07,073] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +18: [2023-05-08 11:26:07,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 7: [2023-05-08 11:26:07,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:26:07,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 7: [2023-05-08 11:26:07,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +24: [2023-05-08 11:26:07,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt... +24: [2023-05-08 11:26:07,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt... +24: [2023-05-08 11:26:07,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt... +24: [2023-05-08 11:26:07,074] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt... + 7: [2023-05-08 11:26:07,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:26:07,074] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:26:07,075] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 1: [2023-05-08 11:26:07,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:26:07,075] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:26:07,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:26:07,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +18: [2023-05-08 11:26:07,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:26:07,076] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +17: [2023-05-08 11:26:07,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:26:07,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:26:07,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:26:07,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:26:07,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:26:07,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:26:07,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 1: [2023-05-08 11:26:07,077] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +14: [2023-05-08 11:26:07,077] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +21: [2023-05-08 11:26:07,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +21: [2023-05-08 11:26:07,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +21: [2023-05-08 11:26:07,078] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +21: [2023-05-08 11:26:07,078] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:26:07,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:26:07,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:26:07,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +21: [2023-05-08 11:26:07,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:26:07,079] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:26:07,079] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:26:07,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:26:07,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 8: [2023-05-08 11:26:07,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 8: [2023-05-08 11:26:07,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 8: [2023-05-08 11:26:07,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 8: [2023-05-08 11:26:07,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,080] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,080] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:26:07,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 4: [2023-05-08 11:26:07,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 4: [2023-05-08 11:26:07,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 4: [2023-05-08 11:26:07,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 4: [2023-05-08 11:26:07,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:26:07,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 4: [2023-05-08 11:26:07,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 4: [2023-05-08 11:26:07,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 4: [2023-05-08 11:26:07,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:26:07,081] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:26:07,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:26:07,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:26:07,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:26:07,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:26:07,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 1: [2023-05-08 11:26:07,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:26:07,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:26:07,082] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:26:07,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +20: [2023-05-08 11:26:07,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +18: [2023-05-08 11:26:07,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 4: [2023-05-08 11:26:07,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +18: [2023-05-08 11:26:07,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +18: [2023-05-08 11:26:07,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +18: [2023-05-08 11:26:07,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +15: [2023-05-08 11:26:07,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:26:07,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +15: [2023-05-08 11:26:07,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:26:07,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:26:07,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +15: [2023-05-08 11:26:07,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:26:07,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 6: [2023-05-08 11:26:07,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 1: [2023-05-08 11:26:07,083] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:26:07,083] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:26:07,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,084] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +20: [2023-05-08 11:26:07,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +26: [2023-05-08 11:26:07,084] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +15: [2023-05-08 11:26:07,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +15: [2023-05-08 11:26:07,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +17: [2023-05-08 11:26:07,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:26:07,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +17: [2023-05-08 11:26:07,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +17: [2023-05-08 11:26:07,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:26:07,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +18: [2023-05-08 11:26:07,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:26:07,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:26:07,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +17: [2023-05-08 11:26:07,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,085] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:26:07,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:26:07,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:26:07,086] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:26:07,086] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 7: [2023-05-08 11:26:07,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,087] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:26:07,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 9: [2023-05-08 11:26:07,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 9: [2023-05-08 11:26:07,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 9: [2023-05-08 11:26:07,087] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +20: [2023-05-08 11:26:07,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:26:07,089] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +23: [2023-05-08 11:26:07,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +23: [2023-05-08 11:26:07,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +23: [2023-05-08 11:26:07,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +23: [2023-05-08 11:26:07,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +14: [2023-05-08 11:26:07,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +21: [2023-05-08 11:26:07,090] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,091] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:26:07,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:26:07,092] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:26:07,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:26:07,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:26:07,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:26:07,093] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:26:07,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:26:07,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt... +11: [2023-05-08 11:26:07,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt... +11: [2023-05-08 11:26:07,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt... +11: [2023-05-08 11:26:07,094] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt... + 4: [2023-05-08 11:26:07,094] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:26:07,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:26:07,095] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:26:07,095] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +21: [2023-05-08 11:26:07,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +21: [2023-05-08 11:26:07,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,096] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,096] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +26: [2023-05-08 11:26:07,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:26:07,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +21: [2023-05-08 11:26:07,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,097] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:26:07,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 4: [2023-05-08 11:26:07,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 4: [2023-05-08 11:26:07,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,098] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:26:07,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +20: [2023-05-08 11:26:07,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:26:07,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +23: [2023-05-08 11:26:07,099] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:26:07,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 9: [2023-05-08 11:26:07,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +14: [2023-05-08 11:26:07,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:26:07,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +31: [2023-05-08 11:26:07,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +31: [2023-05-08 11:26:07,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +21: [2023-05-08 11:26:07,100] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 1: [2023-05-08 11:26:07,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +27: [2023-05-08 11:26:07,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +27: [2023-05-08 11:26:07,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +27: [2023-05-08 11:26:07,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 1: [2023-05-08 11:26:07,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +31: [2023-05-08 11:26:07,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +27: [2023-05-08 11:26:07,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 1: [2023-05-08 11:26:07,101] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 1: [2023-05-08 11:26:07,101] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,102] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:26:07,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +23: [2023-05-08 11:26:07,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:26:07,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:26:07,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,103] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,103] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +14: [2023-05-08 11:26:07,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +14: [2023-05-08 11:26:07,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,104] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:26:07,104] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +21: [2023-05-08 11:26:07,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +14: [2023-05-08 11:26:07,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:26:07,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:26:07,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 8: [2023-05-08 11:26:07,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +21: [2023-05-08 11:26:07,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:26:07,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:26:07,105] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:26:07,105] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:26:07,106] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:26:07,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 4: [2023-05-08 11:26:07,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +26: [2023-05-08 11:26:07,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:26:07,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:26:07,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:26:07,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:26:07,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:26:07,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:26:07,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:26:07,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:26:07,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:26:07,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +26: [2023-05-08 11:26:07,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:26:07,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:26:07,107] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:26:07,107] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:26:07,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +21: [2023-05-08 11:26:07,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:26:07,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +18: [2023-05-08 11:26:07,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +26: [2023-05-08 11:26:07,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:26:07,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:26:07,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +14: [2023-05-08 11:26:07,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 1: [2023-05-08 11:26:07,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +14: [2023-05-08 11:26:07,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 1: [2023-05-08 11:26:07,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +14: [2023-05-08 11:26:07,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:26:07,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 8: [2023-05-08 11:26:07,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 8: [2023-05-08 11:26:07,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:26:07,108] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +18: [2023-05-08 11:26:07,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +18: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +21: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 5: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 5: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +29: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +29: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +29: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +15: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +14: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +14: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +14: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +21: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 4: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 4: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 5: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 5: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +14: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +14: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:26:07,109] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:26:07,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:26:07,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:26:07,110] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:26:07,110] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:26:07,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:26:07,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:26:07,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +31: [2023-05-08 11:26:07,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +27: [2023-05-08 11:26:07,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:26:07,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:26:07,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:26:07,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +27: [2023-05-08 11:26:07,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:26:07,111] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:26:07,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +26: [2023-05-08 11:26:07,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 4: [2023-05-08 11:26:07,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:26:07,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +26: [2023-05-08 11:26:07,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 4: [2023-05-08 11:26:07,112] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +26: [2023-05-08 11:26:07,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 4: [2023-05-08 11:26:07,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:26:07,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:26:07,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:26:07,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +31: [2023-05-08 11:26:07,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:26:07,111] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:26:07,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +25: [2023-05-08 11:26:07,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +25: [2023-05-08 11:26:07,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +25: [2023-05-08 11:26:07,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +20: [2023-05-08 11:26:07,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +20: [2023-05-08 11:26:07,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:26:07,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:26:07,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +11: [2023-05-08 11:26:07,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +11: [2023-05-08 11:26:07,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +18: [2023-05-08 11:26:07,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +20: [2023-05-08 11:26:07,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:26:07,113] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +11: [2023-05-08 11:26:07,113] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +20: [2023-05-08 11:26:07,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +20: [2023-05-08 11:26:07,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 8: [2023-05-08 11:26:07,114] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 8: [2023-05-08 11:26:07,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 8: [2023-05-08 11:26:07,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:26:07,115] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:26:07,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,115] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +20: [2023-05-08 11:26:07,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,116] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,116] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,117] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 9: [2023-05-08 11:26:07,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,117] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 9: [2023-05-08 11:26:07,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 7: [2023-05-08 11:26:07,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt... + 7: [2023-05-08 11:26:07,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt... + 7: [2023-05-08 11:26:07,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt... + 7: [2023-05-08 11:26:07,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt... +14: [2023-05-08 11:26:07,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:26:07,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt... +10: [2023-05-08 11:26:07,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt... +10: [2023-05-08 11:26:07,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt... +10: [2023-05-08 11:26:07,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt... + 4: [2023-05-08 11:26:07,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:26:07,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:26:07,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:26:07,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:26:07,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +29: [2023-05-08 11:26:07,118] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +14: [2023-05-08 11:26:07,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:26:07,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +14: [2023-05-08 11:26:07,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 4: [2023-05-08 11:26:07,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 1: [2023-05-08 11:26:07,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +17: [2023-05-08 11:26:07,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt... +17: [2023-05-08 11:26:07,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt... +17: [2023-05-08 11:26:07,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt... +17: [2023-05-08 11:26:07,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt... +14: [2023-05-08 11:26:07,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:26:07,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 4: [2023-05-08 11:26:07,119] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,119] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:26:07,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,120] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +29: [2023-05-08 11:26:07,120] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,121] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,121] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:26:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +19: [2023-05-08 11:26:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +19: [2023-05-08 11:26:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +25: [2023-05-08 11:26:07,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:26:07,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:26:07,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +25: [2023-05-08 11:26:07,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:26:07,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:26:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +30: [2023-05-08 11:26:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +26: [2023-05-08 11:26:07,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +27: [2023-05-08 11:26:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +11: [2023-05-08 11:26:07,122] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +30: [2023-05-08 11:26:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +30: [2023-05-08 11:26:07,122] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 1: [2023-05-08 11:26:07,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 1: [2023-05-08 11:26:07,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +26: [2023-05-08 11:26:07,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:26:07,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +26: [2023-05-08 11:26:07,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:26:07,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +27: [2023-05-08 11:26:07,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 1: [2023-05-08 11:26:07,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +30: [2023-05-08 11:26:07,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +18: [2023-05-08 11:26:07,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt... +18: [2023-05-08 11:26:07,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt... +18: [2023-05-08 11:26:07,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt... +18: [2023-05-08 11:26:07,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt... +26: [2023-05-08 11:26:07,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:26:07,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:26:07,123] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:26:07,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 5: [2023-05-08 11:26:07,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:26:07,123] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +23: [2023-05-08 11:26:07,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +23: [2023-05-08 11:26:07,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:26:07,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +23: [2023-05-08 11:26:07,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +11: [2023-05-08 11:26:07,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:26:07,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +23: [2023-05-08 11:26:07,124] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 5: [2023-05-08 11:26:07,125] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:26:07,125] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:26:07,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +20: [2023-05-08 11:26:07,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:26:07,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:26:07,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +31: [2023-05-08 11:26:07,126] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +31: [2023-05-08 11:26:07,126] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 0: > overriding learning rate value to 0.0002 + 0: > overriding minimum learning rate value to 2e-05 + 0: > overriding warmup iterations value to 0 +22: [2023-05-08 11:26:07,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:26:07,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:26:07,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +15: [2023-05-08 11:26:07,127] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 0: > overriding total number of iterations value to 1 + 0: > overriding decay style value to cosine +22: [2023-05-08 11:26:07,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +23: [2023-05-08 11:26:07,127] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt... + 1: [2023-05-08 11:26:07,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt... + 1: [2023-05-08 11:26:07,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt... + 1: [2023-05-08 11:26:07,128] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt... +23: [2023-05-08 11:26:07,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +20: [2023-05-08 11:26:07,129] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:26:07,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +20: [2023-05-08 11:26:07,130] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:26:07,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:26:07,130] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:26:07,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:26:07,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:26:07,131] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +27: [2023-05-08 11:26:07,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +27: [2023-05-08 11:26:07,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +27: [2023-05-08 11:26:07,131] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +31: [2023-05-08 11:26:07,132] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +31: [2023-05-08 11:26:07,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +30: [2023-05-08 11:26:07,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +30: [2023-05-08 11:26:07,132] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +30: [2023-05-08 11:26:07,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +30: [2023-05-08 11:26:07,133] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +29: [2023-05-08 11:26:07,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +19: [2023-05-08 11:26:07,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:26:07,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:26:07,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:26:07,134] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +19: [2023-05-08 11:26:07,134] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:26:07,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +11: [2023-05-08 11:26:07,135] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +11: [2023-05-08 11:26:07,135] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +26: [2023-05-08 11:26:07,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +26: [2023-05-08 11:26:07,136] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +26: [2023-05-08 11:26:07,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +26: [2023-05-08 11:26:07,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +22: [2023-05-08 11:26:07,137] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:26:07,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 1: [2023-05-08 11:26:07,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,138] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 1: [2023-05-08 11:26:07,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 1: [2023-05-08 11:26:07,138] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +22: [2023-05-08 11:26:07,139] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:26:07,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +25: [2023-05-08 11:26:07,141] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +25: [2023-05-08 11:26:07,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +25: [2023-05-08 11:26:07,141] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +29: [2023-05-08 11:26:07,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:26:07,142] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:26:07,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +29: [2023-05-08 11:26:07,142] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +31: [2023-05-08 11:26:07,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +25: [2023-05-08 11:26:07,143] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +25: [2023-05-08 11:26:07,143] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +22: [2023-05-08 11:26:07,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:26:07,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:26:07,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt... +21: [2023-05-08 11:26:07,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt... +21: [2023-05-08 11:26:07,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt... +21: [2023-05-08 11:26:07,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt... +26: [2023-05-08 11:26:07,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +22: [2023-05-08 11:26:07,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +26: [2023-05-08 11:26:07,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +22: [2023-05-08 11:26:07,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:26:07,144] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:26:07,144] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:26:07,145] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +22: [2023-05-08 11:26:07,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:26:07,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:26:07,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:26:07,145] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +30: [2023-05-08 11:26:07,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +25: [2023-05-08 11:26:07,146] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +25: [2023-05-08 11:26:07,146] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +29: [2023-05-08 11:26:07,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:26:07,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt... +14: [2023-05-08 11:26:07,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt... +14: [2023-05-08 11:26:07,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt... +14: [2023-05-08 11:26:07,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt... +29: [2023-05-08 11:26:07,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +11: [2023-05-08 11:26:07,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +11: [2023-05-08 11:26:07,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,148] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt... + 1: [2023-05-08 11:26:07,148] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt... + 5: [2023-05-08 11:26:07,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt... + 5: [2023-05-08 11:26:07,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt... +26: [2023-05-08 11:26:07,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,149] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +11: [2023-05-08 11:26:07,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +11: [2023-05-08 11:26:07,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +26: [2023-05-08 11:26:07,150] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +22: [2023-05-08 11:26:07,150] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +19: [2023-05-08 11:26:07,151] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +19: [2023-05-08 11:26:07,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +30: [2023-05-08 11:26:07,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +30: [2023-05-08 11:26:07,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 9: [2023-05-08 11:26:07,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,152] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:26:07,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:26:07,152] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:26:07,153] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:26:07,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +19: [2023-05-08 11:26:07,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +19: [2023-05-08 11:26:07,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +22: [2023-05-08 11:26:07,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +22: [2023-05-08 11:26:07,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +19: [2023-05-08 11:26:07,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +19: [2023-05-08 11:26:07,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +16: [2023-05-08 11:26:07,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:26:07,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:26:07,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:26:07,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:26:07,154] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:26:07,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +16: [2023-05-08 11:26:07,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:26:07,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:26:07,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:26:07,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +30: [2023-05-08 11:26:07,155] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +30: [2023-05-08 11:26:07,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +19: [2023-05-08 11:26:07,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:26:07,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:26:07,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +19: [2023-05-08 11:26:07,155] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:26:07,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +19: [2023-05-08 11:26:07,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +19: [2023-05-08 11:26:07,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:26:07,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:26:07,156] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:26:07,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:26:07,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:26:07,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt... +20: [2023-05-08 11:26:07,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt... +20: [2023-05-08 11:26:07,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt... +20: [2023-05-08 11:26:07,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt... +26: [2023-05-08 11:26:07,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt... +26: [2023-05-08 11:26:07,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt... +26: [2023-05-08 11:26:07,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt... +26: [2023-05-08 11:26:07,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt... + 4: [2023-05-08 11:26:07,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt... + 4: [2023-05-08 11:26:07,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt... + 4: [2023-05-08 11:26:07,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt... + 4: [2023-05-08 11:26:07,156] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt... +30: [2023-05-08 11:26:07,158] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt... + 0: [2023-05-08 11:26:07,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt... + 0: [2023-05-08 11:26:07,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt... + 0: [2023-05-08 11:26:07,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt... + 8: [2023-05-08 11:26:07,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt... + 8: [2023-05-08 11:26:07,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt... + 8: [2023-05-08 11:26:07,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt... + 8: [2023-05-08 11:26:07,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt... +30: [2023-05-08 11:26:07,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +19: [2023-05-08 11:26:07,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +19: [2023-05-08 11:26:07,159] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +19: [2023-05-08 11:26:07,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +19: [2023-05-08 11:26:07,160] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +19: [2023-05-08 11:26:07,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +19: [2023-05-08 11:26:07,160] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,163] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 6: [2023-05-08 11:26:07,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 6: [2023-05-08 11:26:07,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 6: [2023-05-08 11:26:07,164] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 9: [2023-05-08 11:26:07,165] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:26:07,165] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:26:07,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:26:07,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:26:07,166] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:26:07,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:26:07,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +19: [2023-05-08 11:26:07,167] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:26:07,168] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:26:07,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +22: [2023-05-08 11:26:07,168] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +30: [2023-05-08 11:26:07,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:26:07,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:26:07,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:26:07,170] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:26:07,170] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +16: [2023-05-08 11:26:07,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:26:07,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,171] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:26:07,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:26:07,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:26:07,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:26:07,173] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,174] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:26:07,174] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:26:07,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:26:07,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:26:07,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +13: [2023-05-08 11:26:07,175] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +19: [2023-05-08 11:26:07,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:26:07,176] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +19: [2023-05-08 11:26:07,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:26:07,177] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:26:07,177] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +19: [2023-05-08 11:26:07,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:26:07,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +13: [2023-05-08 11:26:07,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +13: [2023-05-08 11:26:07,178] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +13: [2023-05-08 11:26:07,178] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:26:07,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:26:07,179] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:26:07,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:26:07,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:26:07,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:26:07,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +28: [2023-05-08 11:26:07,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +28: [2023-05-08 11:26:07,180] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +28: [2023-05-08 11:26:07,180] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +13: [2023-05-08 11:26:07,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +28: [2023-05-08 11:26:07,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +28: [2023-05-08 11:26:07,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +31: [2023-05-08 11:26:07,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +31: [2023-05-08 11:26:07,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +31: [2023-05-08 11:26:07,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:26:07,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +31: [2023-05-08 11:26:07,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:26:07,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +31: [2023-05-08 11:26:07,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +31: [2023-05-08 11:26:07,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:26:07,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:26:07,181] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:26:07,181] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:26:07,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:26:07,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 9: [2023-05-08 11:26:07,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 9: [2023-05-08 11:26:07,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +31: [2023-05-08 11:26:07,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:26:07,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +16: [2023-05-08 11:26:07,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +16: [2023-05-08 11:26:07,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:26:07,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 9: [2023-05-08 11:26:07,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:26:07,182] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:26:07,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:26:07,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:26:07,182] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 9: [2023-05-08 11:26:07,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +16: [2023-05-08 11:26:07,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +16: [2023-05-08 11:26:07,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +16: [2023-05-08 11:26:07,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:26:07,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:26:07,183] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +19: [2023-05-08 11:26:07,183] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:26:07,184] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +19: [2023-05-08 11:26:07,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:26:07,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:26:07,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:26:07,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:26:07,184] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:26:07,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:26:07,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:26:07,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:26:07,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:26:07,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +19: [2023-05-08 11:26:07,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +19: [2023-05-08 11:26:07,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +19: [2023-05-08 11:26:07,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:26:07,185] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:26:07,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +19: [2023-05-08 11:26:07,185] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:26:07,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:26:07,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:26:07,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:26:07,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:26:07,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:26:07,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:26:07,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +22: [2023-05-08 11:26:07,187] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 9: [2023-05-08 11:26:07,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:26:07,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,188] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,188] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 9: [2023-05-08 11:26:07,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:26:07,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +22: [2023-05-08 11:26:07,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:26:07,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:26:07,190] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +13: [2023-05-08 11:26:07,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +13: [2023-05-08 11:26:07,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:26:07,190] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +16: [2023-05-08 11:26:07,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:26:07,191] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +13: [2023-05-08 11:26:07,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:26:07,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:26:07,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:26:07,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt... +22: [2023-05-08 11:26:07,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt... +22: [2023-05-08 11:26:07,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt... + 6: [2023-05-08 11:26:07,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +22: [2023-05-08 11:26:07,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt... + 6: [2023-05-08 11:26:07,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 6: [2023-05-08 11:26:07,193] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +12: [2023-05-08 11:26:07,193] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +28: [2023-05-08 11:26:07,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +31: [2023-05-08 11:26:07,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +31: [2023-05-08 11:26:07,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:26:07,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:26:07,198] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +31: [2023-05-08 11:26:07,198] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +13: [2023-05-08 11:26:07,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:26:07,199] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +12: [2023-05-08 11:26:07,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,201] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:26:07,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +13: [2023-05-08 11:26:07,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +13: [2023-05-08 11:26:07,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:26:07,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +13: [2023-05-08 11:26:07,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +13: [2023-05-08 11:26:07,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,202] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +13: [2023-05-08 11:26:07,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +13: [2023-05-08 11:26:07,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +13: [2023-05-08 11:26:07,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +13: [2023-05-08 11:26:07,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +13: [2023-05-08 11:26:07,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +13: [2023-05-08 11:26:07,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:26:07,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:26:07,203] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +13: [2023-05-08 11:26:07,203] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:26:07,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:26:07,204] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:26:07,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:26:07,204] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +31: [2023-05-08 11:26:07,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:26:07,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:26:07,205] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:26:07,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:26:07,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:26:07,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:26:07,206] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:26:07,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:26:07,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +31: [2023-05-08 11:26:07,208] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:26:07,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +31: [2023-05-08 11:26:07,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +31: [2023-05-08 11:26:07,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +31: [2023-05-08 11:26:07,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:26:07,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +31: [2023-05-08 11:26:07,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:26:07,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +31: [2023-05-08 11:26:07,209] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +13: [2023-05-08 11:26:07,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +31: [2023-05-08 11:26:07,209] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +31: [2023-05-08 11:26:07,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +31: [2023-05-08 11:26:07,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:26:07,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:26:07,212] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:26:07,212] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:26:07,213] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 2: [2023-05-08 11:26:07,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 2: [2023-05-08 11:26:07,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +28: [2023-05-08 11:26:07,215] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 3: [2023-05-08 11:26:07,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 3: [2023-05-08 11:26:07,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +12: [2023-05-08 11:26:07,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 2: [2023-05-08 11:26:07,216] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +31: [2023-05-08 11:26:07,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:26:07,217] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:26:07,217] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +31: [2023-05-08 11:26:07,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:26:07,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:26:07,218] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:26:07,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:26:07,218] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +13: [2023-05-08 11:26:07,219] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +13: [2023-05-08 11:26:07,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,219] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +13: [2023-05-08 11:26:07,220] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +13: [2023-05-08 11:26:07,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:26:07,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt... +30: [2023-05-08 11:26:07,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt... +30: [2023-05-08 11:26:07,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt... +30: [2023-05-08 11:26:07,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt... +16: [2023-05-08 11:26:07,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt... +16: [2023-05-08 11:26:07,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt... +16: [2023-05-08 11:26:07,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt... +16: [2023-05-08 11:26:07,221] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt... +19: [2023-05-08 11:26:07,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt... +19: [2023-05-08 11:26:07,222] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt... +19: [2023-05-08 11:26:07,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt... +19: [2023-05-08 11:26:07,223] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt... + 9: [2023-05-08 11:26:07,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt... + 9: [2023-05-08 11:26:07,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt... + 9: [2023-05-08 11:26:07,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt... + 9: [2023-05-08 11:26:07,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt... +28: [2023-05-08 11:26:07,224] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:26:07,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 2: [2023-05-08 11:26:07,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +28: [2023-05-08 11:26:07,225] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:26:07,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:26:07,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:26:07,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:26:07,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:26:07,226] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:26:07,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:26:07,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:26:07,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,227] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:26:07,228] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:26:07,230] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 3: [2023-05-08 11:26:07,233] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:26:07,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +15: [2023-05-08 11:26:07,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:26:07,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:26:07,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:26:07,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +15: [2023-05-08 11:26:07,235] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +15: [2023-05-08 11:26:07,235] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:26:07,236] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +15: [2023-05-08 11:26:07,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,237] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,237] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:26:07,238] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 2: [2023-05-08 11:26:07,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 6: [2023-05-08 11:26:07,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,240] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,240] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:26:07,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:26:07,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 6: [2023-05-08 11:26:07,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,242] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:26:07,243] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:26:07,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +13: [2023-05-08 11:26:07,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt... +13: [2023-05-08 11:26:07,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt... +13: [2023-05-08 11:26:07,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt... +13: [2023-05-08 11:26:07,243] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt... +12: [2023-05-08 11:26:07,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt... +12: [2023-05-08 11:26:07,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt... +12: [2023-05-08 11:26:07,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt... +12: [2023-05-08 11:26:07,244] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt... +28: [2023-05-08 11:26:07,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt... +28: [2023-05-08 11:26:07,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt... +28: [2023-05-08 11:26:07,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt... +28: [2023-05-08 11:26:07,247] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt... + 2: [2023-05-08 11:26:07,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 2: [2023-05-08 11:26:07,249] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 3: [2023-05-08 11:26:07,249] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +31: [2023-05-08 11:26:07,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt... +31: [2023-05-08 11:26:07,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt... +31: [2023-05-08 11:26:07,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt... +31: [2023-05-08 11:26:07,250] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt... + 3: [2023-05-08 11:26:07,251] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 3: [2023-05-08 11:26:07,253] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,254] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +15: [2023-05-08 11:26:07,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:26:07,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:26:07,257] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,258] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,258] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 6: [2023-05-08 11:26:07,264] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,264] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:26:07,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:26:07,265] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:26:07,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:26:07,266] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 3: [2023-05-08 11:26:07,266] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:26:07,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +10: [2023-05-08 11:26:07,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +10: [2023-05-08 11:26:07,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +10: [2023-05-08 11:26:07,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 6: [2023-05-08 11:26:07,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 6: [2023-05-08 11:26:07,268] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +15: [2023-05-08 11:26:07,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +15: [2023-05-08 11:26:07,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +15: [2023-05-08 11:26:07,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:26:07,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:26:07,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +15: [2023-05-08 11:26:07,269] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:26:07,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +15: [2023-05-08 11:26:07,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +15: [2023-05-08 11:26:07,270] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:26:07,273] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:26:07,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +15: [2023-05-08 11:26:07,274] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:26:07,274] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:26:07,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,277] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:26:07,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,278] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:26:07,278] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:26:07,279] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,280] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,283] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,283] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +14: [2023-05-08 11:26:07,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +14: [2023-05-08 11:26:07,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +14: [2023-05-08 11:26:07,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +18: [2023-05-08 11:26:07,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +18: [2023-05-08 11:26:07,286] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +14: [2023-05-08 11:26:07,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +18: [2023-05-08 11:26:07,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +18: [2023-05-08 11:26:07,287] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +20: [2023-05-08 11:26:07,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +20: [2023-05-08 11:26:07,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +20: [2023-05-08 11:26:07,288] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +20: [2023-05-08 11:26:07,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +10: [2023-05-08 11:26:07,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:26:07,291] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +10: [2023-05-08 11:26:07,295] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:26:07,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +10: [2023-05-08 11:26:07,295] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +18: [2023-05-08 11:26:07,296] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:26:07,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +14: [2023-05-08 11:26:07,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:26:07,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:26:07,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +18: [2023-05-08 11:26:07,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:26:07,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:26:07,297] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +18: [2023-05-08 11:26:07,299] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:26:07,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:26:07,302] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +15: [2023-05-08 11:26:07,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt... +15: [2023-05-08 11:26:07,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt... +15: [2023-05-08 11:26:07,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt... +15: [2023-05-08 11:26:07,304] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt... +10: [2023-05-08 11:26:07,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:26:07,305] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +10: [2023-05-08 11:26:07,306] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +10: [2023-05-08 11:26:07,307] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +14: [2023-05-08 11:26:07,309] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:26:07,310] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +18: [2023-05-08 11:26:07,311] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:26:07,311] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +18: [2023-05-08 11:26:07,312] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:26:07,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +14: [2023-05-08 11:26:07,313] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:26:07,313] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +20: [2023-05-08 11:26:07,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +20: [2023-05-08 11:26:07,314] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:26:07,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:26:07,315] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 8: [2023-05-08 11:26:07,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 8: [2023-05-08 11:26:07,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 8: [2023-05-08 11:26:07,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 8: [2023-05-08 11:26:07,315] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +18: [2023-05-08 11:26:07,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +20: [2023-05-08 11:26:07,316] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:26:07,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +20: [2023-05-08 11:26:07,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +20: [2023-05-08 11:26:07,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +20: [2023-05-08 11:26:07,316] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 6: [2023-05-08 11:26:07,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt... + 6: [2023-05-08 11:26:07,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt... + 6: [2023-05-08 11:26:07,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt... + 6: [2023-05-08 11:26:07,319] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt... +14: [2023-05-08 11:26:07,319] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +14: [2023-05-08 11:26:07,320] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +20: [2023-05-08 11:26:07,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +20: [2023-05-08 11:26:07,322] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +18: [2023-05-08 11:26:07,323] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +28: [2023-05-08 11:26:07,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +28: [2023-05-08 11:26:07,324] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +18: [2023-05-08 11:26:07,324] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 8: [2023-05-08 11:26:07,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:26:07,325] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 8: [2023-05-08 11:26:07,326] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +28: [2023-05-08 11:26:07,326] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 8: [2023-05-08 11:26:07,327] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +28: [2023-05-08 11:26:07,327] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +28: [2023-05-08 11:26:07,334] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +28: [2023-05-08 11:26:07,335] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +28: [2023-05-08 11:26:07,336] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +28: [2023-05-08 11:26:07,337] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:26:07,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +13: [2023-05-08 11:26:07,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +13: [2023-05-08 11:26:07,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +13: [2023-05-08 11:26:07,337] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 8: [2023-05-08 11:26:07,340] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,341] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +27: [2023-05-08 11:26:07,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt. +27: [2023-05-08 11:26:07,343] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 218 +27: [2023-05-08 11:26:07,347] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 218 +27: [2023-05-08 11:26:07,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt. +27: [2023-05-08 11:26:07,347] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 220 + 8: [2023-05-08 11:26:07,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +13: [2023-05-08 11:26:07,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:26:07,348] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +28: [2023-05-08 11:26:07,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +13: [2023-05-08 11:26:07,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +13: [2023-05-08 11:26:07,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +28: [2023-05-08 11:26:07,349] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 5: [2023-05-08 11:26:07,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 5: [2023-05-08 11:26:07,349] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +27: [2023-05-08 11:26:07,351] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 220 + 5: [2023-05-08 11:26:07,351] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 5: [2023-05-08 11:26:07,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 8: [2023-05-08 11:26:07,352] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:26:07,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +21: [2023-05-08 11:26:07,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +21: [2023-05-08 11:26:07,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 8: [2023-05-08 11:26:07,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:26:07,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 8: [2023-05-08 11:26:07,353] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 8: [2023-05-08 11:26:07,354] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +28: [2023-05-08 11:26:07,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:26:07,356] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:26:07,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +28: [2023-05-08 11:26:07,356] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +27: [2023-05-08 11:26:07,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +27: [2023-05-08 11:26:07,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +27: [2023-05-08 11:26:07,359] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +27: [2023-05-08 11:26:07,360] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +13: [2023-05-08 11:26:07,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:26:07,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 5: [2023-05-08 11:26:07,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 2: [2023-05-08 11:26:07,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt. + 5: [2023-05-08 11:26:07,362] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +28: [2023-05-08 11:26:07,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +28: [2023-05-08 11:26:07,361] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 2: [2023-05-08 11:26:07,362] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 22 +13: [2023-05-08 11:26:07,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +21: [2023-05-08 11:26:07,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:26:07,363] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +19: [2023-05-08 11:26:07,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +19: [2023-05-08 11:26:07,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +19: [2023-05-08 11:26:07,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +19: [2023-05-08 11:26:07,364] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +21: [2023-05-08 11:26:07,364] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:26:07,365] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:26:07,366] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 22 +13: [2023-05-08 11:26:07,366] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +13: [2023-05-08 11:26:07,367] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 1: [2023-05-08 11:26:07,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 1: [2023-05-08 11:26:07,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 1: [2023-05-08 11:26:07,367] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 1: [2023-05-08 11:26:07,368] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +27: [2023-05-08 11:26:07,368] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:26:07,369] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:26:07,371] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:26:07,373] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +13: [2023-05-08 11:26:07,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +13: [2023-05-08 11:26:07,374] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +19: [2023-05-08 11:26:07,376] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:26:07,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt. +13: [2023-05-08 11:26:07,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +13: [2023-05-08 11:26:07,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 2: [2023-05-08 11:26:07,377] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 18 + 1: [2023-05-08 11:26:07,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 1: [2023-05-08 11:26:07,377] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:26:07,378] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt. + 1: [2023-05-08 11:26:07,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:26:07,378] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 188 +19: [2023-05-08 11:26:07,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +19: [2023-05-08 11:26:07,378] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 4: [2023-05-08 11:26:07,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 4: [2023-05-08 11:26:07,380] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 1: [2023-05-08 11:26:07,380] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +19: [2023-05-08 11:26:07,381] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:26:07,381] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 18 +23: [2023-05-08 11:26:07,381] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 188 +21: [2023-05-08 11:26:07,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +21: [2023-05-08 11:26:07,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 4: [2023-05-08 11:26:07,382] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +21: [2023-05-08 11:26:07,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +21: [2023-05-08 11:26:07,383] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 4: [2023-05-08 11:26:07,384] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +25: [2023-05-08 11:26:07,385] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt. +25: [2023-05-08 11:26:07,385] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 206 + 5: [2023-05-08 11:26:07,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 5: [2023-05-08 11:26:07,385] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +21: [2023-05-08 11:26:07,387] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +21: [2023-05-08 11:26:07,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +27: [2023-05-08 11:26:07,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,388] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 5: [2023-05-08 11:26:07,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +27: [2023-05-08 11:26:07,388] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +27: [2023-05-08 11:26:07,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +27: [2023-05-08 11:26:07,389] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +25: [2023-05-08 11:26:07,389] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 206 +19: [2023-05-08 11:26:07,389] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 4: [2023-05-08 11:26:07,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +19: [2023-05-08 11:26:07,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 4: [2023-05-08 11:26:07,390] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +21: [2023-05-08 11:26:07,392] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 4: [2023-05-08 11:26:07,392] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 1: [2023-05-08 11:26:07,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +21: [2023-05-08 11:26:07,393] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +23: [2023-05-08 11:26:07,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +23: [2023-05-08 11:26:07,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 4: [2023-05-08 11:26:07,394] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +23: [2023-05-08 11:26:07,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +23: [2023-05-08 11:26:07,394] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +19: [2023-05-08 11:26:07,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +19: [2023-05-08 11:26:07,396] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +27: [2023-05-08 11:26:07,396] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:26:07,397] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt. +24: [2023-05-08 11:26:07,397] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 196 + 1: [2023-05-08 11:26:07,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 1: [2023-05-08 11:26:07,397] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +27: [2023-05-08 11:26:07,398] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +19: [2023-05-08 11:26:07,400] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +19: [2023-05-08 11:26:07,400] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +24: [2023-05-08 11:26:07,401] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 196 +23: [2023-05-08 11:26:07,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:26:07,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 4: [2023-05-08 11:26:07,404] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +23: [2023-05-08 11:26:07,404] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:26:07,405] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +27: [2023-05-08 11:26:07,405] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt. +27: [2023-05-08 11:26:07,405] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 222 + 4: [2023-05-08 11:26:07,406] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +27: [2023-05-08 11:26:07,406] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:26:07,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt. +19: [2023-05-08 11:26:07,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:26:07,408] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 234 +27: [2023-05-08 11:26:07,408] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +19: [2023-05-08 11:26:07,409] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +27: [2023-05-08 11:26:07,410] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 222 + 1: [2023-05-08 11:26:07,411] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:26:07,412] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 234 + 9: [2023-05-08 11:26:07,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +15: [2023-05-08 11:26:07,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +15: [2023-05-08 11:26:07,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 9: [2023-05-08 11:26:07,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 1: [2023-05-08 11:26:07,414] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 9: [2023-05-08 11:26:07,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 9: [2023-05-08 11:26:07,413] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 4: [2023-05-08 11:26:07,414] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:26:07,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +15: [2023-05-08 11:26:07,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +15: [2023-05-08 11:26:07,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 4: [2023-05-08 11:26:07,415] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 4: [2023-05-08 11:26:07,415] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +30: [2023-05-08 11:26:07,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 4: [2023-05-08 11:26:07,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +30: [2023-05-08 11:26:07,418] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +30: [2023-05-08 11:26:07,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +30: [2023-05-08 11:26:07,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 0: [2023-05-08 11:26:07,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 4: [2023-05-08 11:26:07,418] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +17: [2023-05-08 11:26:07,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +24: [2023-05-08 11:26:07,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +29: [2023-05-08 11:26:07,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt. +17: [2023-05-08 11:26:07,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +29: [2023-05-08 11:26:07,419] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 238 +17: [2023-05-08 11:26:07,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 0: [2023-05-08 11:26:07,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 0: [2023-05-08 11:26:07,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +24: [2023-05-08 11:26:07,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +24: [2023-05-08 11:26:07,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +17: [2023-05-08 11:26:07,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 0: [2023-05-08 11:26:07,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +24: [2023-05-08 11:26:07,419] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +23: [2023-05-08 11:26:07,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 7: [2023-05-08 11:26:07,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. + 7: [2023-05-08 11:26:07,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +23: [2023-05-08 11:26:07,421] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 7: [2023-05-08 11:26:07,421] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +15: [2023-05-08 11:26:07,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:26:07,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:26:07,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:26:07,423] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 238 +23: [2023-05-08 11:26:07,423] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +23: [2023-05-08 11:26:07,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +25: [2023-05-08 11:26:07,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +25: [2023-05-08 11:26:07,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +25: [2023-05-08 11:26:07,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +25: [2023-05-08 11:26:07,423] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +15: [2023-05-08 11:26:07,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:26:07,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +15: [2023-05-08 11:26:07,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 9: [2023-05-08 11:26:07,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 9: [2023-05-08 11:26:07,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +23: [2023-05-08 11:26:07,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +15: [2023-05-08 11:26:07,425] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +16: [2023-05-08 11:26:07,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +16: [2023-05-08 11:26:07,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +12: [2023-05-08 11:26:07,426] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +12: [2023-05-08 11:26:07,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +12: [2023-05-08 11:26:07,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +12: [2023-05-08 11:26:07,427] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +16: [2023-05-08 11:26:07,428] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +17: [2023-05-08 11:26:07,428] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +16: [2023-05-08 11:26:07,429] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt. +17: [2023-05-08 11:26:07,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +17: [2023-05-08 11:26:07,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +30: [2023-05-08 11:26:07,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:26:07,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +30: [2023-05-08 11:26:07,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:26:07,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +30: [2023-05-08 11:26:07,429] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +24: [2023-05-08 11:26:07,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:26:07,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:26:07,430] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,431] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +24: [2023-05-08 11:26:07,432] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:26:07,433] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:26:07,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:26:07,434] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:26:07,434] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt. +11: [2023-05-08 11:26:07,435] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 88 +25: [2023-05-08 11:26:07,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +25: [2023-05-08 11:26:07,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +25: [2023-05-08 11:26:07,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +25: [2023-05-08 11:26:07,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +15: [2023-05-08 11:26:07,435] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:26:07,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +12: [2023-05-08 11:26:07,435] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... + 7: [2023-05-08 11:26:07,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +16: [2023-05-08 11:26:07,436] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:26:07,437] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +16: [2023-05-08 11:26:07,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:26:07,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +12: [2023-05-08 11:26:07,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +15: [2023-05-08 11:26:07,438] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:26:07,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +16: [2023-05-08 11:26:07,438] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +16: [2023-05-08 11:26:07,439] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt... +11: [2023-05-08 11:26:07,439] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 88 +23: [2023-05-08 11:26:07,439] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:26:07,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:26:07,440] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +15: [2023-05-08 11:26:07,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +15: [2023-05-08 11:26:07,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 9: [2023-05-08 11:26:07,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +30: [2023-05-08 11:26:07,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,441] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,441] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 9: [2023-05-08 11:26:07,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +23: [2023-05-08 11:26:07,442] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 5: [2023-05-08 11:26:07,444] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt. + 5: [2023-05-08 11:26:07,445] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 42 + 0: [2023-05-08 11:26:07,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +23: [2023-05-08 11:26:07,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt. +17: [2023-05-08 11:26:07,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +23: [2023-05-08 11:26:07,446] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 186 +17: [2023-05-08 11:26:07,445] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt. + 0: [2023-05-08 11:26:07,446] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 3: [2023-05-08 11:26:07,446] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 30 + 7: [2023-05-08 11:26:07,446] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:26:07,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +30: [2023-05-08 11:26:07,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +30: [2023-05-08 11:26:07,447] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:26:07,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +30: [2023-05-08 11:26:07,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +30: [2023-05-08 11:26:07,447] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +30: [2023-05-08 11:26:07,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt. +30: [2023-05-08 11:26:07,448] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +12: [2023-05-08 11:26:07,448] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,448] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 58 + 5: [2023-05-08 11:26:07,449] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 42 +17: [2023-05-08 11:26:07,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +17: [2023-05-08 11:26:07,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +17: [2023-05-08 11:26:07,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +17: [2023-05-08 11:26:07,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:26:07,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +12: [2023-05-08 11:26:07,449] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +24: [2023-05-08 11:26:07,449] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +24: [2023-05-08 11:26:07,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +24: [2023-05-08 11:26:07,450] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +24: [2023-05-08 11:26:07,450] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 3: [2023-05-08 11:26:07,450] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 30 +23: [2023-05-08 11:26:07,450] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 186 + 0: [2023-05-08 11:26:07,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,451] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 0: [2023-05-08 11:26:07,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 0: [2023-05-08 11:26:07,451] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 7: [2023-05-08 11:26:07,453] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 58 +25: [2023-05-08 11:26:07,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +25: [2023-05-08 11:26:07,453] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +25: [2023-05-08 11:26:07,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +25: [2023-05-08 11:26:07,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +25: [2023-05-08 11:26:07,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +25: [2023-05-08 11:26:07,454] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +16: [2023-05-08 11:26:07,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +17: [2023-05-08 11:26:07,454] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt. +17: [2023-05-08 11:26:07,455] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 136 +16: [2023-05-08 11:26:07,456] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 9: [2023-05-08 11:26:07,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:26:07,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:26:07,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +12: [2023-05-08 11:26:07,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:26:07,457] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 9: [2023-05-08 11:26:07,458] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 2: [2023-05-08 11:26:07,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt. +24: [2023-05-08 11:26:07,458] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +11: [2023-05-08 11:26:07,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +11: [2023-05-08 11:26:07,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +17: [2023-05-08 11:26:07,459] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 136 + 7: [2023-05-08 11:26:07,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +11: [2023-05-08 11:26:07,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 2: [2023-05-08 11:26:07,459] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 16 +11: [2023-05-08 11:26:07,459] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +24: [2023-05-08 11:26:07,460] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 6: [2023-05-08 11:26:07,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 6: [2023-05-08 11:26:07,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 6: [2023-05-08 11:26:07,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 7: [2023-05-08 11:26:07,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +16: [2023-05-08 11:26:07,460] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:26:07,461] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 6: [2023-05-08 11:26:07,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +26: [2023-05-08 11:26:07,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +26: [2023-05-08 11:26:07,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +26: [2023-05-08 11:26:07,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +25: [2023-05-08 11:26:07,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt. +25: [2023-05-08 11:26:07,461] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 202 +26: [2023-05-08 11:26:07,461] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +22: [2023-05-08 11:26:07,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +22: [2023-05-08 11:26:07,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +22: [2023-05-08 11:26:07,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +22: [2023-05-08 11:26:07,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 7: [2023-05-08 11:26:07,462] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +29: [2023-05-08 11:26:07,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +29: [2023-05-08 11:26:07,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +29: [2023-05-08 11:26:07,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +12: [2023-05-08 11:26:07,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:26:07,463] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +12: [2023-05-08 11:26:07,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +12: [2023-05-08 11:26:07,463] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 2: [2023-05-08 11:26:07,463] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 16 + 7: [2023-05-08 11:26:07,464] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +21: [2023-05-08 11:26:07,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt. +29: [2023-05-08 11:26:07,464] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +21: [2023-05-08 11:26:07,464] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 168 +25: [2023-05-08 11:26:07,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +25: [2023-05-08 11:26:07,466] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 202 +24: [2023-05-08 11:26:07,466] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +16: [2023-05-08 11:26:07,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,467] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt. +25: [2023-05-08 11:26:07,467] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +21: [2023-05-08 11:26:07,468] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 168 +16: [2023-05-08 11:26:07,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +24: [2023-05-08 11:26:07,468] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... + 7: [2023-05-08 11:26:07,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt... +11: [2023-05-08 11:26:07,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:26:07,469] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:26:07,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,470] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +31: [2023-05-08 11:26:07,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +31: [2023-05-08 11:26:07,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +31: [2023-05-08 11:26:07,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +31: [2023-05-08 11:26:07,470] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +26: [2023-05-08 11:26:07,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:26:07,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:26:07,471] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:26:07,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:26:07,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:26:07,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:26:07,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:26:07,472] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:26:07,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:26:07,473] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:26:07,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:26:07,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:26:07,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 2: [2023-05-08 11:26:07,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 6: [2023-05-08 11:26:07,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:26:07,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 2: [2023-05-08 11:26:07,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +29: [2023-05-08 11:26:07,474] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt. +29: [2023-05-08 11:26:07,474] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +29: [2023-05-08 11:26:07,474] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 232 + 3: [2023-05-08 11:26:07,475] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt. + 3: [2023-05-08 11:26:07,475] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 24 +29: [2023-05-08 11:26:07,479] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 232 + 3: [2023-05-08 11:26:07,479] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 24 + 3: [2023-05-08 11:26:07,479] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt. + 3: [2023-05-08 11:26:07,480] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 28 +25: [2023-05-08 11:26:07,483] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt. +31: [2023-05-08 11:26:07,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +25: [2023-05-08 11:26:07,483] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 200 +31: [2023-05-08 11:26:07,483] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:26:07,483] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 28 +31: [2023-05-08 11:26:07,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:26:07,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:26:07,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:26:07,484] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +24: [2023-05-08 11:26:07,484] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt. +24: [2023-05-08 11:26:07,485] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 194 +22: [2023-05-08 11:26:07,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 6: [2023-05-08 11:26:07,485] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +31: [2023-05-08 11:26:07,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:26:07,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:26:07,486] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt. + 6: [2023-05-08 11:26:07,486] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +16: [2023-05-08 11:26:07,487] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 132 + 2: [2023-05-08 11:26:07,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:26:07,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +25: [2023-05-08 11:26:07,487] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 200 + 2: [2023-05-08 11:26:07,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:26:07,487] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt. + 2: [2023-05-08 11:26:07,487] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:26:07,487] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 26 + 6: [2023-05-08 11:26:07,488] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,488] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +24: [2023-05-08 11:26:07,489] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 194 +11: [2023-05-08 11:26:07,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:26:07,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +22: [2023-05-08 11:26:07,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:26:07,489] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +22: [2023-05-08 11:26:07,489] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +22: [2023-05-08 11:26:07,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 2: [2023-05-08 11:26:07,490] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:26:07,490] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 132 + 3: [2023-05-08 11:26:07,491] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 26 +26: [2023-05-08 11:26:07,490] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:26:07,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +26: [2023-05-08 11:26:07,491] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:26:07,491] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +26: [2023-05-08 11:26:07,492] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +26: [2023-05-08 11:26:07,492] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +11: [2023-05-08 11:26:07,493] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,494] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +11: [2023-05-08 11:26:07,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +29: [2023-05-08 11:26:07,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:26:07,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:26:07,494] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:26:07,495] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:26:07,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +29: [2023-05-08 11:26:07,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +29: [2023-05-08 11:26:07,495] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +31: [2023-05-08 11:26:07,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 3: [2023-05-08 11:26:07,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 3: [2023-05-08 11:26:07,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +10: [2023-05-08 11:26:07,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +11: [2023-05-08 11:26:07,496] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 3: [2023-05-08 11:26:07,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +25: [2023-05-08 11:26:07,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt. +10: [2023-05-08 11:26:07,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +10: [2023-05-08 11:26:07,496] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +25: [2023-05-08 11:26:07,497] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 204 +10: [2023-05-08 11:26:07,497] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +21: [2023-05-08 11:26:07,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt. +21: [2023-05-08 11:26:07,498] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 172 + 2: [2023-05-08 11:26:07,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +31: [2023-05-08 11:26:07,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +29: [2023-05-08 11:26:07,498] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 2: [2023-05-08 11:26:07,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 2: [2023-05-08 11:26:07,499] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 2: [2023-05-08 11:26:07,499] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +31: [2023-05-08 11:26:07,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +29: [2023-05-08 11:26:07,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +31: [2023-05-08 11:26:07,500] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +17: [2023-05-08 11:26:07,500] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt. +17: [2023-05-08 11:26:07,500] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 140 +24: [2023-05-08 11:26:07,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt. +27: [2023-05-08 11:26:07,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt. +27: [2023-05-08 11:26:07,501] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 216 +24: [2023-05-08 11:26:07,501] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 198 +18: [2023-05-08 11:26:07,501] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt. +25: [2023-05-08 11:26:07,501] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 204 +18: [2023-05-08 11:26:07,501] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 144 +21: [2023-05-08 11:26:07,502] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 172 +22: [2023-05-08 11:26:07,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,502] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 8: [2023-05-08 11:26:07,503] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +22: [2023-05-08 11:26:07,503] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +17: [2023-05-08 11:26:07,504] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 140 +18: [2023-05-08 11:26:07,505] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 144 +27: [2023-05-08 11:26:07,505] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 216 +10: [2023-05-08 11:26:07,506] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:26:07,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:26:07,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +24: [2023-05-08 11:26:07,507] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 198 + 8: [2023-05-08 11:26:07,507] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 3: [2023-05-08 11:26:07,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:26:07,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 3: [2023-05-08 11:26:07,507] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:26:07,508] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 2: [2023-05-08 11:26:07,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +31: [2023-05-08 11:26:07,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt. +14: [2023-05-08 11:26:07,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +14: [2023-05-08 11:26:07,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +14: [2023-05-08 11:26:07,509] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 3: [2023-05-08 11:26:07,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 1: [2023-05-08 11:26:07,510] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 8 + 2: [2023-05-08 11:26:07,510] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +14: [2023-05-08 11:26:07,510] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +31: [2023-05-08 11:26:07,511] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 7: [2023-05-08 11:26:07,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt. + 7: [2023-05-08 11:26:07,511] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 62 + 2: [2023-05-08 11:26:07,511] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt. +31: [2023-05-08 11:26:07,512] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 2: [2023-05-08 11:26:07,512] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 20 + 8: [2023-05-08 11:26:07,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +31: [2023-05-08 11:26:07,513] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 1: [2023-05-08 11:26:07,515] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 8 + 8: [2023-05-08 11:26:07,515] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 7: [2023-05-08 11:26:07,516] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 62 + 2: [2023-05-08 11:26:07,517] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 20 + 8: [2023-05-08 11:26:07,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 8: [2023-05-08 11:26:07,517] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:26:07,517] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:26:07,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt. + 2: [2023-05-08 11:26:07,518] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +11: [2023-05-08 11:26:07,518] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 92 +14: [2023-05-08 11:26:07,518] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:26:07,519] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt. +10: [2023-05-08 11:26:07,519] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +20: [2023-05-08 11:26:07,519] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 162 + 2: [2023-05-08 11:26:07,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +10: [2023-05-08 11:26:07,520] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:26:07,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +14: [2023-05-08 11:26:07,520] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +10: [2023-05-08 11:26:07,521] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:26:07,521] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +10: [2023-05-08 11:26:07,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:26:07,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +11: [2023-05-08 11:26:07,522] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 92 +14: [2023-05-08 11:26:07,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +14: [2023-05-08 11:26:07,522] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:26:07,523] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt. + 5: [2023-05-08 11:26:07,523] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 46 +20: [2023-05-08 11:26:07,523] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 162 +11: [2023-05-08 11:26:07,522] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt. +11: [2023-05-08 11:26:07,523] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 94 + 8: [2023-05-08 11:26:07,525] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +11: [2023-05-08 11:26:07,527] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 94 + 5: [2023-05-08 11:26:07,528] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 46 +10: [2023-05-08 11:26:07,528] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt. +10: [2023-05-08 11:26:07,529] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 82 + 3: [2023-05-08 11:26:07,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,529] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 3: [2023-05-08 11:26:07,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 8: [2023-05-08 11:26:07,530] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 7: [2023-05-08 11:26:07,530] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt. + 7: [2023-05-08 11:26:07,531] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 56 +14: [2023-05-08 11:26:07,531] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +10: [2023-05-08 11:26:07,533] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 82 +23: [2023-05-08 11:26:07,533] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt. +23: [2023-05-08 11:26:07,534] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 184 +14: [2023-05-08 11:26:07,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 3: [2023-05-08 11:26:07,534] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,534] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 7: [2023-05-08 11:26:07,535] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 56 +18: [2023-05-08 11:26:07,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt. +18: [2023-05-08 11:26:07,536] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 146 +14: [2023-05-08 11:26:07,536] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:26:07,537] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +17: [2023-05-08 11:26:07,537] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt. +17: [2023-05-08 11:26:07,537] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 142 +21: [2023-05-08 11:26:07,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt. +21: [2023-05-08 11:26:07,538] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 174 +23: [2023-05-08 11:26:07,538] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 184 + 8: [2023-05-08 11:26:07,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,538] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 3: [2023-05-08 11:26:07,538] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,540] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +18: [2023-05-08 11:26:07,540] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 146 +10: [2023-05-08 11:26:07,539] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt. +10: [2023-05-08 11:26:07,539] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 86 + 8: [2023-05-08 11:26:07,541] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,541] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +17: [2023-05-08 11:26:07,541] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 142 +21: [2023-05-08 11:26:07,542] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 174 +28: [2023-05-08 11:26:07,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +28: [2023-05-08 11:26:07,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +28: [2023-05-08 11:26:07,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +28: [2023-05-08 11:26:07,542] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +10: [2023-05-08 11:26:07,543] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 86 +14: [2023-05-08 11:26:07,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:26:07,544] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +23: [2023-05-08 11:26:07,544] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt. +23: [2023-05-08 11:26:07,545] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 190 +14: [2023-05-08 11:26:07,545] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:26:07,547] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 5: [2023-05-08 11:26:07,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 5: [2023-05-08 11:26:07,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 5: [2023-05-08 11:26:07,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 5: [2023-05-08 11:26:07,547] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 1: [2023-05-08 11:26:07,548] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt. + 1: [2023-05-08 11:26:07,548] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 14 +23: [2023-05-08 11:26:07,549] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 190 + 8: [2023-05-08 11:26:07,549] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:26:07,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 8: [2023-05-08 11:26:07,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt. +15: [2023-05-08 11:26:07,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +15: [2023-05-08 11:26:07,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 8: [2023-05-08 11:26:07,550] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 64 +15: [2023-05-08 11:26:07,550] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 8: [2023-05-08 11:26:07,551] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 1: [2023-05-08 11:26:07,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt. + 1: [2023-05-08 11:26:07,551] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 10 + 4: [2023-05-08 11:26:07,551] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 4: [2023-05-08 11:26:07,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 4: [2023-05-08 11:26:07,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 4: [2023-05-08 11:26:07,552] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +28: [2023-05-08 11:26:07,552] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 1: [2023-05-08 11:26:07,553] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 14 +28: [2023-05-08 11:26:07,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +13: [2023-05-08 11:26:07,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +13: [2023-05-08 11:26:07,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +13: [2023-05-08 11:26:07,553] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +28: [2023-05-08 11:26:07,553] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +28: [2023-05-08 11:26:07,554] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +13: [2023-05-08 11:26:07,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 8: [2023-05-08 11:26:07,554] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 64 + 7: [2023-05-08 11:26:07,554] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 1: [2023-05-08 11:26:07,555] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 10 + 7: [2023-05-08 11:26:07,555] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 4: [2023-05-08 11:26:07,556] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt. + 4: [2023-05-08 11:26:07,556] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 32 + 5: [2023-05-08 11:26:07,558] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:26:07,558] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt. +17: [2023-05-08 11:26:07,558] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 138 +15: [2023-05-08 11:26:07,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:26:07,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:26:07,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:26:07,559] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 7: [2023-05-08 11:26:07,559] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 7: [2023-05-08 11:26:07,560] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 4: [2023-05-08 11:26:07,561] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 32 +22: [2023-05-08 11:26:07,561] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt. +15: [2023-05-08 11:26:07,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:26:07,562] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 176 +15: [2023-05-08 11:26:07,561] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 4: [2023-05-08 11:26:07,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:26:07,562] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 138 + 4: [2023-05-08 11:26:07,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +13: [2023-05-08 11:26:07,562] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 4: [2023-05-08 11:26:07,563] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +13: [2023-05-08 11:26:07,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 4: [2023-05-08 11:26:07,564] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:26:07,564] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt. +22: [2023-05-08 11:26:07,565] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 180 +13: [2023-05-08 11:26:07,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:26:07,565] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt. +15: [2023-05-08 11:26:07,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +21: [2023-05-08 11:26:07,565] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 170 +13: [2023-05-08 11:26:07,565] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 7: [2023-05-08 11:26:07,566] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +22: [2023-05-08 11:26:07,566] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 176 +18: [2023-05-08 11:26:07,566] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt. +18: [2023-05-08 11:26:07,566] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 150 + 7: [2023-05-08 11:26:07,567] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:26:07,568] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt. + 5: [2023-05-08 11:26:07,568] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 40 +22: [2023-05-08 11:26:07,569] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 180 +21: [2023-05-08 11:26:07,569] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 170 + 7: [2023-05-08 11:26:07,569] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +18: [2023-05-08 11:26:07,570] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 150 +31: [2023-05-08 11:26:07,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt. +31: [2023-05-08 11:26:07,571] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 252 + 0: [2023-05-08 11:26:07,571] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt. + 0: [2023-05-08 11:26:07,572] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 6 + 7: [2023-05-08 11:26:07,572] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +28: [2023-05-08 11:26:07,572] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt. +28: [2023-05-08 11:26:07,573] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 230 +28: [2023-05-08 11:26:07,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 5: [2023-05-08 11:26:07,573] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 40 +28: [2023-05-08 11:26:07,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +15: [2023-05-08 11:26:07,573] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 0: [2023-05-08 11:26:07,574] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +31: [2023-05-08 11:26:07,575] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 252 + 1: [2023-05-08 11:26:07,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt. + 1: [2023-05-08 11:26:07,576] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 12 +28: [2023-05-08 11:26:07,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +13: [2023-05-08 11:26:07,575] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +28: [2023-05-08 11:26:07,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +15: [2023-05-08 11:26:07,576] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 5: [2023-05-08 11:26:07,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,576] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 6 + 5: [2023-05-08 11:26:07,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +14: [2023-05-08 11:26:07,576] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt. + 5: [2023-05-08 11:26:07,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 5: [2023-05-08 11:26:07,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +14: [2023-05-08 11:26:07,577] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 112 +28: [2023-05-08 11:26:07,577] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 230 +15: [2023-05-08 11:26:07,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +15: [2023-05-08 11:26:07,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +13: [2023-05-08 11:26:07,577] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +13: [2023-05-08 11:26:07,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +13: [2023-05-08 11:26:07,577] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 9: [2023-05-08 11:26:07,578] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt. + 9: [2023-05-08 11:26:07,578] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 72 + 0: [2023-05-08 11:26:07,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt. +13: [2023-05-08 11:26:07,579] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +13: [2023-05-08 11:26:07,579] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 1: [2023-05-08 11:26:07,579] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 12 +13: [2023-05-08 11:26:07,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +13: [2023-05-08 11:26:07,580] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 0: [2023-05-08 11:26:07,580] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 4 +14: [2023-05-08 11:26:07,580] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 112 + 7: [2023-05-08 11:26:07,580] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,581] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 4: [2023-05-08 11:26:07,581] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt. + 4: [2023-05-08 11:26:07,582] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 36 +11: [2023-05-08 11:26:07,582] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt. +11: [2023-05-08 11:26:07,582] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 90 + 9: [2023-05-08 11:26:07,582] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 72 +15: [2023-05-08 11:26:07,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt. +15: [2023-05-08 11:26:07,583] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 8: [2023-05-08 11:26:07,583] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 70 +28: [2023-05-08 11:26:07,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 4: [2023-05-08 11:26:07,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:26:07,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +24: [2023-05-08 11:26:07,583] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 4: [2023-05-08 11:26:07,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 4: [2023-05-08 11:26:07,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 4: [2023-05-08 11:26:07,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +28: [2023-05-08 11:26:07,584] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,584] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,585] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 4 + 5: [2023-05-08 11:26:07,585] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +28: [2023-05-08 11:26:07,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 0: [2023-05-08 11:26:07,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,585] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 4: [2023-05-08 11:26:07,586] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 36 +28: [2023-05-08 11:26:07,586] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +11: [2023-05-08 11:26:07,586] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 90 + 5: [2023-05-08 11:26:07,587] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 8: [2023-05-08 11:26:07,587] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 70 +19: [2023-05-08 11:26:07,587] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt. +19: [2023-05-08 11:26:07,587] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 152 +15: [2023-05-08 11:26:07,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:26:07,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +24: [2023-05-08 11:26:07,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +22: [2023-05-08 11:26:07,588] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt. +22: [2023-05-08 11:26:07,589] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 182 +20: [2023-05-08 11:26:07,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +20: [2023-05-08 11:26:07,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +20: [2023-05-08 11:26:07,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +15: [2023-05-08 11:26:07,589] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 4: [2023-05-08 11:26:07,589] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt. +12: [2023-05-08 11:26:07,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. + 4: [2023-05-08 11:26:07,590] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 34 +20: [2023-05-08 11:26:07,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +12: [2023-05-08 11:26:07,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +12: [2023-05-08 11:26:07,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +12: [2023-05-08 11:26:07,590] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +26: [2023-05-08 11:26:07,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt. +26: [2023-05-08 11:26:07,591] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 210 +13: [2023-05-08 11:26:07,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt. +13: [2023-05-08 11:26:07,591] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 110 + 5: [2023-05-08 11:26:07,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:26:07,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +17: [2023-05-08 11:26:07,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +17: [2023-05-08 11:26:07,591] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +19: [2023-05-08 11:26:07,592] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 152 +17: [2023-05-08 11:26:07,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +29: [2023-05-08 11:26:07,592] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt. +29: [2023-05-08 11:26:07,592] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 236 + 7: [2023-05-08 11:26:07,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +22: [2023-05-08 11:26:07,593] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 182 + 5: [2023-05-08 11:26:07,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt. +24: [2023-05-08 11:26:07,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 5: [2023-05-08 11:26:07,593] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 4: [2023-05-08 11:26:07,593] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,594] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 44 + 4: [2023-05-08 11:26:07,594] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 34 + 4: [2023-05-08 11:26:07,594] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:26:07,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:26:07,595] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 210 + 4: [2023-05-08 11:26:07,595] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +13: [2023-05-08 11:26:07,595] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 110 +30: [2023-05-08 11:26:07,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt. +30: [2023-05-08 11:26:07,595] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 242 +16: [2023-05-08 11:26:07,595] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt. +16: [2023-05-08 11:26:07,596] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 130 +26: [2023-05-08 11:26:07,596] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt. +26: [2023-05-08 11:26:07,596] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 214 + 4: [2023-05-08 11:26:07,596] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +29: [2023-05-08 11:26:07,596] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 236 +24: [2023-05-08 11:26:07,597] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... + 7: [2023-05-08 11:26:07,597] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,598] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 44 +16: [2023-05-08 11:26:07,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +16: [2023-05-08 11:26:07,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +20: [2023-05-08 11:26:07,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:26:07,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +16: [2023-05-08 11:26:07,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt. +20: [2023-05-08 11:26:07,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:26:07,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:26:07,598] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt. + 7: [2023-05-08 11:26:07,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +12: [2023-05-08 11:26:07,599] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 96 +30: [2023-05-08 11:26:07,599] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 242 +24: [2023-05-08 11:26:07,598] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +26: [2023-05-08 11:26:07,599] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 214 +20: [2023-05-08 11:26:07,600] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:26:07,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +20: [2023-05-08 11:26:07,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:26:07,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:26:07,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:26:07,601] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:26:07,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +20: [2023-05-08 11:26:07,602] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,601] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,602] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +16: [2023-05-08 11:26:07,602] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 130 +17: [2023-05-08 11:26:07,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:26:07,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:26:07,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +17: [2023-05-08 11:26:07,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +20: [2023-05-08 11:26:07,603] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt. +20: [2023-05-08 11:26:07,603] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 166 +12: [2023-05-08 11:26:07,603] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +12: [2023-05-08 11:26:07,604] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 96 + 9: [2023-05-08 11:26:07,604] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt. +20: [2023-05-08 11:26:07,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +20: [2023-05-08 11:26:07,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +20: [2023-05-08 11:26:07,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +20: [2023-05-08 11:26:07,604] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 9: [2023-05-08 11:26:07,604] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 76 +18: [2023-05-08 11:26:07,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +18: [2023-05-08 11:26:07,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 0: [2023-05-08 11:26:07,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,605] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +18: [2023-05-08 11:26:07,605] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +18: [2023-05-08 11:26:07,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 0: [2023-05-08 11:26:07,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 0: [2023-05-08 11:26:07,606] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,606] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +20: [2023-05-08 11:26:07,607] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 166 + 9: [2023-05-08 11:26:07,608] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 76 +24: [2023-05-08 11:26:07,608] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt. +24: [2023-05-08 11:26:07,609] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 192 +24: [2023-05-08 11:26:07,610] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:26:07,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +24: [2023-05-08 11:26:07,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +16: [2023-05-08 11:26:07,611] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:26:07,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +16: [2023-05-08 11:26:07,612] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt... +24: [2023-05-08 11:26:07,612] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 192 +12: [2023-05-08 11:26:07,613] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:26:07,615] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:26:07,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +18: [2023-05-08 11:26:07,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:26:07,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +18: [2023-05-08 11:26:07,615] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:26:07,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +18: [2023-05-08 11:26:07,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +18: [2023-05-08 11:26:07,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +18: [2023-05-08 11:26:07,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:26:07,616] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt. +18: [2023-05-08 11:26:07,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +26: [2023-05-08 11:26:07,616] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 212 +18: [2023-05-08 11:26:07,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +18: [2023-05-08 11:26:07,616] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 8: [2023-05-08 11:26:07,617] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt. + 8: [2023-05-08 11:26:07,617] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 68 +18: [2023-05-08 11:26:07,617] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +17: [2023-05-08 11:26:07,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:26:07,618] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +18: [2023-05-08 11:26:07,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +17: [2023-05-08 11:26:07,618] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +18: [2023-05-08 11:26:07,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +18: [2023-05-08 11:26:07,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +17: [2023-05-08 11:26:07,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:26:07,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +18: [2023-05-08 11:26:07,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +17: [2023-05-08 11:26:07,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +17: [2023-05-08 11:26:07,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +18: [2023-05-08 11:26:07,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +24: [2023-05-08 11:26:07,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:26:07,619] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +20: [2023-05-08 11:26:07,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +20: [2023-05-08 11:26:07,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +20: [2023-05-08 11:26:07,619] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +20: [2023-05-08 11:26:07,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +17: [2023-05-08 11:26:07,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +17: [2023-05-08 11:26:07,620] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +13: [2023-05-08 11:26:07,620] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt. + 8: [2023-05-08 11:26:07,621] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 68 +13: [2023-05-08 11:26:07,621] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 106 +20: [2023-05-08 11:26:07,621] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt. +26: [2023-05-08 11:26:07,621] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 212 +20: [2023-05-08 11:26:07,621] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 164 +16: [2023-05-08 11:26:07,624] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:26:07,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt. +24: [2023-05-08 11:26:07,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +13: [2023-05-08 11:26:07,625] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 106 +12: [2023-05-08 11:26:07,625] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 100 +20: [2023-05-08 11:26:07,625] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 164 +16: [2023-05-08 11:26:07,625] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +24: [2023-05-08 11:26:07,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +12: [2023-05-08 11:26:07,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:26:07,626] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +12: [2023-05-08 11:26:07,626] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +12: [2023-05-08 11:26:07,627] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +12: [2023-05-08 11:26:07,627] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt. +12: [2023-05-08 11:26:07,628] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 98 +16: [2023-05-08 11:26:07,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +16: [2023-05-08 11:26:07,628] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +12: [2023-05-08 11:26:07,629] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 100 +16: [2023-05-08 11:26:07,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:26:07,630] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +16: [2023-05-08 11:26:07,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +16: [2023-05-08 11:26:07,630] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... + 0: [2023-05-08 11:26:07,631] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt. + 0: [2023-05-08 11:26:07,631] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 2 +12: [2023-05-08 11:26:07,632] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 98 +30: [2023-05-08 11:26:07,634] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt. +30: [2023-05-08 11:26:07,634] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 244 + 0: [2023-05-08 11:26:07,635] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 2 +12: [2023-05-08 11:26:07,637] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt. +30: [2023-05-08 11:26:07,638] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 244 +12: [2023-05-08 11:26:07,638] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt... +27: [2023-05-08 11:26:07,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +27: [2023-05-08 11:26:07,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +27: [2023-05-08 11:26:07,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +27: [2023-05-08 11:26:07,639] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +28: [2023-05-08 11:26:07,644] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt. +28: [2023-05-08 11:26:07,644] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 228 +19: [2023-05-08 11:26:07,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +19: [2023-05-08 11:26:07,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +19: [2023-05-08 11:26:07,645] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 8: [2023-05-08 11:26:07,646] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt. + 8: [2023-05-08 11:26:07,646] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 66 +28: [2023-05-08 11:26:07,648] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 228 +27: [2023-05-08 11:26:07,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +27: [2023-05-08 11:26:07,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +27: [2023-05-08 11:26:07,649] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:26:07,649] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +27: [2023-05-08 11:26:07,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:26:07,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +27: [2023-05-08 11:26:07,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +27: [2023-05-08 11:26:07,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +14: [2023-05-08 11:26:07,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt. +27: [2023-05-08 11:26:07,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:26:07,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +14: [2023-05-08 11:26:07,650] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 118 +27: [2023-05-08 11:26:07,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +27: [2023-05-08 11:26:07,650] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +19: [2023-05-08 11:26:07,650] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 8: [2023-05-08 11:26:07,650] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 66 +16: [2023-05-08 11:26:07,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt. +16: [2023-05-08 11:26:07,652] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 134 +27: [2023-05-08 11:26:07,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +27: [2023-05-08 11:26:07,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +27: [2023-05-08 11:26:07,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +27: [2023-05-08 11:26:07,652] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +14: [2023-05-08 11:26:07,654] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 118 +19: [2023-05-08 11:26:07,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:26:07,656] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 134 +19: [2023-05-08 11:26:07,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +19: [2023-05-08 11:26:07,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:26:07,656] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +19: [2023-05-08 11:26:07,656] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:26:07,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +14: [2023-05-08 11:26:07,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt. +12: [2023-05-08 11:26:07,657] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt. +14: [2023-05-08 11:26:07,657] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 116 +12: [2023-05-08 11:26:07,657] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 102 +18: [2023-05-08 11:26:07,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt... +18: [2023-05-08 11:26:07,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt... +18: [2023-05-08 11:26:07,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt... +18: [2023-05-08 11:26:07,658] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt... +15: [2023-05-08 11:26:07,658] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt. +19: [2023-05-08 11:26:07,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +15: [2023-05-08 11:26:07,659] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 126 +20: [2023-05-08 11:26:07,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt. +20: [2023-05-08 11:26:07,659] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 160 +19: [2023-05-08 11:26:07,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +19: [2023-05-08 11:26:07,659] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +30: [2023-05-08 11:26:07,659] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt. +30: [2023-05-08 11:26:07,660] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 240 +19: [2023-05-08 11:26:07,660] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +20: [2023-05-08 11:26:07,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt... +20: [2023-05-08 11:26:07,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt... +19: [2023-05-08 11:26:07,661] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +19: [2023-05-08 11:26:07,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +20: [2023-05-08 11:26:07,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt... +20: [2023-05-08 11:26:07,661] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt... +14: [2023-05-08 11:26:07,661] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 116 +19: [2023-05-08 11:26:07,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +19: [2023-05-08 11:26:07,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +19: [2023-05-08 11:26:07,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +19: [2023-05-08 11:26:07,662] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +12: [2023-05-08 11:26:07,662] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 102 +15: [2023-05-08 11:26:07,663] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 126 +20: [2023-05-08 11:26:07,663] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 160 + 4: [2023-05-08 11:26:07,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt. +21: [2023-05-08 11:26:07,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +30: [2023-05-08 11:26:07,663] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 240 +21: [2023-05-08 11:26:07,663] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +21: [2023-05-08 11:26:07,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 4: [2023-05-08 11:26:07,664] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 38 +21: [2023-05-08 11:26:07,664] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 7: [2023-05-08 11:26:07,666] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt. + 7: [2023-05-08 11:26:07,666] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 60 +13: [2023-05-08 11:26:07,667] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt. +13: [2023-05-08 11:26:07,667] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 104 + 4: [2023-05-08 11:26:07,668] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 38 + 9: [2023-05-08 11:26:07,668] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt. + 9: [2023-05-08 11:26:07,668] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 74 +26: [2023-05-08 11:26:07,670] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt. +26: [2023-05-08 11:26:07,670] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 208 +28: [2023-05-08 11:26:07,671] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt. + 7: [2023-05-08 11:26:07,671] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 60 +28: [2023-05-08 11:26:07,671] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 226 +13: [2023-05-08 11:26:07,671] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 104 + 9: [2023-05-08 11:26:07,673] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 74 +21: [2023-05-08 11:26:07,674] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +26: [2023-05-08 11:26:07,674] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 208 +21: [2023-05-08 11:26:07,674] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:26:07,675] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 226 +21: [2023-05-08 11:26:07,675] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +21: [2023-05-08 11:26:07,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +21: [2023-05-08 11:26:07,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +21: [2023-05-08 11:26:07,676] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +21: [2023-05-08 11:26:07,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:26:07,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:26:07,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +21: [2023-05-08 11:26:07,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +21: [2023-05-08 11:26:07,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +21: [2023-05-08 11:26:07,677] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +21: [2023-05-08 11:26:07,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +21: [2023-05-08 11:26:07,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +21: [2023-05-08 11:26:07,677] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +21: [2023-05-08 11:26:07,678] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +25: [2023-05-08 11:26:07,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +25: [2023-05-08 11:26:07,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +25: [2023-05-08 11:26:07,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +25: [2023-05-08 11:26:07,680] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +16: [2023-05-08 11:26:07,681] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt. +16: [2023-05-08 11:26:07,682] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 128 +10: [2023-05-08 11:26:07,684] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt. +10: [2023-05-08 11:26:07,684] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 84 +16: [2023-05-08 11:26:07,685] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 128 +10: [2023-05-08 11:26:07,689] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 84 +27: [2023-05-08 11:26:07,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt... +27: [2023-05-08 11:26:07,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt... +27: [2023-05-08 11:26:07,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt... +27: [2023-05-08 11:26:07,689] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt... +25: [2023-05-08 11:26:07,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:26:07,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:26:07,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:26:07,690] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:26:07,690] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:26:07,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +25: [2023-05-08 11:26:07,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +25: [2023-05-08 11:26:07,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +25: [2023-05-08 11:26:07,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:26:07,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +25: [2023-05-08 11:26:07,691] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +25: [2023-05-08 11:26:07,691] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 1: [2023-05-08 11:26:07,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 1: [2023-05-08 11:26:07,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 1: [2023-05-08 11:26:07,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 1: [2023-05-08 11:26:07,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +23: [2023-05-08 11:26:07,692] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +23: [2023-05-08 11:26:07,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +23: [2023-05-08 11:26:07,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +23: [2023-05-08 11:26:07,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +25: [2023-05-08 11:26:07,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +25: [2023-05-08 11:26:07,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +25: [2023-05-08 11:26:07,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +25: [2023-05-08 11:26:07,693] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +14: [2023-05-08 11:26:07,695] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt. +14: [2023-05-08 11:26:07,695] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 114 + 9: [2023-05-08 11:26:07,699] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt. +14: [2023-05-08 11:26:07,699] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 114 + 9: [2023-05-08 11:26:07,700] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 78 +19: [2023-05-08 11:26:07,700] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt. +19: [2023-05-08 11:26:07,700] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 156 + 1: [2023-05-08 11:26:07,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 1: [2023-05-08 11:26:07,701] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 1: [2023-05-08 11:26:07,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,701] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 1: [2023-05-08 11:26:07,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:26:07,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 1: [2023-05-08 11:26:07,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 1: [2023-05-08 11:26:07,702] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 1: [2023-05-08 11:26:07,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,702] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:26:07,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 1: [2023-05-08 11:26:07,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 1: [2023-05-08 11:26:07,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +23: [2023-05-08 11:26:07,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:26:07,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:26:07,703] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +23: [2023-05-08 11:26:07,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:26:07,703] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:26:07,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +23: [2023-05-08 11:26:07,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +23: [2023-05-08 11:26:07,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +23: [2023-05-08 11:26:07,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 9: [2023-05-08 11:26:07,704] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 78 +23: [2023-05-08 11:26:07,704] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +30: [2023-05-08 11:26:07,704] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt. +30: [2023-05-08 11:26:07,704] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 246 +19: [2023-05-08 11:26:07,705] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 156 + 1: [2023-05-08 11:26:07,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 1: [2023-05-08 11:26:07,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 1: [2023-05-08 11:26:07,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 1: [2023-05-08 11:26:07,705] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +23: [2023-05-08 11:26:07,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +23: [2023-05-08 11:26:07,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +23: [2023-05-08 11:26:07,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +23: [2023-05-08 11:26:07,706] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +19: [2023-05-08 11:26:07,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt... +19: [2023-05-08 11:26:07,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt... +19: [2023-05-08 11:26:07,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt... +19: [2023-05-08 11:26:07,708] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt... +30: [2023-05-08 11:26:07,708] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 246 +28: [2023-05-08 11:26:07,711] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt. +28: [2023-05-08 11:26:07,711] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 224 +21: [2023-05-08 11:26:07,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt... +21: [2023-05-08 11:26:07,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt... +21: [2023-05-08 11:26:07,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt... +21: [2023-05-08 11:26:07,714] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt... +28: [2023-05-08 11:26:07,715] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 224 + 6: [2023-05-08 11:26:07,718] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt. + 6: [2023-05-08 11:26:07,719] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 52 +31: [2023-05-08 11:26:07,719] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt. +31: [2023-05-08 11:26:07,719] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 248 +15: [2023-05-08 11:26:07,720] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt. +15: [2023-05-08 11:26:07,720] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 122 + 6: [2023-05-08 11:26:07,722] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 52 +31: [2023-05-08 11:26:07,723] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 248 +15: [2023-05-08 11:26:07,724] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 122 + 6: [2023-05-08 11:26:07,724] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt. + 6: [2023-05-08 11:26:07,725] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 54 + 0: [2023-05-08 11:26:07,727] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt. + 0: [2023-05-08 11:26:07,728] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 0 +31: [2023-05-08 11:26:07,728] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt. +31: [2023-05-08 11:26:07,728] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 250 + 6: [2023-05-08 11:26:07,729] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 54 +25: [2023-05-08 11:26:07,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt... +25: [2023-05-08 11:26:07,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt... +25: [2023-05-08 11:26:07,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt... +25: [2023-05-08 11:26:07,731] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt... +31: [2023-05-08 11:26:07,731] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 250 + 0: [2023-05-08 11:26:07,732] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 0 + 6: [2023-05-08 11:26:07,732] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt. + 6: [2023-05-08 11:26:07,732] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 48 + 0: could not find arguments in the checkpoint ... + 0: checkpoint version 3.0 + 6: [2023-05-08 11:26:07,736] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 48 +30: [2023-05-08 11:26:07,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +30: [2023-05-08 11:26:07,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +30: [2023-05-08 11:26:07,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +30: [2023-05-08 11:26:07,737] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +13: [2023-05-08 11:26:07,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt. +19: [2023-05-08 11:26:07,739] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt. +19: [2023-05-08 11:26:07,739] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 158 +13: [2023-05-08 11:26:07,739] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 108 + 1: [2023-05-08 11:26:07,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt... + 1: [2023-05-08 11:26:07,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt... + 1: [2023-05-08 11:26:07,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt... + 1: [2023-05-08 11:26:07,740] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt... +23: [2023-05-08 11:26:07,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt... +23: [2023-05-08 11:26:07,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt... +23: [2023-05-08 11:26:07,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt... +23: [2023-05-08 11:26:07,741] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt... +19: [2023-05-08 11:26:07,743] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 158 +13: [2023-05-08 11:26:07,744] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 108 +30: [2023-05-08 11:26:07,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:26:07,747] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:26:07,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:26:07,748] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:26:07,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:26:07,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +30: [2023-05-08 11:26:07,748] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +30: [2023-05-08 11:26:07,749] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:26:07,749] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +30: [2023-05-08 11:26:07,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +30: [2023-05-08 11:26:07,750] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +30: [2023-05-08 11:26:07,750] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +30: [2023-05-08 11:26:07,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +30: [2023-05-08 11:26:07,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +30: [2023-05-08 11:26:07,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +30: [2023-05-08 11:26:07,751] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +10: [2023-05-08 11:26:07,760] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt. +10: [2023-05-08 11:26:07,760] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 80 +10: [2023-05-08 11:26:07,765] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 80 +15: [2023-05-08 11:26:07,768] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt. +15: [2023-05-08 11:26:07,768] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 124 +22: [2023-05-08 11:26:07,770] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt. +22: [2023-05-08 11:26:07,770] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 178 +15: [2023-05-08 11:26:07,772] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 124 +22: [2023-05-08 11:26:07,774] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 178 + 8: [2023-05-08 11:26:07,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 8: [2023-05-08 11:26:07,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 8: [2023-05-08 11:26:07,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +18: [2023-05-08 11:26:07,777] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt. +18: [2023-05-08 11:26:07,778] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 148 + 8: [2023-05-08 11:26:07,778] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +18: [2023-05-08 11:26:07,782] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 148 +31: [2023-05-08 11:26:07,785] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt. +31: [2023-05-08 11:26:07,785] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 254 +30: [2023-05-08 11:26:07,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt... +30: [2023-05-08 11:26:07,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt... +30: [2023-05-08 11:26:07,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt... +30: [2023-05-08 11:26:07,785] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt... + 8: [2023-05-08 11:26:07,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 8: [2023-05-08 11:26:07,787] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 8: [2023-05-08 11:26:07,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,788] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,788] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 8: [2023-05-08 11:26:07,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 8: [2023-05-08 11:26:07,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 8: [2023-05-08 11:26:07,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 8: [2023-05-08 11:26:07,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +31: [2023-05-08 11:26:07,789] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 254 + 8: [2023-05-08 11:26:07,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 8: [2023-05-08 11:26:07,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 8: [2023-05-08 11:26:07,789] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +26: [2023-05-08 11:26:07,789] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +26: [2023-05-08 11:26:07,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +26: [2023-05-08 11:26:07,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +26: [2023-05-08 11:26:07,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 6: [2023-05-08 11:26:07,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 6: [2023-05-08 11:26:07,790] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 6: [2023-05-08 11:26:07,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 8: [2023-05-08 11:26:07,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 8: [2023-05-08 11:26:07,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 8: [2023-05-08 11:26:07,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 6: [2023-05-08 11:26:07,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 8: [2023-05-08 11:26:07,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 9: [2023-05-08 11:26:07,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 9: [2023-05-08 11:26:07,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 9: [2023-05-08 11:26:07,791] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 9: [2023-05-08 11:26:07,792] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +29: [2023-05-08 11:26:07,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +29: [2023-05-08 11:26:07,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +29: [2023-05-08 11:26:07,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +29: [2023-05-08 11:26:07,794] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +26: [2023-05-08 11:26:07,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +26: [2023-05-08 11:26:07,799] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +26: [2023-05-08 11:26:07,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:26:07,799] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +11: [2023-05-08 11:26:07,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +11: [2023-05-08 11:26:07,800] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 6: [2023-05-08 11:26:07,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +26: [2023-05-08 11:26:07,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +26: [2023-05-08 11:26:07,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +26: [2023-05-08 11:26:07,800] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +26: [2023-05-08 11:26:07,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:26:07,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +26: [2023-05-08 11:26:07,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +26: [2023-05-08 11:26:07,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +26: [2023-05-08 11:26:07,801] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 9: [2023-05-08 11:26:07,801] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 6: [2023-05-08 11:26:07,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 9: [2023-05-08 11:26:07,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 6: [2023-05-08 11:26:07,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 6: [2023-05-08 11:26:07,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +11: [2023-05-08 11:26:07,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 6: [2023-05-08 11:26:07,802] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 4: [2023-05-08 11:26:07,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 4: [2023-05-08 11:26:07,802] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 6: [2023-05-08 11:26:07,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:26:07,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +26: [2023-05-08 11:26:07,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +26: [2023-05-08 11:26:07,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +26: [2023-05-08 11:26:07,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 6: [2023-05-08 11:26:07,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +26: [2023-05-08 11:26:07,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 9: [2023-05-08 11:26:07,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 9: [2023-05-08 11:26:07,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 9: [2023-05-08 11:26:07,803] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +29: [2023-05-08 11:26:07,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:26:07,803] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 9: [2023-05-08 11:26:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:26:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 9: [2023-05-08 11:26:07,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +10: [2023-05-08 11:26:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +29: [2023-05-08 11:26:07,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 6: [2023-05-08 11:26:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 6: [2023-05-08 11:26:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 6: [2023-05-08 11:26:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 9: [2023-05-08 11:26:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +29: [2023-05-08 11:26:07,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 9: [2023-05-08 11:26:07,804] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 6: [2023-05-08 11:26:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +29: [2023-05-08 11:26:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:26:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +29: [2023-05-08 11:26:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 9: [2023-05-08 11:26:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 9: [2023-05-08 11:26:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 9: [2023-05-08 11:26:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +29: [2023-05-08 11:26:07,804] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:26:07,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +29: [2023-05-08 11:26:07,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 9: [2023-05-08 11:26:07,805] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +29: [2023-05-08 11:26:07,805] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 4: [2023-05-08 11:26:07,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +15: [2023-05-08 11:26:07,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt. +29: [2023-05-08 11:26:07,806] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 4: [2023-05-08 11:26:07,806] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +15: [2023-05-08 11:26:07,806] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 120 +29: [2023-05-08 11:26:07,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +29: [2023-05-08 11:26:07,807] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +17: [2023-05-08 11:26:07,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +17: [2023-05-08 11:26:07,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +17: [2023-05-08 11:26:07,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +29: [2023-05-08 11:26:07,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +29: [2023-05-08 11:26:07,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +29: [2023-05-08 11:26:07,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +29: [2023-05-08 11:26:07,807] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +17: [2023-05-08 11:26:07,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +13: [2023-05-08 11:26:07,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +13: [2023-05-08 11:26:07,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +13: [2023-05-08 11:26:07,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +13: [2023-05-08 11:26:07,808] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +11: [2023-05-08 11:26:07,809] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +11: [2023-05-08 11:26:07,810] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +15: [2023-05-08 11:26:07,810] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 120 +11: [2023-05-08 11:26:07,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +11: [2023-05-08 11:26:07,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +11: [2023-05-08 11:26:07,811] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:26:07,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 4: [2023-05-08 11:26:07,811] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 4: [2023-05-08 11:26:07,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +11: [2023-05-08 11:26:07,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 4: [2023-05-08 11:26:07,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:26:07,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:26:07,812] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:26:07,812] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 4: [2023-05-08 11:26:07,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 4: [2023-05-08 11:26:07,813] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +11: [2023-05-08 11:26:07,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +11: [2023-05-08 11:26:07,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +11: [2023-05-08 11:26:07,813] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +11: [2023-05-08 11:26:07,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:26:07,814] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +11: [2023-05-08 11:26:07,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +11: [2023-05-08 11:26:07,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +10: [2023-05-08 11:26:07,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:26:07,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 4: [2023-05-08 11:26:07,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:26:07,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +11: [2023-05-08 11:26:07,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +10: [2023-05-08 11:26:07,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:26:07,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:26:07,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +10: [2023-05-08 11:26:07,815] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 4: [2023-05-08 11:26:07,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +10: [2023-05-08 11:26:07,815] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:26:07,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 4: [2023-05-08 11:26:07,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +10: [2023-05-08 11:26:07,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +10: [2023-05-08 11:26:07,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +17: [2023-05-08 11:26:07,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:26:07,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 4: [2023-05-08 11:26:07,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 4: [2023-05-08 11:26:07,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 4: [2023-05-08 11:26:07,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +17: [2023-05-08 11:26:07,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +17: [2023-05-08 11:26:07,816] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 4: [2023-05-08 11:26:07,816] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:26:07,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +17: [2023-05-08 11:26:07,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +17: [2023-05-08 11:26:07,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 4: [2023-05-08 11:26:07,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +17: [2023-05-08 11:26:07,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +17: [2023-05-08 11:26:07,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +17: [2023-05-08 11:26:07,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +17: [2023-05-08 11:26:07,817] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +10: [2023-05-08 11:26:07,817] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +10: [2023-05-08 11:26:07,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +10: [2023-05-08 11:26:07,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +17: [2023-05-08 11:26:07,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +10: [2023-05-08 11:26:07,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +13: [2023-05-08 11:26:07,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +17: [2023-05-08 11:26:07,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +17: [2023-05-08 11:26:07,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +22: [2023-05-08 11:26:07,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +22: [2023-05-08 11:26:07,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +13: [2023-05-08 11:26:07,818] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +13: [2023-05-08 11:26:07,818] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:26:07,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 2: [2023-05-08 11:26:07,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 2: [2023-05-08 11:26:07,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +13: [2023-05-08 11:26:07,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +13: [2023-05-08 11:26:07,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +22: [2023-05-08 11:26:07,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +13: [2023-05-08 11:26:07,819] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +17: [2023-05-08 11:26:07,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +17: [2023-05-08 11:26:07,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +17: [2023-05-08 11:26:07,819] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +17: [2023-05-08 11:26:07,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 3: [2023-05-08 11:26:07,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 3: [2023-05-08 11:26:07,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 3: [2023-05-08 11:26:07,820] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +13: [2023-05-08 11:26:07,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:26:07,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +13: [2023-05-08 11:26:07,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +13: [2023-05-08 11:26:07,821] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +13: [2023-05-08 11:26:07,821] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +13: [2023-05-08 11:26:07,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +13: [2023-05-08 11:26:07,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 2: [2023-05-08 11:26:07,822] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +13: [2023-05-08 11:26:07,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +13: [2023-05-08 11:26:07,823] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +13: [2023-05-08 11:26:07,823] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +13: [2023-05-08 11:26:07,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 3: [2023-05-08 11:26:07,824] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 8: [2023-05-08 11:26:07,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt... + 8: [2023-05-08 11:26:07,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt... + 8: [2023-05-08 11:26:07,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt... + 8: [2023-05-08 11:26:07,827] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt... +22: [2023-05-08 11:26:07,828] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:26:07,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:26:07,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:26:07,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:26:07,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 2: [2023-05-08 11:26:07,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:26:07,829] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 2: [2023-05-08 11:26:07,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:26:07,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +22: [2023-05-08 11:26:07,829] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 2: [2023-05-08 11:26:07,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 2: [2023-05-08 11:26:07,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:26:07,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:26:07,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 2: [2023-05-08 11:26:07,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 3: [2023-05-08 11:26:07,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:26:07,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 3: [2023-05-08 11:26:07,830] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +22: [2023-05-08 11:26:07,830] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:26:07,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +22: [2023-05-08 11:26:07,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 3: [2023-05-08 11:26:07,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +22: [2023-05-08 11:26:07,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:26:07,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +28: [2023-05-08 11:26:07,831] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +22: [2023-05-08 11:26:07,831] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +22: [2023-05-08 11:26:07,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +22: [2023-05-08 11:26:07,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +22: [2023-05-08 11:26:07,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +22: [2023-05-08 11:26:07,832] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 3: [2023-05-08 11:26:07,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 3: [2023-05-08 11:26:07,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 3: [2023-05-08 11:26:07,832] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:26:07,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 2: [2023-05-08 11:26:07,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 5: [2023-05-08 11:26:07,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 5: [2023-05-08 11:26:07,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 3: [2023-05-08 11:26:07,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +28: [2023-05-08 11:26:07,833] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 2: [2023-05-08 11:26:07,833] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:26:07,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +28: [2023-05-08 11:26:07,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 2: [2023-05-08 11:26:07,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 2: [2023-05-08 11:26:07,834] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 2: [2023-05-08 11:26:07,834] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 3: [2023-05-08 11:26:07,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 3: [2023-05-08 11:26:07,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 3: [2023-05-08 11:26:07,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 3: [2023-05-08 11:26:07,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 3: [2023-05-08 11:26:07,835] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 3: [2023-05-08 11:26:07,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 5: [2023-05-08 11:26:07,835] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 3: [2023-05-08 11:26:07,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 5: [2023-05-08 11:26:07,836] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +26: [2023-05-08 11:26:07,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt... +26: [2023-05-08 11:26:07,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt... +26: [2023-05-08 11:26:07,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt... +26: [2023-05-08 11:26:07,838] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt... + 6: [2023-05-08 11:26:07,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt... + 6: [2023-05-08 11:26:07,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt... + 6: [2023-05-08 11:26:07,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt... + 6: [2023-05-08 11:26:07,840] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt... +28: [2023-05-08 11:26:07,841] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:26:07,842] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:26:07,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +28: [2023-05-08 11:26:07,842] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:26:07,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:26:07,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:26:07,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +28: [2023-05-08 11:26:07,843] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:26:07,843] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +29: [2023-05-08 11:26:07,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt... +29: [2023-05-08 11:26:07,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt... +29: [2023-05-08 11:26:07,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt... +29: [2023-05-08 11:26:07,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt... +28: [2023-05-08 11:26:07,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:26:07,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:26:07,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 5: [2023-05-08 11:26:07,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 5: [2023-05-08 11:26:07,844] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +28: [2023-05-08 11:26:07,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +28: [2023-05-08 11:26:07,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +28: [2023-05-08 11:26:07,844] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 5: [2023-05-08 11:26:07,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +28: [2023-05-08 11:26:07,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 5: [2023-05-08 11:26:07,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 9: [2023-05-08 11:26:07,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt... + 9: [2023-05-08 11:26:07,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt... + 5: [2023-05-08 11:26:07,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 9: [2023-05-08 11:26:07,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt... + 9: [2023-05-08 11:26:07,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt... + 5: [2023-05-08 11:26:07,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 5: [2023-05-08 11:26:07,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 5: [2023-05-08 11:26:07,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,845] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 5: [2023-05-08 11:26:07,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 5: [2023-05-08 11:26:07,845] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 6: [2023-05-08 11:26:07,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt. + 6: [2023-05-08 11:26:07,847] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 50 + 5: [2023-05-08 11:26:07,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 5: [2023-05-08 11:26:07,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 5: [2023-05-08 11:26:07,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 5: [2023-05-08 11:26:07,847] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 4: [2023-05-08 11:26:07,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt... + 4: [2023-05-08 11:26:07,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt... + 4: [2023-05-08 11:26:07,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt... + 4: [2023-05-08 11:26:07,850] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt... + 6: [2023-05-08 11:26:07,851] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 50 +11: [2023-05-08 11:26:07,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt... +11: [2023-05-08 11:26:07,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt... +11: [2023-05-08 11:26:07,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt... +11: [2023-05-08 11:26:07,852] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt... +14: [2023-05-08 11:26:07,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +14: [2023-05-08 11:26:07,852] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +14: [2023-05-08 11:26:07,853] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +10: [2023-05-08 11:26:07,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt... +14: [2023-05-08 11:26:07,855] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +10: [2023-05-08 11:26:07,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt... +10: [2023-05-08 11:26:07,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt... +10: [2023-05-08 11:26:07,855] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt... +17: [2023-05-08 11:26:07,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt... +17: [2023-05-08 11:26:07,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt... +17: [2023-05-08 11:26:07,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt... +17: [2023-05-08 11:26:07,857] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt... +13: [2023-05-08 11:26:07,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt... +13: [2023-05-08 11:26:07,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt... +13: [2023-05-08 11:26:07,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt... +13: [2023-05-08 11:26:07,859] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt... +31: [2023-05-08 11:26:07,859] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +31: [2023-05-08 11:26:07,861] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +14: [2023-05-08 11:26:07,861] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +14: [2023-05-08 11:26:07,862] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +31: [2023-05-08 11:26:07,863] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +14: [2023-05-08 11:26:07,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +14: [2023-05-08 11:26:07,863] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:26:07,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +14: [2023-05-08 11:26:07,864] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +14: [2023-05-08 11:26:07,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +14: [2023-05-08 11:26:07,864] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +14: [2023-05-08 11:26:07,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +14: [2023-05-08 11:26:07,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +14: [2023-05-08 11:26:07,865] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:26:07,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +12: [2023-05-08 11:26:07,865] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +14: [2023-05-08 11:26:07,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +14: [2023-05-08 11:26:07,866] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +14: [2023-05-08 11:26:07,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +14: [2023-05-08 11:26:07,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +14: [2023-05-08 11:26:07,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +14: [2023-05-08 11:26:07,866] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +12: [2023-05-08 11:26:07,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +12: [2023-05-08 11:26:07,867] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +22: [2023-05-08 11:26:07,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt... +22: [2023-05-08 11:26:07,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt... +22: [2023-05-08 11:26:07,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt... +22: [2023-05-08 11:26:07,867] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt... + 2: [2023-05-08 11:26:07,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt... + 2: [2023-05-08 11:26:07,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt... +31: [2023-05-08 11:26:07,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 2: [2023-05-08 11:26:07,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt... + 2: [2023-05-08 11:26:07,871] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt... +31: [2023-05-08 11:26:07,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 7: [2023-05-08 11:26:07,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 7: [2023-05-08 11:26:07,872] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +31: [2023-05-08 11:26:07,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 7: [2023-05-08 11:26:07,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +31: [2023-05-08 11:26:07,873] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +31: [2023-05-08 11:26:07,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +31: [2023-05-08 11:26:07,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +31: [2023-05-08 11:26:07,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:26:07,873] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +31: [2023-05-08 11:26:07,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +31: [2023-05-08 11:26:07,874] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +31: [2023-05-08 11:26:07,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +31: [2023-05-08 11:26:07,874] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 3: [2023-05-08 11:26:07,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt... + 3: [2023-05-08 11:26:07,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt... + 3: [2023-05-08 11:26:07,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt... + 3: [2023-05-08 11:26:07,875] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt... +31: [2023-05-08 11:26:07,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +31: [2023-05-08 11:26:07,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +31: [2023-05-08 11:26:07,875] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +31: [2023-05-08 11:26:07,876] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +12: [2023-05-08 11:26:07,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:26:07,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:26:07,878] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:26:07,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:26:07,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:26:07,878] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:26:07,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +12: [2023-05-08 11:26:07,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +12: [2023-05-08 11:26:07,879] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +12: [2023-05-08 11:26:07,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +12: [2023-05-08 11:26:07,880] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +12: [2023-05-08 11:26:07,880] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +12: [2023-05-08 11:26:07,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +12: [2023-05-08 11:26:07,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +12: [2023-05-08 11:26:07,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +12: [2023-05-08 11:26:07,881] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +28: [2023-05-08 11:26:07,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt... +28: [2023-05-08 11:26:07,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt... +28: [2023-05-08 11:26:07,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt... +28: [2023-05-08 11:26:07,881] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt... + 7: [2023-05-08 11:26:07,882] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 7: [2023-05-08 11:26:07,883] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,883] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 5: [2023-05-08 11:26:07,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt... + 5: [2023-05-08 11:26:07,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt... + 5: [2023-05-08 11:26:07,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt... + 5: [2023-05-08 11:26:07,884] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt... + 7: [2023-05-08 11:26:07,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 7: [2023-05-08 11:26:07,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 7: [2023-05-08 11:26:07,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 7: [2023-05-08 11:26:07,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 7: [2023-05-08 11:26:07,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,885] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 7: [2023-05-08 11:26:07,885] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 7: [2023-05-08 11:26:07,886] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 7: [2023-05-08 11:26:07,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 7: [2023-05-08 11:26:07,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 7: [2023-05-08 11:26:07,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 7: [2023-05-08 11:26:07,886] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +15: [2023-05-08 11:26:07,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +15: [2023-05-08 11:26:07,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +15: [2023-05-08 11:26:07,887] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +15: [2023-05-08 11:26:07,888] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +24: [2023-05-08 11:26:07,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +24: [2023-05-08 11:26:07,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +24: [2023-05-08 11:26:07,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +24: [2023-05-08 11:26:07,892] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +15: [2023-05-08 11:26:07,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:26:07,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:26:07,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +15: [2023-05-08 11:26:07,897] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +15: [2023-05-08 11:26:07,897] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:26:07,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +15: [2023-05-08 11:26:07,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +15: [2023-05-08 11:26:07,898] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +15: [2023-05-08 11:26:07,898] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 0: [2023-05-08 11:26:07,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +15: [2023-05-08 11:26:07,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +15: [2023-05-08 11:26:07,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +15: [2023-05-08 11:26:07,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 0: [2023-05-08 11:26:07,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 0: [2023-05-08 11:26:07,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 0: [2023-05-08 11:26:07,900] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +15: [2023-05-08 11:26:07,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +15: [2023-05-08 11:26:07,901] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +15: [2023-05-08 11:26:07,901] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +15: [2023-05-08 11:26:07,902] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +14: [2023-05-08 11:26:07,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt... +14: [2023-05-08 11:26:07,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt... +14: [2023-05-08 11:26:07,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt... +14: [2023-05-08 11:26:07,902] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt... +24: [2023-05-08 11:26:07,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +24: [2023-05-08 11:26:07,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +24: [2023-05-08 11:26:07,904] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +24: [2023-05-08 11:26:07,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +24: [2023-05-08 11:26:07,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +24: [2023-05-08 11:26:07,904] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +24: [2023-05-08 11:26:07,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +24: [2023-05-08 11:26:07,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +24: [2023-05-08 11:26:07,905] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +24: [2023-05-08 11:26:07,906] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +24: [2023-05-08 11:26:07,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +24: [2023-05-08 11:26:07,907] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +24: [2023-05-08 11:26:07,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +24: [2023-05-08 11:26:07,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +24: [2023-05-08 11:26:07,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +24: [2023-05-08 11:26:07,907] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 0: [2023-05-08 11:26:07,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,910] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 0: [2023-05-08 11:26:07,911] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,911] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +31: [2023-05-08 11:26:07,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt... +31: [2023-05-08 11:26:07,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt... +31: [2023-05-08 11:26:07,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt... +31: [2023-05-08 11:26:07,912] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt... + 0: [2023-05-08 11:26:07,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... + 0: [2023-05-08 11:26:07,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 0: [2023-05-08 11:26:07,913] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. + 0: [2023-05-08 11:26:07,913] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... + 0: [2023-05-08 11:26:07,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 0: [2023-05-08 11:26:07,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 0: [2023-05-08 11:26:07,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. + 0: [2023-05-08 11:26:07,914] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +12: [2023-05-08 11:26:07,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt... +12: [2023-05-08 11:26:07,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt... +12: [2023-05-08 11:26:07,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt... +12: [2023-05-08 11:26:07,917] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt... +16: [2023-05-08 11:26:07,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +16: [2023-05-08 11:26:07,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +16: [2023-05-08 11:26:07,922] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. +16: [2023-05-08 11:26:07,923] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt. + 7: [2023-05-08 11:26:07,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt... + 7: [2023-05-08 11:26:07,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt... + 7: [2023-05-08 11:26:07,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt... + 7: [2023-05-08 11:26:07,925] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt... +16: [2023-05-08 11:26:07,932] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:26:07,932] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +16: [2023-05-08 11:26:07,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:26:07,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +16: [2023-05-08 11:26:07,933] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +16: [2023-05-08 11:26:07,933] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +15: [2023-05-08 11:26:07,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt... +15: [2023-05-08 11:26:07,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt... +15: [2023-05-08 11:26:07,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt... +15: [2023-05-08 11:26:07,934] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt... +16: [2023-05-08 11:26:07,935] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:26:07,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +16: [2023-05-08 11:26:07,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +16: [2023-05-08 11:26:07,936] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt... +16: [2023-05-08 11:26:07,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +16: [2023-05-08 11:26:07,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +16: [2023-05-08 11:26:07,936] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +16: [2023-05-08 11:26:07,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt. +16: [2023-05-08 11:26:07,937] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt... +16: [2023-05-08 11:26:07,937] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt. +24: [2023-05-08 11:26:07,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt... +24: [2023-05-08 11:26:07,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt... +24: [2023-05-08 11:26:07,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt... +24: [2023-05-08 11:26:07,942] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt... + 0: [2023-05-08 11:26:07,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt... + 0: [2023-05-08 11:26:07,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt... + 0: [2023-05-08 11:26:07,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt... + 0: [2023-05-08 11:26:07,946] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt... +16: [2023-05-08 11:26:07,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt... +16: [2023-05-08 11:26:07,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt... +16: [2023-05-08 11:26:07,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt... +16: [2023-05-08 11:26:07,974] [INFO] [torch_checkpoint_engine.py:21:load] [Torch] Loading checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt... +18: [2023-05-08 11:26:07,986] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt. +18: [2023-05-08 11:26:07,986] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 147 +18: [2023-05-08 11:26:07,990] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 147 +19: [2023-05-08 11:26:08,016] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt. +19: [2023-05-08 11:26:08,016] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 154 +19: [2023-05-08 11:26:08,020] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 154 +27: [2023-05-08 11:26:08,021] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt. +27: [2023-05-08 11:26:08,021] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 219 +18: [2023-05-08 11:26:08,023] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt. +18: [2023-05-08 11:26:08,023] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 151 +20: [2023-05-08 11:26:08,024] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt. +20: [2023-05-08 11:26:08,024] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 167 +27: [2023-05-08 11:26:08,025] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 219 +18: [2023-05-08 11:26:08,027] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 151 +20: [2023-05-08 11:26:08,028] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 167 +20: [2023-05-08 11:26:08,047] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt. +20: [2023-05-08 11:26:08,047] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 165 +20: [2023-05-08 11:26:08,051] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt. +20: [2023-05-08 11:26:08,051] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 161 +20: [2023-05-08 11:26:08,051] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 165 +20: [2023-05-08 11:26:08,057] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 161 +23: [2023-05-08 11:26:08,062] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt. +23: [2023-05-08 11:26:08,063] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 189 +20: [2023-05-08 11:26:08,063] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt. +20: [2023-05-08 11:26:08,064] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 163 +23: [2023-05-08 11:26:08,066] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 189 +20: [2023-05-08 11:26:08,068] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 163 +27: [2023-05-08 11:26:08,068] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt. +27: [2023-05-08 11:26:08,068] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 221 +27: [2023-05-08 11:26:08,072] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 221 +23: [2023-05-08 11:26:08,081] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt. +23: [2023-05-08 11:26:08,081] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 185 +18: [2023-05-08 11:26:08,082] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt. +18: [2023-05-08 11:26:08,082] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 145 +23: [2023-05-08 11:26:08,085] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 185 +18: [2023-05-08 11:26:08,087] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 145 +18: [2023-05-08 11:26:08,088] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt. +18: [2023-05-08 11:26:08,088] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 149 +18: [2023-05-08 11:26:08,092] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 149 +19: [2023-05-08 11:26:08,093] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt. +19: [2023-05-08 11:26:08,093] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 157 +25: [2023-05-08 11:26:08,097] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt. +25: [2023-05-08 11:26:08,097] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 207 +19: [2023-05-08 11:26:08,097] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 157 +25: [2023-05-08 11:26:08,099] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt. +25: [2023-05-08 11:26:08,099] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 203 +21: [2023-05-08 11:26:08,100] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt. +21: [2023-05-08 11:26:08,100] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 171 +25: [2023-05-08 11:26:08,101] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 207 +25: [2023-05-08 11:26:08,104] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 203 +21: [2023-05-08 11:26:08,105] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 171 + 1: [2023-05-08 11:26:08,108] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt. + 1: [2023-05-08 11:26:08,108] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 15 +21: [2023-05-08 11:26:08,112] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt. +21: [2023-05-08 11:26:08,112] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 169 + 1: [2023-05-08 11:26:08,112] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 15 +27: [2023-05-08 11:26:08,114] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt. +27: [2023-05-08 11:26:08,114] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 217 +21: [2023-05-08 11:26:08,117] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 169 +21: [2023-05-08 11:26:08,118] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt. +21: [2023-05-08 11:26:08,118] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 173 +27: [2023-05-08 11:26:08,118] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 217 +21: [2023-05-08 11:26:08,122] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 173 +19: [2023-05-08 11:26:08,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt. +19: [2023-05-08 11:26:08,124] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 153 +27: [2023-05-08 11:26:08,124] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt. +27: [2023-05-08 11:26:08,125] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 223 +23: [2023-05-08 11:26:08,128] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt. +27: [2023-05-08 11:26:08,128] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 223 +23: [2023-05-08 11:26:08,129] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 191 +19: [2023-05-08 11:26:08,129] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 153 +23: [2023-05-08 11:26:08,132] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 191 +25: [2023-05-08 11:26:08,136] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt. +25: [2023-05-08 11:26:08,136] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 201 + 8: [2023-05-08 11:26:08,137] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt. + 8: [2023-05-08 11:26:08,137] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 65 +25: [2023-05-08 11:26:08,140] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 201 + 8: [2023-05-08 11:26:08,141] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 65 +21: [2023-05-08 11:26:08,147] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt. +21: [2023-05-08 11:26:08,148] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 175 +30: [2023-05-08 11:26:08,149] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt. +30: [2023-05-08 11:26:08,149] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 241 +21: [2023-05-08 11:26:08,152] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 175 +30: [2023-05-08 11:26:08,153] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 241 +19: [2023-05-08 11:26:08,154] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt. +19: [2023-05-08 11:26:08,155] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 159 +19: [2023-05-08 11:26:08,159] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 159 +26: [2023-05-08 11:26:08,159] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt. +26: [2023-05-08 11:26:08,159] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 215 +26: [2023-05-08 11:26:08,163] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 215 + 6: [2023-05-08 11:26:08,166] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt. + 6: [2023-05-08 11:26:08,167] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 49 +25: [2023-05-08 11:26:08,169] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt. +25: [2023-05-08 11:26:08,169] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 205 + 6: [2023-05-08 11:26:08,170] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 49 + 1: [2023-05-08 11:26:08,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt. + 1: [2023-05-08 11:26:08,172] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt. + 1: [2023-05-08 11:26:08,172] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 11 + 1: [2023-05-08 11:26:08,172] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 9 +25: [2023-05-08 11:26:08,173] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 205 + 8: [2023-05-08 11:26:08,175] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt. + 8: [2023-05-08 11:26:08,176] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 71 + 1: [2023-05-08 11:26:08,176] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 11 + 1: [2023-05-08 11:26:08,176] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 9 +26: [2023-05-08 11:26:08,179] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt. +26: [2023-05-08 11:26:08,179] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 211 + 8: [2023-05-08 11:26:08,179] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 71 +26: [2023-05-08 11:26:08,184] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 211 +30: [2023-05-08 11:26:08,186] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt. +30: [2023-05-08 11:26:08,186] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 245 +10: [2023-05-08 11:26:08,189] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt. +10: [2023-05-08 11:26:08,190] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 83 +30: [2023-05-08 11:26:08,190] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 245 +10: [2023-05-08 11:26:08,191] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt. +10: [2023-05-08 11:26:08,192] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 81 +10: [2023-05-08 11:26:08,193] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 83 + 8: [2023-05-08 11:26:08,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt. + 8: [2023-05-08 11:26:08,194] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 69 +23: [2023-05-08 11:26:08,194] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt. +23: [2023-05-08 11:26:08,194] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 187 + 9: [2023-05-08 11:26:08,196] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt. + 9: [2023-05-08 11:26:08,196] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 75 +10: [2023-05-08 11:26:08,197] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 81 + 4: [2023-05-08 11:26:08,197] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt. + 8: [2023-05-08 11:26:08,198] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 69 + 4: [2023-05-08 11:26:08,198] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 33 +23: [2023-05-08 11:26:08,198] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 187 +11: [2023-05-08 11:26:08,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt. +11: [2023-05-08 11:26:08,200] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 89 + 1: [2023-05-08 11:26:08,200] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt. + 1: [2023-05-08 11:26:08,201] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 13 + 9: [2023-05-08 11:26:08,201] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 75 +11: [2023-05-08 11:26:08,202] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt. +11: [2023-05-08 11:26:08,202] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 93 + 4: [2023-05-08 11:26:08,204] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 33 +11: [2023-05-08 11:26:08,204] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 89 + 1: [2023-05-08 11:26:08,205] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 13 +22: [2023-05-08 11:26:08,206] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt. +22: [2023-05-08 11:26:08,207] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 183 +29: [2023-05-08 11:26:08,207] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt. +29: [2023-05-08 11:26:08,207] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 237 +11: [2023-05-08 11:26:08,208] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 93 +30: [2023-05-08 11:26:08,210] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt. +22: [2023-05-08 11:26:08,210] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 183 +30: [2023-05-08 11:26:08,210] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 243 +26: [2023-05-08 11:26:08,211] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt. +26: [2023-05-08 11:26:08,211] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 213 +29: [2023-05-08 11:26:08,211] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 237 +30: [2023-05-08 11:26:08,215] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 243 +26: [2023-05-08 11:26:08,216] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 213 + 4: [2023-05-08 11:26:08,220] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt. + 4: [2023-05-08 11:26:08,220] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 35 +24: [2023-05-08 11:26:08,222] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt. +24: [2023-05-08 11:26:08,222] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 193 + 4: [2023-05-08 11:26:08,223] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt. +19: [2023-05-08 11:26:08,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt. +19: [2023-05-08 11:26:08,224] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 155 + 4: [2023-05-08 11:26:08,224] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 39 + 4: [2023-05-08 11:26:08,224] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 35 +28: [2023-05-08 11:26:08,224] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt. +28: [2023-05-08 11:26:08,225] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 231 +17: [2023-05-08 11:26:08,225] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt. +17: [2023-05-08 11:26:08,225] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 143 +24: [2023-05-08 11:26:08,227] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 193 +22: [2023-05-08 11:26:08,227] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt. +22: [2023-05-08 11:26:08,228] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 179 + 4: [2023-05-08 11:26:08,228] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 39 +19: [2023-05-08 11:26:08,228] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 155 +28: [2023-05-08 11:26:08,228] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 231 +30: [2023-05-08 11:26:08,228] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt. +30: [2023-05-08 11:26:08,228] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 247 + 3: [2023-05-08 11:26:08,229] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt. +17: [2023-05-08 11:26:08,230] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 143 + 3: [2023-05-08 11:26:08,230] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 25 +11: [2023-05-08 11:26:08,230] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt. +11: [2023-05-08 11:26:08,230] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 91 +22: [2023-05-08 11:26:08,231] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 179 +30: [2023-05-08 11:26:08,232] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 247 + 3: [2023-05-08 11:26:08,234] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 25 +11: [2023-05-08 11:26:08,235] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 91 + 9: [2023-05-08 11:26:08,236] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt. + 9: [2023-05-08 11:26:08,236] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 79 +24: [2023-05-08 11:26:08,239] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt. +24: [2023-05-08 11:26:08,239] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 197 + 9: [2023-05-08 11:26:08,241] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 79 +26: [2023-05-08 11:26:08,241] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt. +26: [2023-05-08 11:26:08,241] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 209 +24: [2023-05-08 11:26:08,243] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 197 +22: [2023-05-08 11:26:08,244] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt. +22: [2023-05-08 11:26:08,244] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 181 +26: [2023-05-08 11:26:08,245] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 209 +13: [2023-05-08 11:26:08,247] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt. +13: [2023-05-08 11:26:08,248] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 105 +22: [2023-05-08 11:26:08,248] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 181 + 7: [2023-05-08 11:26:08,250] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt. + 7: [2023-05-08 11:26:08,251] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 61 +13: [2023-05-08 11:26:08,251] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 105 + 4: [2023-05-08 11:26:08,254] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt. + 4: [2023-05-08 11:26:08,254] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 37 + 7: [2023-05-08 11:26:08,255] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 61 +12: [2023-05-08 11:26:08,255] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt. +12: [2023-05-08 11:26:08,256] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 99 + 4: [2023-05-08 11:26:08,258] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 37 +12: [2023-05-08 11:26:08,259] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 99 +22: [2023-05-08 11:26:08,259] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt. +22: [2023-05-08 11:26:08,260] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 177 +10: [2023-05-08 11:26:08,261] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt. +10: [2023-05-08 11:26:08,262] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 85 + 9: [2023-05-08 11:26:08,262] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt. + 9: [2023-05-08 11:26:08,263] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 73 + 5: [2023-05-08 11:26:08,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt. + 3: [2023-05-08 11:26:08,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt. + 5: [2023-05-08 11:26:08,263] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 41 +13: [2023-05-08 11:26:08,263] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt. + 3: [2023-05-08 11:26:08,263] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 29 +22: [2023-05-08 11:26:08,264] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 177 +13: [2023-05-08 11:26:08,264] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 109 + 3: [2023-05-08 11:26:08,265] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt. +10: [2023-05-08 11:26:08,265] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 85 + 3: [2023-05-08 11:26:08,266] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 31 + 9: [2023-05-08 11:26:08,267] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 73 + 6: [2023-05-08 11:26:08,267] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt. + 6: [2023-05-08 11:26:08,267] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 53 + 3: [2023-05-08 11:26:08,267] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 29 + 5: [2023-05-08 11:26:08,268] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 41 +13: [2023-05-08 11:26:08,268] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 109 +14: [2023-05-08 11:26:08,268] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt. +14: [2023-05-08 11:26:08,268] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 117 + 6: [2023-05-08 11:26:08,269] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt. + 6: [2023-05-08 11:26:08,269] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 55 + 3: [2023-05-08 11:26:08,269] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 31 + 2: [2023-05-08 11:26:08,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt. +12: [2023-05-08 11:26:08,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt. + 6: [2023-05-08 11:26:08,271] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 53 + 2: [2023-05-08 11:26:08,271] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 17 +17: [2023-05-08 11:26:08,271] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt. +12: [2023-05-08 11:26:08,272] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 101 +17: [2023-05-08 11:26:08,272] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 141 +14: [2023-05-08 11:26:08,274] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 117 + 6: [2023-05-08 11:26:08,274] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 55 +28: [2023-05-08 11:26:08,275] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt. +28: [2023-05-08 11:26:08,275] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 225 +17: [2023-05-08 11:26:08,275] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 141 +12: [2023-05-08 11:26:08,276] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 101 + 2: [2023-05-08 11:26:08,276] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 17 + 8: [2023-05-08 11:26:08,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt. + 8: [2023-05-08 11:26:08,277] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 67 + 3: [2023-05-08 11:26:08,277] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt. + 3: [2023-05-08 11:26:08,278] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 27 +28: [2023-05-08 11:26:08,279] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 225 + 9: [2023-05-08 11:26:08,280] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt. +12: [2023-05-08 11:26:08,281] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt. +12: [2023-05-08 11:26:08,281] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 103 + 9: [2023-05-08 11:26:08,281] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 77 + 8: [2023-05-08 11:26:08,281] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 67 + 3: [2023-05-08 11:26:08,282] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 27 +12: [2023-05-08 11:26:08,285] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 103 + 9: [2023-05-08 11:26:08,285] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 77 +15: [2023-05-08 11:26:08,289] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt. +15: [2023-05-08 11:26:08,289] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 125 +29: [2023-05-08 11:26:08,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt. +29: [2023-05-08 11:26:08,290] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 233 + 5: [2023-05-08 11:26:08,290] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt. + 5: [2023-05-08 11:26:08,291] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 43 +13: [2023-05-08 11:26:08,292] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt. +13: [2023-05-08 11:26:08,292] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 107 +15: [2023-05-08 11:26:08,293] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 125 +28: [2023-05-08 11:26:08,294] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt. +28: [2023-05-08 11:26:08,294] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 227 + 5: [2023-05-08 11:26:08,295] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 43 +29: [2023-05-08 11:26:08,296] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 233 +13: [2023-05-08 11:26:08,297] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 107 + 2: [2023-05-08 11:26:08,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt. + 2: [2023-05-08 11:26:08,297] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 19 + 7: [2023-05-08 11:26:08,297] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt. + 7: [2023-05-08 11:26:08,298] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 57 +28: [2023-05-08 11:26:08,300] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 227 + 2: [2023-05-08 11:26:08,302] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 19 + 7: [2023-05-08 11:26:08,303] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 57 +29: [2023-05-08 11:26:08,304] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt. +29: [2023-05-08 11:26:08,304] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 239 + 2: [2023-05-08 11:26:08,305] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt. + 2: [2023-05-08 11:26:08,305] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 23 + 2: [2023-05-08 11:26:08,309] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 23 +29: [2023-05-08 11:26:08,313] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 239 +28: [2023-05-08 11:26:08,317] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt. +28: [2023-05-08 11:26:08,317] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 229 + 0: [2023-05-08 11:26:08,321] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt. + 0: [2023-05-08 11:26:08,321] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 3 +24: [2023-05-08 11:26:08,322] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt. +24: [2023-05-08 11:26:08,322] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 195 +28: [2023-05-08 11:26:08,323] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 229 +24: [2023-05-08 11:26:08,326] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 195 + 0: [2023-05-08 11:26:08,328] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 3 +31: [2023-05-08 11:26:08,329] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt. +31: [2023-05-08 11:26:08,329] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 255 +11: [2023-05-08 11:26:08,331] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt. +11: [2023-05-08 11:26:08,331] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 95 + 5: [2023-05-08 11:26:08,332] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt. + 5: [2023-05-08 11:26:08,332] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 47 +31: [2023-05-08 11:26:08,333] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 255 + 7: [2023-05-08 11:26:08,333] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt. + 7: [2023-05-08 11:26:08,334] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 59 +10: [2023-05-08 11:26:08,334] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt. +10: [2023-05-08 11:26:08,334] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 87 +11: [2023-05-08 11:26:08,335] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 95 + 5: [2023-05-08 11:26:08,336] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 47 + 7: [2023-05-08 11:26:08,336] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt. + 7: [2023-05-08 11:26:08,336] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 63 + 7: [2023-05-08 11:26:08,338] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 59 +10: [2023-05-08 11:26:08,338] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 87 +15: [2023-05-08 11:26:08,338] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt. +15: [2023-05-08 11:26:08,339] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 123 +15: [2023-05-08 11:26:08,339] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt. +15: [2023-05-08 11:26:08,340] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 121 + 7: [2023-05-08 11:26:08,340] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 63 + 6: [2023-05-08 11:26:08,342] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt. + 6: [2023-05-08 11:26:08,342] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 51 +15: [2023-05-08 11:26:08,343] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt. +15: [2023-05-08 11:26:08,343] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 127 +15: [2023-05-08 11:26:08,344] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 121 +16: [2023-05-08 11:26:08,344] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt. +15: [2023-05-08 11:26:08,344] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 123 +16: [2023-05-08 11:26:08,344] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 131 +29: [2023-05-08 11:26:08,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt. +31: [2023-05-08 11:26:08,345] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt. +31: [2023-05-08 11:26:08,345] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 251 +29: [2023-05-08 11:26:08,345] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 235 +16: [2023-05-08 11:26:08,346] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt. +16: [2023-05-08 11:26:08,346] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 133 + 6: [2023-05-08 11:26:08,346] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 51 +15: [2023-05-08 11:26:08,347] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 127 +14: [2023-05-08 11:26:08,347] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt. +14: [2023-05-08 11:26:08,348] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 115 +13: [2023-05-08 11:26:08,348] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt. +13: [2023-05-08 11:26:08,348] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 111 +16: [2023-05-08 11:26:08,348] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 131 +31: [2023-05-08 11:26:08,349] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 251 +16: [2023-05-08 11:26:08,350] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 133 +29: [2023-05-08 11:26:08,351] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 235 +14: [2023-05-08 11:26:08,351] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 115 +13: [2023-05-08 11:26:08,352] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 111 +31: [2023-05-08 11:26:08,353] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt. +31: [2023-05-08 11:26:08,353] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 253 +14: [2023-05-08 11:26:08,354] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt. +14: [2023-05-08 11:26:08,355] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 113 +31: [2023-05-08 11:26:08,357] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 253 + 0: [2023-05-08 11:26:08,358] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt. + 0: [2023-05-08 11:26:08,358] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 7 +14: [2023-05-08 11:26:08,359] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 113 + 2: [2023-05-08 11:26:08,361] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt. + 2: [2023-05-08 11:26:08,361] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 21 +16: [2023-05-08 11:26:08,362] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt. +16: [2023-05-08 11:26:08,363] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 129 + 0: [2023-05-08 11:26:08,364] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 7 + 2: [2023-05-08 11:26:08,366] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 21 +16: [2023-05-08 11:26:08,367] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 129 +16: [2023-05-08 11:26:08,375] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt. +16: [2023-05-08 11:26:08,375] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 135 + 5: [2023-05-08 11:26:08,376] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt. + 5: [2023-05-08 11:26:08,377] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 45 +31: [2023-05-08 11:26:08,377] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt. +31: [2023-05-08 11:26:08,377] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 249 +16: [2023-05-08 11:26:08,379] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 135 + 5: [2023-05-08 11:26:08,381] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 45 +31: [2023-05-08 11:26:08,381] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 249 +17: [2023-05-08 11:26:08,391] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt. +17: [2023-05-08 11:26:08,391] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 137 +17: [2023-05-08 11:26:08,393] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt. +17: [2023-05-08 11:26:08,394] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 139 +17: [2023-05-08 11:26:08,396] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 137 +17: [2023-05-08 11:26:08,397] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 139 + 0: [2023-05-08 11:26:08,408] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt. + 0: [2023-05-08 11:26:08,409] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 1 + 0: [2023-05-08 11:26:08,413] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 1 +12: [2023-05-08 11:26:08,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt. +12: [2023-05-08 11:26:08,417] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 97 +14: [2023-05-08 11:26:08,417] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt. +14: [2023-05-08 11:26:08,418] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 119 +12: [2023-05-08 11:26:08,421] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 97 +14: [2023-05-08 11:26:08,422] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 119 +24: [2023-05-08 11:26:08,455] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt. +24: [2023-05-08 11:26:08,455] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 199 + 0: [2023-05-08 11:26:08,457] [INFO] [torch_checkpoint_engine.py:23:load] [Torch] Loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt. + 0: [2023-05-08 11:26:08,458] [INFO] [engine.py:2844:_get_all_zero_checkpoint_state_dicts] successfully read 128 ZeRO state_dicts for rank 5 +24: [2023-05-08 11:26:08,458] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 199 + 0: [2023-05-08 11:26:08,462] [INFO] [engine.py:2784:_load_zero_checkpoint] loading 128 zero partition checkpoints for rank 5 + 0: successfully loaded checkpoint from lm1-4b2-84b-c4-repetitions/4b284b6bc4subopt at iteration 0 +31: time (ms) | load-checkpoint: 12177.71 + 0: estimated model parameters: 2.50035712 + 0: estimated model parameters without embeddings: 2.36076544 + 0: [after model, optimizer, and learning rate scheduler are built] datetime: 2023-05-08 11:26:09 + 0: > building train, validation, and test datasets ... + 0: > datasets target sizes (minimum size): + 0: train: 1 + 0: validation: 102400 + 0: test: 102400 + 0: > building train, validation, and test datasets for GPT ... + 0: > building dataset index ... + 0: reading sizes... + 0: reading pointers... + 0: reading document index... + 0: creating numpy buffer of mmap... + 0: creating memory view of numpy buffer... + 0: > finished creating indexed dataset in 0.031635 seconds + 0: number of documents: 3133972 + 0: > dataset split: + 0: train: + 0: document indices in [0, 3133972) total of 3133972 documents + 0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_doc_idx.npy + 0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_sample_idx.npy + 0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_1B5_text_document_train_indexmap_1ns_2048sl_1234s_shuffle_idx.npy + 0: loaded indexed file in 0.084 seconds + 0: total number of samples: 731002 + 0: total number of epochs: 1 + 0: > building dataset index ... + 0: reading sizes... + 0: reading pointers... + 0: reading document index... + 0: creating numpy buffer of mmap... + 0: creating memory view of numpy buffer... + 0: > finished creating indexed dataset in 0.005456 seconds + 0: number of documents: 364608 + 0: > dataset split: + 0: validation: + 0: document indices in [0, 364608) total of 364608 documents + 0: > loading doc-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_102400ns_2048sl_1234s_doc_idx.npy + 0: > loading sample-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_102400ns_2048sl_1234s_sample_idx.npy + 0: > loading shuffle-idx mapping from /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document_validation_indexmap_102400ns_2048sl_1234s_shuffle_idx.npy + 0: loaded indexed file in 0.092 seconds + 0: total number of samples: 169955 + 0: total number of epochs: 2 + 0: > finished creating GPT datasets ... + 0: [after dataloaders are built] datetime: 2023-05-08 11:26:14 + 0: done with setup ... + 0: training ... +31: time (ms) | model-and-optimizer-setup: 29930.29 | train/valid/test-data-iterators-setup: 3807.27 + 0: [after training is done] datetime: 2023-05-08 11:26:14 +31: ----------------------------------------------------------------------------------------------------------------- +31: validation loss at the end of training for val data | lm loss value: 2.650107E+00 | lm loss PPL: 1.415555E+01 | +31: ----------------------------------------------------------------------------------------------------------------- +END 3476842: Mon 08 May 2023 11:27:48 AM EEST diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e38701a268e56d3cb8bd2b2ac4ad235e1a427cde --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:467adc4f5c9187abc4a1d308cbfc04751f02b6b7eb9b8b553c20cc4595fe2f5d +size 117208855 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf5043f43e84c741c7adb80f4695ccba10d89605 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_0_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52beb3df982aab49ab40690c415b69aaa85a0f35162a1612827a45475b042063 +size 117208855 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8bbad5cf059a5800220149cef6135e2175b18ca1 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_100_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a6f5940ff22d460952e9bd0fc517b853f31058795f2af439ad2f5e5931d86807 +size 117208877 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..75553ecf88d0b4b26321818febe8bd7eb28594f0 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_100_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc46a6a080c67752585dbdc73fa81b18a0b0cbcfebac7d961928e65932249626 +size 117208877 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf9ff21406335c1da47db9361d597101e08b050e --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_101_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a943207ca99a89e845ed22e9870ba52a2d130007f2d9525c4b484d61411de22 +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ffe42333625c06fc1512240cb41b9452b96fc4e3 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_101_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecc2aff8e973788e2fc2197c640e240113c207c68c05856836652206c2b1ad04 +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ef91e67f67a278da9588b6aba0bade9cde03fae4 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_102_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:084eddd72b6c6819f4b3e124bed994dc89db178eca806567642394184a0f8d67 +size 117209005 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..38facac190f492d5e9dc904bfc684f90c8eed69f --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_102_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4203f02d74218eee62eeac6776b4ec886bb9ab3a342e2a73442e9277508c5c7d +size 117209005 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e492ef2adf0cb3ed373ce73e3a6b6e70e9ff3f97 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_103_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd173ccb5c0aade0eed56cecdb40af2e8379b0c0aece9ec8670df21695df3d86 +size 117208877 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..afa6597e03b954efd43fef546e45ce5b385821e0 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_103_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13515efd5b00e3c377254e34d545b344be8cfb6ddb1cbe05718e8bc22bd2066d +size 117208877 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f89fec3279216d5977cfb44cc7bb16461a4ae18e --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_104_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa1490b12539898d21b8c03af8e99158b52209621ed7ebf87931673ff7d57f4a +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ce810da423c2089b81d89122eacc19e72c28d5c3 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_104_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74a5d8024de48bee192a84df6c79272501d6da43685b04e702db4b4d5cc8a139 +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8bf8c59170acc61afd467bc02493389f24a1655a --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_105_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:210a98b96a7fe3cc1a4f921ac74009b4c0458f07976ab716897bea07b690de32 +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..70e03ba3eb7c46c213b540e846c757524d37d4f7 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_105_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eac648107a6c4ec789e07d21c619d92c991acd2b400aa00c1ae3ca0f8d50c77e +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..67e2e05ace47b499fd603dfa23012618132ee556 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_106_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b950d91f3abd2da86b0bcb22fd9aa92f26a30d9d9e2b83e4c0152728f4a1d9bd +size 117208813 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5e53baeebb70e54a92804462c384d50c9d5a313a --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_106_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3855520bf1f88f5f063c7246f0bb79248d3fc242d80dd3993284f88d0a84e560 +size 117208813 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f8743e88ca9e46fa3ef30b43a6759632bef8ca5 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_107_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61f4c643d879f905018ec932a96c11b9be178b27d84346f0c388c022b0536718 +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..34ef0afeaf5479cf8919c2b2722e581e52443419 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_107_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0bf11b1194a64024250f8da8700610b71b4fe904e27a93946df0646589226c7 +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a8ec4d190c747e9b04867db4cf7a61a2c45ed04c --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_108_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d7bea47d06f82dc289cb75dafc4ea9a2c84c5dddf2eb34da21fb73e044d3ad5 +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..36daf89150a24c862d443559f5b864e1c34636fe --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_108_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:151d5d673c9c2aab20f2dab0ba7b2029fa5d8fc2da9a47386f4b5e96c4d84937 +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..defd3ce756b903c4c130ad79630ac49cc7061b15 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_109_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b0d727ae4142f9c2bb792dcf85d8c96c9b914f1044ca3b8978f00ffc009d86b +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4e3bc166324348afd3e8d7f8d439fbc572480fcd --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_109_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77fbd22c89a1de9e7a6675176cc9c10eb52eeb1e4fd7baff24662705fc8f226a +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..36900ce1a514be6281671b60f150068f12e4327e --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_10_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc079a416838bda5cc2ed9f4aaa499517ec938dd3c064dc47482c9f794a1daeb +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7c60adfb44c62d02e0af2dca3aa70fe5e762bf1a --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_10_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cf62d8bf4fd0a450a16c5bcb39f9af994e06a253fbc9e56ab24a718f328d95a +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..97d4c31c32259aa011ac429343f53d3cd46011c8 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_110_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5a3723d3d56ecb412f21c4a4cd137cdc4fed3e490b1a5db09833089608ef421 +size 117208813 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c06f5b21df96ba8361f88b196292bbcad55d4d0 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_110_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc3ac2eda6de0ac786472b1ced288365230f5a4d2a4f33a6d95e1fb7be8e275d +size 117208813 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6080465cc238c35c98c6094464b53ea8b042249 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_111_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f3fc322d0189988355087a4db52c66deb9a99b9cd6081bec096be06292739e0 +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..df3ce55128d680ce4139243c1babb3488ae03f75 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_111_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1e1c97ed96994501e36c2f8725e78cba9471fe1ff019d13f3de9e2c70e5e5b3 +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b77c13086dea79c49212b5e7878ba270a75320cc --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_112_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ac8d81637b919b12f90c40d4adbff58539d5fbe7b191e8f6e83ebd53d3679d8 +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a868c11d24c128415a20dd30c86d6fbe9855ecba --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_112_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e0d2a727097fcc77f22e2d7b7932ae2bd40a5f007715d3b69358050edb2f162 +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f5bc31f0c402f3c744bbc880cfaf7eb81b0846c --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_113_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94d4b1002fc59fab5e56728a6ccf098aaf29e1ce53e04fc7fe75b4d64b85c946 +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dba93b504982420d89b7637d0526fb99891d4e9c --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_113_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d739c3cb568f7e63f7ae2e149f83685dafec785aba5c75e466397090c883f5be +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ada30eb8b80c28cbb786a99dae56340a833a785b --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_114_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3507c840651386bf4bb1c7116c90cfadfbd3ed08585adce8d4b7f1f44a12aa4 +size 117208877 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4312b72a3ba0a6900023ba8de6a1da99a579dd88 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_114_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:538c08d30d46e49635f7425ce2156c3f9ae75bb2036b6d82eb9e64006c2a3362 +size 117208877 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b242ca0107ceaf9f5133967eacc29b4d2c090aa0 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_115_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c95116d0771a18468ca5db9885caebac6a83fd60f2208573af89c34f40905cb +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c77cfbb2047be70ce0a69f03074ca8b377b58630 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_115_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:572d4a8247abfcc5b80fa3f410db5c95cdf11e1b1d470596eef6e2e0e036678d +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e966bd6b31df55a09ab680ffb95a7ce4b4ff5614 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_116_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a2527b0cb664e0002d144f0b22cc0034160d75d05bbc9f1dc562d2937eb4356 +size 117209005 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d3d09d411c922064f388a5726981b4d8cb11d1c1 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_116_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89f2af024c7665a1cbb61ab0244ff33b8e23e87183d131678d6cd5ee7ba4c07e +size 117209005 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..78f24dacf4b7793d18076866aa577aa0c52f75b8 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_117_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98a33a584b9f2a9137b4dc54d68a81d94163e266ee6634bf843f54cd03866e24 +size 117208877 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..562622280e1ed402ddd5921b2ccc6789a4169b3e --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_117_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c03ea0b0ba78817cb84940714211c3c09827653cac73b0b28019c40a819b2c9c +size 117208877 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0f057ae574f55c904b14e1934d9bd2ea155165f8 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_118_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e76a1fe9903319c755ff072af7879f18a57418308ab501fadf68e276112dd882 +size 117208813 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f335ee66b8c247437df0fe0cf0b2ae5af52d0bd0 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_118_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccd73aeb7939cc0cdf57eb35f292ef80efa4eaa4aa7354a8441cffadcc5a3953 +size 117208813 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a5a36f26c27f9f0d60a88c78d500681e7bccb5cb --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_119_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6235d5414d258f9d3dd46fdb1f9da0cd6e0d9020e48304dd6da4f42b94dbbcf +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..749eec090134c8739aa9fa3b4d8df42d16f6927c --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_119_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02a74298c1aed579196535b2f6276f85bd94adff8290a7e4d7ffe83c4a90514d +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5a482db3d6c67987a185372d2d84ec60e96f614c --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_11_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43001f9a47eecc360eb62fef161bb40916c7fdd2c81e83d1674ae20b3fb42545 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..14fab9f6555da532a640a7231ef8a44a6aa0f13d --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_11_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e4b1f6eee80f445dd1f1e4d850caa632e9492b55db688f0e5e7b6c0abab89c0 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..89c551a823bc2584b9bd4a785877d924df14a501 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_120_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aefdb167c8a7fb5c6587631cee7e6e0d0d0fc6c4240a7ab0e0be04b53ed5320f +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..506aec92605a6784c2f1a3bf36e203e8102fb816 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_120_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:374a0a9261b3825a5239404102c6b46fd890124719e246e18bb5a3e1f6603f5b +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5438c63a8e2ab5a340f3df3b4638b834a268c4ed --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_121_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32e5011df9b7c812829918d6ac3b93590af3fd04f5323d26c780ae21b930d0e9 +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b617ee86553cee135e2d3363382de45084e007c9 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_121_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c24da05525bbc9c7bd6a50f76d458c943bef48b70234477a24180f078e9b8ee +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b800933c64e0f7dcbc059f05a3afe8dbd3539608 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_122_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b36483bef0dfcc1c9af361d8c08dea8d83d6653099ec70bab0db6bf38b70ec82 +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4755f419729f858522f612c82a6e05b76ee22181 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_122_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4d2c80f761f0d2205bf258fcedd8e908bab96522798a707a350e9593298a6243 +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..715947a238d89963979069d4b647d89ce89f2fb5 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_123_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e630cf9c49f4a1c4565b60a695f43500a047f99b0eb8b227d8a9d3afacacbec1 +size 117208877 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..784d224ce8a56ef4b718e94762ed872ec56e9cab --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_123_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3948764403cd151e718859fa7f4271c4f3cab3a7eb8e7fb013f87149b1c1d7a7 +size 117208877 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4a3a52b89b4866f378bf28043396ae525abe7df5 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_124_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d390e81b8c1e5c81a577b475c3bd6703f6212d45ad28db52596d1223af925b01 +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c2a5b8e945b1f60e2cc713019ffafff4e4f7289 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_124_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b266eb49eb22fda747b08f716d190f14c3729255f95243d2aeb0443d337cc2f7 +size 117208941 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ba275e03f06b6961680c39dd12f62e548b0395aa --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_125_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b75f7addb02ef58ebd9e7db3aab5b12973929800dfaf95373842a83255ac1f1 +size 117209005 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ee41102456f4d7ff35397104fbf677f58e425eb0 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_125_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:852ffb31c33e8cf1cc77a4544eff10f2f11af64becef4a491e3ccfe8751df8af +size 117209005 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..52587ea84d82857b0033096f79bba6cb8b0d8509 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_126_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:baa52e44411c7073510fa597ef8c990c23054643e532a297a968819726c8d3e6 +size 117208749 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2f6c28f9bfdcc0024b2fe216bef3e35d86942b9 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_126_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7eeb1fb6751943202da0a8bac70134f2ffa132773e74111674634e23f7867d8d +size 117208749 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d0550b515a338c17473769c36b5e10e104bacda2 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_127_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd20d8f1cfd1fecc93f351d36045fe887583271709ff85e7870c516d03e55f27 +size 117208813 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b01ef8da341140c20b6e4732d9a022618fbe9f70 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_127_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7149a95d0c48a5a64919dc081495b561e8e15ca6e892c051ec3d6738b1c2bbf1 +size 117208813 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..41f2e4ada7f6347de23c9d5a1d16816c0080735f --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_12_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bef87c2a234e3381c1233a8fcdb6d8ff5d6472c6e09adf69c0efb623fbbeac32 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..91f386bb8c1c3f2302a13cf3557069a3506f2f75 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_12_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42b6b6a536ff0fffe9520c6589ef3f4b3708ac137a0249e5720d4986326df686 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..37b25af8966255ceedd5f7198c3c7edc81313d62 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_13_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d08fa375e42ebabc980d1912d0f26920a8f9f0ac619119c8eb397c03cd0f3cc3 +size 117209058 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2fcac5bcf2141edc28ba14b04a5917c90cd9988 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_13_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cf0f16817fa2474ebcc456da208f7c952046290a6af40d790ab54510997aeee +size 117209058 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c6c311e58adf27c7b6c422629418d8d35fe587d --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_14_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b8e2b194e578c1315ba5c03d8a439bc3511e0d30ae2d76af4681d5cfe9efed4 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4e00a91abc0cd3ac987038b5265f55f47297e332 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_14_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94127912ec473eee12a429105eb57e09ebdacf7e3076c838cca31ba4aca715d8 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c7344ca124d75b4ff47c5854be4a3ff532665e87 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_15_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66f3b40323e8b41439069ede50c94d472996bf2e2bce3e0be847de748603dfff +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..31714234a0e574eb957850f476940954aa501221 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_15_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c675a97dfc2b711b7bca510f1d4dda24da7f8f8c62fc855faf84de4390ec6e07 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..609812f9ebfdcf3d6d7eeead985bc1f5b5ad5d29 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_16_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a77ba2e6665f6bb7866eb89d613c5ea987d3e45c63b514603226a829a1f42fd +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f455f24d1a51c2b597e0f374e83a59bdda9a51a7 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_16_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c97b0ce8451a6498ef26c7ba7f1d671247a0b07725623b142fc070206eb3c88c +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..44080147783db0a04a51928936f8f4e9e557853c --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_17_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72f40a24bde6a910c815d246fbc2890dffc5120030993b47662b9bb8855a9ae0 +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c14ce1c0a866c6b552a662760725cea83af1a9df --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_17_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec5ddb68dc078056c6e36e839b6b9121b0d1c48d55800af56609d91375259eb1 +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e097b6b72b4615e906109a2a171c665cba27da2d --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_18_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7a6e9f014faff7cac1f28c7df26053d12038df829e05f89b6344d3195c2e5eba +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4d995225506f4f991c32d242cf81fc2b2e8f7ede --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_18_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d5f062cbc2e2363c015661c5b1324d6cd4f99f80443b3b3ec4a6fc85a3d10b6 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..59b7f25e8f291f0c071901ecdb42a386eb756480 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_19_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:857f65454c06aa3285b1842cdb34ed57c8706054845baf3127065b2f1019907e +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2bbbca40d28b1a390af1e52c3369246626c40fc --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_19_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9e78196cb4f01746b94483553f6c9fadcaebc414460471affb47c87360ccb3a +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..018e6eb42de0a8537ffbb866ae99e36a496da2f2 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d4f9c8814b3ce07dcb3422cde31c11ddf4085ffe490f06f0b2c87d8a2b1b069 +size 117208791 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e916ccf37e03f7cbe22df22b09893cbb444be769 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_1_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7e86691913b70de7d7134e16a9329c7440012d43159dd28259a36d11573b672 +size 117208791 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..87b429c9805f6228921d8189e3fe00ec8d1afd75 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_20_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bec7a37d5f21c04ac394296d64b9ec77fa739eb280e6ffce7c251299aecd87a +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4e0546782c68e5d3ee5e478df48beba22f42a0bc --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_20_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0f3821d1a2c25e13a456388c68f3458133f392045af0239ef1934a309444ae8 +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..76eaeb79b8fb68c546ec953f5f80c01ce213b117 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_21_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d60561679b4c2f42ad90f6546bcc23a402ac2d672d59b2fbdd328629ab1fcbf9 +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5f49571102a4687353b51a7411b66b117f7330d5 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_21_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:715206d4c61d1b1b1a1c3674f5a5c3543f8d2ac04d03884317605c431f204961 +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..03751a39c4755e54ca9f6d8d8c50967315d88543 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_22_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b1cdf65c1b006fba4d98245a107448da98fde7595b9a27085835813714bec9ac +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..de2de7592217670e73899e5b7a9cbfafb5f21c70 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_22_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e60e47bd4bc896b025a6c2184c6d30581e80862f67699261da2ffb30763e343 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..57dd4ea52b78442df3ccde8708193dce7adfac54 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_23_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c0d6b476584235597702d02d142832882ed417fa3a666c1f8800c2274746319 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..be30c53b92d26e55904f35ab4a4bfcedb7073e6f --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_23_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:735687b4193be24cc43a88c8aa6cbf0cbc537ae90268cf1149f69c50b002841a +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ac85d94e976de23815e08a61c639c90304e29e14 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_24_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f08cd0c7760c2c7a2e2587dfb818980bd5ea883e23edf9fd4ca05d3815dc0371 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c7f9abb4b66717017c5168d447105ec6ce145def --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_24_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f5a972923033abe30592e4cc1c6eeeda9f374413039350154c33a8a19ea200b +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..02c58f9164f4c7b382a9cbc6b4930e024d8b23ff --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_25_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:af3a4b07b533d9345c6e4ff367e3143ee988dbde5b575e56cdc6e714d9fe9731 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..932e72542cb585e8db7e7499a38138830e837146 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_25_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7a1944e178a07f2d86dc03aac8918993de82601fe86e4ee864055e4768daf6c +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..25ba14c962693b03445b9ebd1089c9b452c69ef5 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_26_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28c75ce6200a761776e85b19df94993ce64e95261280ccf5d79601bc186d9c96 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb53b5d672ecc7f33f87838df32b3ee1cb56b0eb --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_26_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81802601e06e9962f93a00d99cd244220786ee917639d44070d459644e39c6e9 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3e9c2642af1c21e30ff072e92c1545517578e9d7 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_27_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66ef789cf605c3c2ad7ed9afe5b4d08cd428502389e7bf574f3d93afad57dee8 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..83a38a3ae90425b895e8f8f959fd1f0051c4ec96 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_27_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2070c63881446649a7b2d1657b07ac37d04a1bf1b3de9abf1235475bec52a38d +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..20157f672dbcae52dc5554a4018cc64c8ba6f5de --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_28_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:68a72348dd05d5eccd52049c219e834769089095cb65658facaa4f82f222bb5d +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6b47244438b17313fe49847b2d97e548adcb53c --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_28_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1828282199c2b71618eba6f2ec8630ab0a6547a4c055d28ab8aa54c80464af95 +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e9c9b5c2f6d48636716601ad0a7ea7e44ff794cb --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_29_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e13308df4a77a1e8ce4c44a3c664ebf38f47e5682ea18bff72daba6b7cc23bc7 +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4489759355a5cec9298093c6f6d96eb6a0c8246b --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_29_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bc2c90a6a31472def216bc574d30f844dc6977c9ca9e699fa035708c89e6ee2 +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..70c48f8d7616d5ac65e13a0bc443c6ebaa340ad9 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ce35d7ebc2580e5805292fd4637b40c4b86b32051a756943e8d29e1c98b6231 +size 117208983 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e0965a0304f3c977e406bd7822ad97683f827404 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_2_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbbb53ce5e492bd373cf6ca857e37abc54642858f0bcb0441ee99a98d49856f8 +size 117208983 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..67ed182cf2fb0ddd1b3f498bbbeb65faf7c18a40 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_30_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:572422d8e5a69157bc1e431c8f0527ac7a8d56e8e83c137b762cbe62da627f6d +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e4b680a84480f0c68f9ef8a5d1c0a41c884665cf --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_30_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51607b97ee9ae6333a414281fa14388555248a8463081fd948b57b21d68fb2d4 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f51a72be591c0d72ef479c1f8017cbdaa5ed3a6 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_31_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fec6541901038af959a9a0a051dd17ad8fd995fa04e22cfce216ac275acf0a5 +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a1e3ae9d7d8aa41b79c3ddcdbd0d844c888c1715 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_31_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f83ce6766938e405df590147c0bd107f7d03e60630e6b6b349fa7153e67a7aaa +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b78064cbd0ed1c430a5ba6a58077e44df37c6bd --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_32_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ae9d1f381f82d40b921b4bb906b39a8ab3309e9e0dca82e67119158b6eabd9f +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..09db4aea32bc164eb2d31edad6d425b7ed31ecdc --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_32_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d17a51381cea2e96ddd6834d03c96f269cfdfefc288d73a2c6ab904ee2350706 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..58ce1ed2da69802231959847965a323ac51f56b1 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_33_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f78af42e36ac950961d5eb795ef595bf466311f677c84ec99603bea9f4d3e8c8 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b2b8a5c27a8d008dda41b485480c92e379bb2104 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_33_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7f4bc7d64a622e46d20336c0c73ed33c26a85a3f65ab0e8193852d103a083e1 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b846cb697f99662de7eb12f1b7a89bc91c86a952 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_34_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a691ff3554bcdbe8175b510e0b352839664b421c9d5c6f80d8b2ef997d5ff3d +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8f884ca67032fd8d7591250cf120749d5ba14582 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_34_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:585e759c97bcad06ff7c9d91cdec5ce3a65df7372b605eab0f090a81186724fd +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c0efb17dee613a3a63f31f4001d8726701c2dfc8 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_35_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ce945fdf37e75a84cfe857afce7503546cb34f87aab69938b10bbc1f899ad44 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cfb67ffee28bbaca62d1c77107bd67074c2b2e67 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_35_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c177c7d55a4e0bfb9a8afdd7f537ceb5dc656e9e9dbb46b408bb67ab61543f70 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9bf6d815b2bdf7fcbf2612a989a08d9211dde229 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_36_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a86ea18865df9fbfba7384016d0e3d729c808935ac47c17bb344eba340bccfe2 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ef1a7143d02a5fffa623c9594846a15853c7b2c9 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_36_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b0521ab0df2c62cf3dff3fe90f491fd784e2fd1062dc34d1bdbdce78c19bc47b +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..297f552c6a43e38570bbe504d67488cff42155e0 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_37_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fe084946cc39bba29eb60d3d15fbe82a15d98e5a8646e16b650f8a0cf2be258 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..90b9edab84d6eb3cb85aeb30163b5fba9dbc2406 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_37_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9d8da5548aa8af3e5cd2c4ee0c0753f3736b98185bf58a8a43a5b0a6fe3d91d +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..30e221906612276fb6d48ad64cc0f5c11e8dd9b4 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_38_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86a1e20bce3e848f182691dc5e90e156f1e0d39936bea936e57bce2f833e6a84 +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..88ca01f3ef4788318c5f525da859a064f48aa573 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_38_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f544a58e585aaf2413cf299a5f3b858ebc57df175c74ad430b3538625d64156f +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2bc6092a56dbe0e1259decb32ea2ec7760d2546c --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_39_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3d712cdab5ef2b02cb5f42b33259841e451a9028799d28b6fae1d9646ea96de +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ce893499e714fe4049bb31e0cf510bae74c6daf2 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_39_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5d936440894573c8ac7660511c71c3139dcb3689fc00fac5e8b9204ed8b698a +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9e7388c3cd80d8555580c4cdbdf6f1a084172442 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:251cd34e0bf3fd8b8737c2c7937f84a4698a05f86550b743687555a3fc603275 +size 117208791 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0b2a5f9afe699c8acc3b207b5969239ca3ff018d --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_3_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab8b8b043077a21385b540d5687515c9be441e93314520ae4a99eded953b7a17 +size 117208791 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7e4798f2403ae4d30150236b8855f4401aa283ba --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_40_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2634f48532f6f3f711d8e5f2fd4426522d17cb7b4942e27385ea02a1d35395c4 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..87eea6962be9d5381066be56c02c13b0df5fc8f7 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_40_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e9ed952bc383fdb511b8d0ff2bea1c04ab31046d3105a5d8fbeec37e9302ebf +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b452c4b7bab22da4f44f2014b8985de6b3375b83 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_41_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08e8899f731279823dc47590b1ef7dcb1b2d1a360f70403054597496e2bee437 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5a3bab19989670e815a4d9a61e761b63899b8aef --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_41_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc9ef65a8a90970d484fba5ed46c2b7d9891f4e5224775524f07bfe7f4ebae10 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2cbe071c3ddd0cc67e25c7a535ab2dc0f45dbf16 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_42_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61e29f19336d25f6e4d889b16d6650cb4fdc613ee5f380e3c59df91a1cccf978 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cbea51b6b21237e5a77a86acaa8f699c1c217e25 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_42_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f17a080e39ad08de6940d9f37b1e1d38207c0cac2ee42b97f065020aef445f6 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b68bc061ad5854676071f9e37810704941b9c642 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_43_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d10e2363dd6608d76205d3c540948086d20762afbb941008ca778417b95b770d +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5873058a93ad9a41c70349e2a7149e795c21347 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_43_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:08036d303659ed122cac67b41718017ea80a1ce359c1844543f7a357bb325b15 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0f5e664d94f211a72d0be24e8ebe64f849a2f51a --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_44_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fd829e873a90aebc5f4b97b01b7c12a45d897cbfb4d3d8ed486cf76088804a1 +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..857127a60a0e4194697631e7faf538c7230ecec3 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_44_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:094f53c9e73b7d60ac548d9d07ffb06f082237fc2175d422119c645db1153107 +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e59e9b7a194cab94e8e917d8e1c7e34e32b7a089 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_45_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aa3a277316edb7dacb22d2b7dc5c4e4577fb176fc17c967c3702b25373eb24bf +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f4540468bb0e8dc2c627cd93833eb872e8714b5c --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_45_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8230f56d2c84f73033e2d7794028ac06ab11b78f73ec85b5344ca8ac93597ff3 +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e73b8323e4cb573213a3ca1bc4507e7067afd802 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_46_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6153dec4f03d7624d91d8b53b894631c6263c9cc1f2c07c6ef53a503ffbfb516 +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d2fd2dac704e77309ec0d898459f2abc8c0c1d3 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_46_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d364731282c2127664a253f7d959cadcdd77bbfd8fafb457ae7c0a3d38ba42ea +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c5a922716b7f408bd11834bde8cd733ae3ad20c7 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_47_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3da300417e291e706e02ee3f5489ee8e226a689d8fecb1a5f9fed28b7e6f2ad9 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..189a90260b5914658a662451beadfc954a50c528 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_47_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1026dd5f9de6a06d481e1531384b3a0acfe53e675fe4ffe43e4972613934e7f2 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..522a0349cb3ae40beb562b32344971320d455826 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_48_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f115d321ab7c1d3a260e100d1ae1d9304bafd12839de568191a6c2bd7d06782f +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f306a1edda5b6a4970fea199c3cce04c420f957 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_48_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06b64860ca7932c2a3ea18ae8ea725a588084aef6b1820c43bc761bc4daa5ac4 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f5a3a1696a97ad32e37622aee55522b6c1611226 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_49_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:081b2e71589a70fc02aba76fcc28e5b9104bf15e263a837c7dfbc0ff19fbe36b +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..040e27e21cc9bc42feff3b89a577ac6cdf513918 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_49_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8fd7ce41b1a2f7379b2594fcdbcb1c5db7a451e898e11e0a1139bc4a7614635 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2587fa704dabc6486d84f4998b6615d891ea67e --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61b5b98c0775687d467f53ace0fa28d21954baada774de69cb37a039d4c28b4f +size 117208919 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fad25663855522d1fe684aa5dc8c663805202da4 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_4_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f40ff0d89a196a07ed267bd9b56753645f5c5fdb0622926e0d8bd65ecfbe946 +size 117208919 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ec47e2b39647a2a2184d2c43889c2293e9320ad --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_50_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3d705711a5d06caec0b9664c1b58c008b64d9a706928fd5ab2bfdb07576c070 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e06395079e00c538ea83b2da8a08b022de0a3845 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_50_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46957b97cc86149742ccfda7e3a55d8211bc8ab89d7b1a47224a8d3551c03339 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6831b9ba01144c01fa863c3fe202d0151c74e6d2 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_51_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44183c41d490f73cfb2df961f0050a8703c1c83fc59ec3ca8cf8d6bc28bf701d +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..afa8b0eee1b4ef4038b3a09549e61256c1d7f242 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_51_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e4e210d4ef316d349723e5f7a4c707d9ae19512307961fa75ae931d844baf4a +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cbf383ac50fc5d228da1235512245545aa8f39ec --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_52_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8c73f1c7a5c730383c3875d0c04d85df867340934431386df5abee3ab4d0359 +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..06d61e1b1179298d2d88d863ef6c74a6eddab536 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_52_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38993c561a39eb266a8a51873fcaeed6b1bdbb16dc3f15cab89bb673789b5bab +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c0915d3664d42c827f2b83f74ae2cf3b46f5447c --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_53_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3440e385ee415e4918dea40edaeb7896f9e3c8f764db2d38fd30f90d4bf22afa +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d0a554ba769518a3dfceda10767ffe637f9f4da5 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_53_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df599d047d8bd31e506e09fd910a0657c0cab41756f0b7b429937009a85a97a0 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..816901f251afe9655cf5338c3b4435ef3f8dbc63 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_54_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19da9b98e7b1482336c33bd0fe968583d47701764a52e3041b931a8e83a707c2 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2554a50d7b64b094489e28375e7cd54fe792ee4b --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_54_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bff3026eaf970b7c549091299aef02abdef76624b84b5d15c7a6e3e7706d30a8 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..312e30469decb81fc2492763daa71821075630fa --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_55_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e9b09d31a62e426885eb8abd66b54656248960b50f080439bb753c23147f4f2 +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2ed68d3d0d17d5434752c299ff49317b29976f2c --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_55_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1552bb23c3479421b9f942e6c33006321332ed8b9a916a5ffff6288b52164258 +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..95b4973ac693157b253f527519507bf8376fdef2 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_56_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2712fb3db6dfcb6552b592ec947f564f58d48786360341945f3088a14e41c37e +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..32d048185cac7fbd5337617979f58e22f7cd17e6 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_56_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb2f55e3e58dc0b93a03bddcaeab85daea83c3b496a5b06a5f8c73015fa027d6 +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ab52e9aabca35f7c54f2e258b7119bd302344c24 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_57_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:488c0e70d0ef271c64f1414cda0169a6485a01fae977a7163b572d53f86e3149 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cce4dfdc5345070960e600ccde7401ce545332ff --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_57_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad105be9e79a7ec3d23a5a79d810c42ac2bdef5f3fcc50406bf72461aef2a1c4 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..bc25a04ec7699a1fc13c3f1ecd2bc8c678363706 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_58_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:01ace0826bd80af3bc159156a5349e00650bca2a159a77b947c812354f50b15e +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c86b4ceca0c2c5bbed5e6e4017821c48fc1b3651 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_58_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8053f99f44c68f4f75fe677e545100506e01de7f10ff81bbc9996f1144e0a88e +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..05271642dc007a87c32fcf5aa39ce2db4440925e --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_59_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2072a6afee476d24f09277ce0e81e183dac7eef24da55774ae235d1aa40d3ce8 +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe692d5a5cbf55c5a17e6134201612a399ee3cf3 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_59_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e278e0f0b84fb67585f4ac60ef0b98d688dc632c22813674f9c33174a2ac155a +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2048d7a97d9fd9c520c2c340122e3ced80c005f9 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe181c501cfb3717f1e770f8fcd26d1aad4beaabd34274c86d91948ba863898f +size 117208919 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ab94410cecc5f3f5c849be71a35a4e32dab68658 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_5_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e97d5414e60a8e1ff154daaeb300107bcf544f578b27e6ef4c13876a30df56eb +size 117208919 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..99cb5f23ac11e2337664f6e69de4ba1d9777d42c --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_60_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acef2b0da7020a58a3962404a29959b3e853c3ebe7687db5f2de9166ad07649b +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d78fab19679e6280510d9012c9726c4ba45116c1 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_60_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:384595907fb92a28c2c2395e674f7c812b3d80ff6f6f20c8c25838b9787c3457 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d3a731cb4bc87368340180ae32eec034a26a7bbe --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_61_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32917535c222ba1af66c04d386cfe9d571d17c8e7cbb62f1f273e433664496cb +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c13252ecb1bc215908d7b96d3ad55233f8c8d22e --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_61_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4456b502f2bda708024d526659dcfba6eaa26a55c9ced832c6df237ba10beb38 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3478591f318297b2777a0499f45cb7e134873e0d --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_62_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee2926153e572479819c2d8040994939c1bfc18a9b81f6ba32490d5dea579f86 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1291ff8942a489642cac3d6156247563afcc7cf7 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_62_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:419fc6c58709d7b7f1f39a114a631dff60b96205767cd4073935f0b1e15ed833 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8e32157e84259691487987580047cf2e57bc5bd9 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_63_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1a865f03e8bf642d1acf42e7bae800198cc7deb704e575bb2730eab9aed229c +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..801e775a311e1c9eb11884b3105dccf242037281 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_63_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3399a33004a74f47c0898930c1dda8c65087342409f3133b434994f269de453e +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e3af5e5cb366c5491a023599601afcbd5bf4072c --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_64_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:774b3c4dbc4489cf52692bfb7f46c559ca16ab4967655562a9ba31f229758d6c +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..06857de2f0d681a1ef993c5328f0cd1f2eda38e6 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_64_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b982e883b8862e4216b499ca446cc5204f40f256c601e436222ad0f8a598e57 +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f10d6d691a0d47021649dddd22dca590ef3f61a --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_65_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e19e0ad4d865756655083b495311e78de6ae1c5d0bb7d75858c09609de3daef8 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1e338a1227824126d6d5b6d311ffb42dc36a99e2 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_65_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac99878d2bcedef59c9032b3169b9a2a23592b31028c65e82a6925a8105f8d7a +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c7ef97bda6ea921e63b4301e3f19e1d3f817f3a8 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_66_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7072e518daf0550dd8e10dab25118f8988eaccefa012e8aecc22ad69ae174565 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1059c24e6268cc57945b22b046474bb6e79a8a1c --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_66_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d40731d3cde987bba06bc9968f22aa1dc5d664cede0cd4cc1b48b97247b4698 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..74c63ce46f9873653f49234f1f3a12d022f4815b --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_67_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4df4950ee8235c2ca3c9430a39268057f237a2f73fd9a71140bfc453b8a2ebe2 +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..49853664291a174e0ae14012097229866c561f39 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_67_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfd41679aad796ddf80bb5962a4f4a3f7e9e6a90070d491cf33a33519505b574 +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..35ef48f6e557888baaf075ee0ef13369112bd27d --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_68_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6582ed4c999c35d29193b4bf35a42288850e07d24bb84330c2ec4e09eff05dc6 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..51d981fa31171471fb39e505305c8ab1d26d0951 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_68_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:501cbe08470c96e57417eb76afafe6526ec98900ad1ac951a7383975b480a3cc +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6824bcf4adcf0e5d2fed64b149d68c4a27ec7db --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_69_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0f991a004d4e41f130a1aecde61c207ec13a275d738a7b1505f6027f0bccb13 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6e6e5693ba323947e1dcced86dea9f1aecab258 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_69_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c75767b969cef0b4858cff6f3e3c7e4c79894035c7c91313738cd94c48d9920 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0377948a04b7c32099be1a54776a5c9926d4abdf --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:465a5f62285172d0228f7b416c884243b0740fe004c01869def873e9cfe7ed23 +size 117208855 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8786997f9487e77c03b4cef2c7f8eff6c6602cf5 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_6_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:62081ab38ba373fbefce56cedd30303a3096416eefd28138f5014e5a33ca8f7c +size 117208855 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..62030ca3591fc287072518ac0570416ce90b2ba6 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_70_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:180a45bcb3f4ead9830b3722e2ea319b69eb210f9fae8fa292f676cf88855cce +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe6e4a7bf32a13b8cd88c4e889ca6cb8d44f2ac4 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_70_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2517ce583f195eaced7ac2601ff25cb7f2586ad1d6d121c0fe1b8047ef3cffb +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..57d75a63c1dd2f24687bda14fc7cc99c83c0375a --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_71_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88ab68fadccc58848d2f708894bfde69eb89d33fc1184861cf51a205fe1fd8d8 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..95ff1fea4b452fe774b839659487208a2be32e55 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_71_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3f450d934d876b50dbca5f340d8112a75be8b395b9d0a82cf5d7f84f0e2a133 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ee0c0bd885ce2bbff317aa5855aa9450f3c9ffe9 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_72_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:add9f8325d700de4330bbcf096beecb1387f2c028565ca5c4aa4b25c451a6b6a +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..03f9db5e154f3c47e7634753c8549270807162e5 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_72_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4782f54c262da28bbb3b12973938ac4a3a1df2424563cba3d3a4ebe414abc617 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a9bebc7c5ad9250e5caa1f3b7ce868b1dcd380df --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_73_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8dfb34aaf0a70331538ebeecd3a0fc1083fe824e450ce6ade8a78b51c2d886bf +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f5b506d89bb8ff81b19fb250e563262bf9651d77 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_73_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff6175fea3ddf2477ca17595d91dfcece9ce46c524ccd80758c71e65cb0a2ede +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..caeed632ee275df55330267b245dd99f4b16283b --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_74_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c69a5a77e8e03e675d734a6c1519dbc2c09416a3996a26ebd078240f050d1ce8 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..23b5f2e5481c79ea2bb2f8b4e9b05bba476d8c6e --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_74_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12685221faec6e6dc78bf456f5a9eec987bdcc0fd4263fd39fc3480928d92362 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f08334e54dffef2febc32c18419de1458103baf --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_75_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f62b1a11660dbc2d89023e81d8dc8bdf75aca56f31fafbf9470e303d38366898 +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8665c1716130b02d3ae7dedc3bbd9bc592442443 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_75_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a1615acf5611f2b0f27307faa829f6cd252c2762c3e6c55a404a0a3acda8d24 +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..83a45510f844245ada2baad1a9e0ec1fe60178f4 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_76_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:669ccb18369a86a567ee57fd330e7900ce00464de197706de7a4ff048154683e +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0f2a52859469865f7c94afff5909166f46d3ede5 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_76_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bf63fcb29e8e7c38eea4739e2c84cfe556e1ce48cb1c678cc2dda2cfec3a8bc +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6bf4b5a0381dc2177a963c07aa18846953b5960 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_77_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff186ec6cb4e6d325df678e661664e6be1e6d9210a4c9cf86200fd65cac22f10 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1d80619e3e70e53233af5550c2044da6e838ec44 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_77_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b992d4271bc1312756d8981b400ee428d4499bfc7aeec7c0ac9025c1781128b +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb321a46882d131b6f1d4147cca2d71e201adae9 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_78_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6053998793c4e32defb5e6addc81871dd0d875f7df8d12fca6f394b63706f3c +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..25442d5afd22be4998374fee67e5f3daf27d3ecb --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_78_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f16d4bcc5f3799cbf0eee51fba47c8f868499473c05f6322b5e055034c2152bc +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..19e3ab16c59fb01657e5c301f611535944639141 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_79_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b07b7ced747ba4d6313ff58c9a0ef982dc4cbfeede8d8f38c83a0e205d7c20b +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a953568445a429a94c9d3bf842671f8002e1e1d4 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_79_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f9d853c24caf38a3d7c2ced195ce21d2c4868dc9bbcb19a7f87570d052fde45 +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..866becb40f4d9838c3ce421b0576d491fe1d4dea --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83f2ccccefffe37775b91aa5057b16224fe90a79c19dadec383f314e54a30fea +size 117208791 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7a41a34104848d9a7a7f64d916633dfcf1467fe5 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_7_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:def9969fe15e7f009fb4ac5f269d1dbb3ce8dfd84d89e0f88899a068fb25a0ca +size 117208791 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fdbb13ca508f04d0acd7b8bdb0387d31bb31d92a --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_80_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9ed4dbc5e3a8916ceaea70e4c650e098a93f506c5c806c4b673b24149315497 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ca33b8252f3e800ab7173da3b5b46fd76b0acc97 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_80_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2675e3947a032b4a71920c8c3c1b0db086fe258bbf5aaeedd330a11ea83a2fd2 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7339b95d4f3f551e28e480d9307c3f2bfbb80f4d --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_81_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d0cc9494674d2ec02e73f8c6b14516cfb480a2c77764cbd68c2763f704a2277 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7364b5e0ab813920b6b14c3599e665bb4e4d477a --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_81_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e2ffec13c8519541751d02368d4f6bf4611fa97a65830a820c142eb4fc793fb +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..756e7ccc9782492d5f5daf850d311b954a64b47c --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_82_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d047a49e31b18481e78343ef39128956a8870f8f54a97b01ffd33b55207bccf3 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..234841c69750e01b2c1c803002ccbacf29214cf3 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_82_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f02f093d42f16fc589c1493339989a4dfae51eea8660012e64611077db2618a +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..64e610c1d62200180bf7a3f9ad27653ace9b8c5f --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_83_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7923605e46dcfbaf3230d74a85c524469daff3bab7c32b20b55ddbcdc089603 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..56e7f70242ed043d7796d9f11e3a50834810a6f7 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_83_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7398b3b640af4bd09fd9298559c11f3172c258684a13c5d8bf70231288d7660f +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2c5de505aa775697686b76be77966cb722d3486c --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_84_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c4437c447f638e1beb6c0f361cf1f6bb60ea04478f95c78bf2d42a43bf40dbb +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9720beaca883f8951beeab7b57528aa6e8172a43 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_84_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:915ca327fc8f9b27088e7ef4ce9850429295ab6684d9faebb1f06eae755cc749 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..89d11d19c71c45955850adbbc3eb5def855b6d97 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_85_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:254913ab6f82f58c5bea27cdbe02a5a8d5f4adff87e3130b6ac6cfe489e85031 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..87266f0cf93debf3c1c49bdff88fb40861006e92 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_85_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a8175f31c734c77ae14f134767da4ad0fc1bb15caa22be44f0528be7066c1a8 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..11be3268eb6f0b35f56cfa22c10ec2cd6921c3ae --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_86_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce13604c17ef3838d836ac38af55c9130447081181530e2b145b5902a5eb74fc +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a15cb0ce3999554eaa727d6aa2c9ae85a9c85348 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_86_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a170409271db5355e481ad968fc14752f6744aae5da0a624ef124c2f543f802 +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c90e70a6449b4b988c0d2f9c98487573e353fb87 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_87_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3adcc098e676cb1663e5da61e9745c165224e841a9b10f97549f0e77bf3fbf59 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ba122c3cd18ebed1110e460a625d3321a7e10105 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_87_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fe8714bee84f55a31fbe6e52710059a8fde423fb41562bc5de4dfa55d6dab5f +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a57c4d2785368a1a995c7333cdacd59d673b706a --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_88_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:005c21306f92a41a66865b4386f19677b284c95bcd2e243d197a9c43088a7061 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e17335c24b327d2fb76626b12b4fc9898127fe00 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_88_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5bc77d90948cff4ad66368970b3d39bb41fa880cb7912651242933328336ef4 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..f9fa5d0424a479192a79a90159185e11348ca0c6 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_89_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf1284b3482cac53488bae34d8c58db47151114bc96d8f9abaa470c5ef38ee21 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a7d1b996872fe4d0220459b23706c5834a90c5cf --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_89_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8fc4de4cf227da06c709b1bb34c9d806f4e3fa7b76e461425519ff9d4c596ca +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ee426383b17b7d68fe207d3e3ea24a737df76824 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_8_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c00fbe179f3656c00cc09528fdda64275b616866a5a05fac25b64c383a43f7ac +size 117208919 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8379e8cca03b4b3e5cc9a919e67c75cbb5ef5dc3 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_8_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b24da699ddcbb16f85c3332e772b6774381969563e7991f310b6bcbf6aca7dd7 +size 117208919 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6bcbf3d8f1b774f8ef81bd00ed16432b4b785df5 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_90_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a0f16c10ee01855ef36090fecc56180eb881cfc4310d66c53274d4bdb64ee01 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e8584f2a0731d8893fc6bfdba1a9782d0367c301 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_90_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a8f14a7132861dcdb7b6285ff81fd3418732f05ef5f660cb5774a656d4476e4 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..7b08775383f03330ca58ae2c12ec1a922fa3ab95 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_91_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8156af3f76c1cb820a5a4b9c0e2a8bccdb0f63e5aaa267f3f5eb68e4aa55588f +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..48a2f2078e59a056ab9c9a7d9e8a0a0e4a85c8cc --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_91_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3595eb50afff23fc4e715493067beedd54fe849590d745e6274ada82103ea55b +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ae3f4569b339243fea649e6ab6059ec49a811a84 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_92_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61d2f140bcaf9b11e881833e8303029d74a8386310a12adefc47be542c3b0533 +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e45bba9eaf59ac8c079c6d4ae71673a0b4fa33d9 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_92_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d24c4f30fe5c2efc5b3b32222c3a5153c1ea2743b42805bbdbf3617220bdf94d +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..aa7e977c90e85bbb3b41029bab15a0a1dff597b9 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_93_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56df5c978cf999c9394a12b793b94f0bacad11a8005b3052054724f372ab90f3 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..43ece8fbffcea1e51cec8213b2d0aef42f9f8839 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_93_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:38e015e371518d18d93ded59ac731fcfa0cc191ac8761569d597a2d42e9e8ad4 +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a82fd286f99add927c5e09497bde65b5b627bef3 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_94_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6d89cef0f9bdbb5a6006e71354d9b4195a20bf2f45c7d06d3e91dc4883f6fd6 +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..17e7da8860c1502a7565c4b85b7caf921de933dc --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_94_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf727ccafb62c49f9b739c30341a7aea946c1e73a33939b4e1d48dc3f72be9b9 +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..67546215d08ded254e380beb457b37c670e5047d --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_95_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5c5f3b853a9bfdacd92c7387a14d7b3682f0739c56d66aab07d2e1d13804e7b +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c5f8966f1ddaff5262a68f1db31312a862ad4719 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_95_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5c7d4c7dedffb5d9e9cf8d9c746eed459eff871c6981f6314e0932456eb756a +size 117208866 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec7bf39fb76642ad61437da5680db737e3a48f58 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_96_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f70d97eef19b645f1d8ca0d0bcb98b5edbee5718b718590b66b3254eeed0f378 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec98f6cd501872cec6107f2fd72eefd45f58191e --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_96_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8ad79fe6b487874dc651483bba2a17f3ffd541dacd06633aa3638fa998f75d7f +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..02c79f33aba5f8c8ca1d6b45af91c58d12753cdd --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_97_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c6c85be7e763b0c0e3dfb8687810c2e0d6bb3c9c8903b401bee91779cf3428f +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4a07d446dae76c28cd452c08ff9d5227b884e706 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_97_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2455218ad466663532d006c0be6e2aef4f54b30531de1a550e903756113860d6 +size 117208930 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..642941e7b1ba3d25d5e47f85e9e0db2fc6432189 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_98_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7880a7d736991e98d5f8f6e21b44b0c9f1532a287963e52fe1afe88fb77519d6 +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..04793369680c0e251bedb0ebb1dd30e73b7a747a --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_98_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ce60b88865bfc70b3a551f4e9a3c75b05777e7ab0a9104aa00b2b04eb4a1695 +size 117208802 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..880b106d18664a52475f8d7c528c9f2b54808abf --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_99_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:494c706c3f1482a661caa0885bda402f1649e6963cc43272a28593abd92bca6e +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe22236800c6768dcfb80e19723386d738e2a8f2 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_99_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ad80bcda5a5c518bbbef76f626c03846d607b30cfd9077b2396f8f72b15ad26 +size 117208994 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dd8becb3c8ac9b347ddf5d9bcfa2fda948fff729 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_9_mp_rank_00_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33954dcbd26beab49bd3cae1183fd2946afa3f28992ee08180c3e9609ef0511d +size 117208791 diff --git a/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..21bd0463f5db181af0caf9fc214b428fe1af32f7 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/bf16_zero_pp_rank_9_mp_rank_01_optim_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2ed3c6e6176a11ac51ec8e39aade18d5964555ff10cc51ed659bd0bf711c9c7 +size 117208791 diff --git a/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0ce17aee4c3f3083223c1c11250da2294e45b3e0 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_01-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:98b4b66a86e42feecc3437daaf670b88760b36c28b41cd66b16b08f9c5b6ac0f +size 139592963 diff --git a/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ff16e30c7d326b44634e5d5cb2b79faafe8264b2 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_01-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca0fb1c0d2d135a777bc8231b499da8d714ea9e76e9f84cedb07cc7b74aabc62 +size 139592963 diff --git a/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d51dae7f638ece27f874e80d8daa478afd362009 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_03-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd27fe239b7ddc4dd31801d91acea9cfe61d8827fddcd7c00544f70aec3f4e90 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..40212b7e42e8b2b63ab09e567e0fae4cd805763c --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_03-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:416e92a4c1018dcb38e4272f2d841266adfb3f2c97050f1f994b179216f54e1c +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a19adf5181679be7265f8f4160e3fcde7eaaa9ca --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_04-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a277b46370d3da302d185b435f5f0459386d262607c13a6779be0348ffca8a5c +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c9592e1ee796f3a851afe9f76c72cd9f6ff9d215 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_04-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afb4bb3685c32d21881fdcf2512918fbabed5392bed335c3fc82439d0952e3d8 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a72177f617720e50bac387edb5d5eecc1a26ed59 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_05-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57d1c9ab1f40f06978cf7d48d8d88a1920e1205b6e8e5f4c41bd63533b993288 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8fb7d3e69f31309eb08a691bae2b4a3fd3018f53 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_05-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4bf08ecb4eec840064a7881fb9ad69c32c366860b310f4ea55a8e877946b9e10 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..93c9f135588a9f4eb3aa2ef8c059674cd2150ae2 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_06-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69c0939bf2a800b4e4619cac527ecf6654e8155ceb85d7ac12f93e7f1cac3811 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fc523f035e0d6ca011e6d1a590f924ef7a10a55f --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_06-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c614fabf2ad9057f012f32b9a1e978e3ea14486364394a244c537f9b24805a0e +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..335b86642b274d68d280bedfee9338185cac87ff --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_07-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c7822939218d383e975a4c2d67601af1c99accb10ead617919932568528eb01 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c73067659b89f61043ce9e248973b05bbfe6cc17 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_07-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:620bdde6fc9ae7e2cb66901ea9328330fbb131c68a293aad08e08138612b0c2f +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b8356387e78695f9d2f05d27e174967c1bc3b7e2 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_08-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0fe2523cbef91dbe4da368b6415a287c54ff3ac89fd1cfbd450a4da4e1b83d8c +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8dbf2cbe6c1b641c64c92be068c2e572e9c914fa --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_08-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f4111582f7030003b482baae25bece8ed53ad6a817adefdaf0499a3ba6a1d86 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..056b33d62de79347953becc949b8b2845004456b --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_09-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daccb2080339b2db047e7aa6954315906ce661bf428cfd9ed0520d37df0c37a0 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2bd96550db10af4e7c46fd4d21bb06a9a385b75 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_09-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:288e04a8ebc363884393353203f7a4be5f2a42fa3012211c141fb3156907dfe4 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..85518941463c2b84c0c37384853adc421fbf107e --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_10-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aef5abecd5093b8c7496b28bd8d4809eee6c0637e35e062a43b9414ee86bf72a +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cbd5e67cb23042cdb4b95fd9c615bf4736d09aaf --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_10-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a759f1bac37d4b5f621d7e55e4c3bc2b93678a6e54bf9d20b3b9f71b5e8252c +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..24b6f7217f2b94117d2cfe5e14902968b7804e1a --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_11-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76492a7acc368c15d2dbd960da4819645a184510a17acd85850ca902eea8d643 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b6854cd5a5dca0d3fd4122775c6a1a677c3eb9b7 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_11-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0fee32f73549d9aa4aa523dfac0c681316abf13724c2845607084a0bf13d516 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c1e050a7b39b597790bc481df29494b8b5b5a618 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_12-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:962a96c5b506d7251f86e5228e902092c71280551be0835dc4d7c204ed373aad +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..785d29a4a5b26d1a70d79365a7ef40c99b5f9f37 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_12-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cb19ab03c3f7e44eaa0c3bcc5f9569eb3ce5f988c801754778964804658e386 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..cf66423fc1ff3080c4d170dc3951e585bd97faa3 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_13-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7abe43fddefa1f99df27e936bd055c2fc61b326c19a87bf35001065a8ed64343 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b31a5a7644aae2453206fb6aae3e469aed077b29 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_13-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a77d386d4cc70bd0bc6f875c0e8a9746ec1c82077fb99335dee37083148af58 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ac360e0e642a88da3c3617aa10155cedf1ba0ce3 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_14-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95eff3912900d1796ca60734fbdc269da71df53db45ec2c58fbfdefdc7b888e5 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a5a4eaff52c3e273f3c3e6c19a5052ada0799432 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_14-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44a3736cd8a8d0a4609efc86725a79b840237b44b86602cbd15a6d4d11ebe998 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..a7062bd380f8174f8ed0f522ef30b1c801960ca0 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_15-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdb1aee2fe991ccaaeb145615ecce416b5e9ae16f0690de6cd0d50d88bd700b3 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1821b4d870d117e7bd63a65609c2d6f328b2a007 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_15-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6eaa6979843c72c99f421ab0fb54b6fa56337f38f270b51b3400b1220fd34dc +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..0136270943b38d80d53a15d1ef4c6eae865558b6 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_16-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63d2f124497cf593cca4fce6bc6f631aac5f00908a52428ee655b218eba16d1a +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6907fdb5ef1c8d721cc2ea44728d1a379c1d614d --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_16-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ef61bd3d273ab117d64b857112eb0b3f22af657e7e7e082b02f5c3050b658a23 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ecfedb45815b7e5ec00741caeccdb4769e610cfa --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_17-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e35a04180a46807c67e291b5c288a5763a0dced16daf1c4fc5b3ee137281918 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d1b76328395a1e18eb3b58261c2a120d5b6710d6 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_17-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:321cdc222b3fa9507afe5a6ec2e5751930aaa83398c367ca4c26cbe5e40c6398 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..60b4071b92812a4872569813d3c0236a9dadd693 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_18-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88d1e6d40b5c303496ac8ce0ecb57779dc9eccba18559b9a903ff558b9b7c3cd +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..de08d9486c327d608bdbf016a3174bb6d94c9b9e --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_18-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:378715b92af56e31a2fbb17c141dcba2693b1bc6cf9c8aca984fad7c2285ddc7 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb9315eb73c3c05ecee11d910cdef4a1c7a0f453 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_19-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e01a72bc0d913b6f19ede1f0629edebaffdf8bcf6b18a20b53cfbe4d4a06a107 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..83332b5ea29dabc72b21012a3725d5a3671344dc --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_19-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e50a787809975b66c67f5e066af6282a7ee2bfca98c70624889d8c97cd1299d +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..88f72e4e1166472f9ea950532eae0f7545483b71 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_20-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:10172d05c9dde26b75c0ca4a7a1d05431447934970b8f52ae78e3230b5492c76 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e378e2e770e22333153e6293ad6395dfa1cff3ce --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_20-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:316715c7acc4f8041554cafb404b95e71f58e5219195e782675c5632b5590c6c +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..6887a0b41091d342742c176b5a290b20b12fe47b --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_21-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2699c1400c327984aea518d6ac966cd78cfabc9df986db5d93760b616173e2b5 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..de254737c5233712d5de4ef5ab17d26d3a540d16 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_21-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a7d6578b424f9a638ed96aea7de7cb7628d9ff115b9cb23b0818623a31ac9f47 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5998bd7007000e6e7bda769b536e3c254a4514e4 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_22-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1fd434f163c4abbb17ac0eed7b9cc3e7f06a3cdec1fe3d55db2932cd26ca0ac +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8ea890a322b8feba07e7d1a6a722919c9f4ac671 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_22-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15c9961bdfa8df5d9cd87d01dfe191b4c59cc88ed5087291c78277970d3838da +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..31ee81f2da2965013831d2564fde6810ce6b8b55 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_23-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c69765e21d9ae530d59552a35515ca4d8abb269aec999815a7e2fa5c55df6ec0 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..dcbcdf83e63007dede5a96df3fa73f62fdbbb5d1 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_23-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e96c6d9d24c0fdfa0cd311005a5f86c266b84471516c6d7de2504f510b5e4f6 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..3c0363369a6f7006bf04719d186dde482b872de6 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_24-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c1bb8a554193d24b14ce7973257cb43d98b034dd1270651f1b853e951aa6716 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..1d74644895b8dda72c7e31d6c190f3a8b677e811 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_24-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91ef67a049d4628293718981db94026fefe6a85179222be974864963690905e4 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..960f46842bb3b6a24e78283a901d817cdff0adc2 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_25-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2bdedc82048b8d0b6aef6e7c76dadaf4cea054374027acf2e0697cb0226f8c3 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e9d51f745ea9c5fba1999c324f2ce9cafd182b9f --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_25-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:548d22c43438be4e73d7980fc36886c8a4d25e7a5424c33d3040fb4f07fd23e3 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb3167daf252a16234f5ffe1a17c1950157d6cb8 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_26-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:943a7c9896dc3eb953d682bd192a1dc872753e0a4c5ee4c9bc30653fdcc739d6 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..4dbf28215dc6b53d459797a6264ea6d404d92c15 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_26-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ac260e402067472c2416def4969ce7b4fc18397967a11b999efa6034202da08 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..819e40281876a6ed58da3e11e040c0daa16a9721 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_27-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d7466377392a836c161c7bd7c8783441f02fef0920154fd2d9545265f032f55 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..c8a0123fd3887ac75021d7882061435f672a0558 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_27-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecce5108164beed444914da271d17e6f0036c916e7c4ceb2ed7df51ae113acc6 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..04045acc091c4c24db0a08eed5290054b9818f04 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_28-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa9651c63c220de7a1805e27d33966daa85c40442bdcbadadc38b1b6786d64e7 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..eb282807b8b763d3d3a2eb26120b1a4f45c98a49 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_28-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e12d63ab5833ea965b5db7cfd2101fe2e35ee1391432cab208160d0202dae858 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..2cbd38f0516533a8c5fd8656a45659ad778faf32 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_29-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb8eed61e129c94795701527901916e46f4bbb0daa7f22a51971f23a27eaaf28 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..19c8aef1bb4b9035b1d8e4996e3d91572a2e70d0 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_29-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf0555e210579cdb7b77dc8d5f79cb9322aa94399d9ea90e98fbfe875eb2de8d +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..725a6ec81fa918d0d08fba418d1ccab99bb0e9da --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_30-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59f9beafef4d29172dd660fae3f944c2db03ff233d82bb9baceb9aeefe50e008 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..18a7cfd5edf763bbcb367fa772f1a19d40146ce5 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_30-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df1d0b6a9252ffce10abbbc103eee6b8bcc1d8365d177ce3b8689e8a9be097da +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..d139810170cd8c63ac40a33301474ab11e204ad7 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_31-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39d27a8e34282f3c8f2ab2aef11d9a2e83ada0aa55093c474622c22a18c465f4 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..5728c9926c2f5186566e636a202c039f1f7f2686 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_31-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bab76edb79a6d5026f7b55299ab2a2698103e31083fea8efc5f7aceb0ac74af3 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..8fca46c94061daac0ec906b0ded9464b5d7ffee2 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_32-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f57dd9d9ab10191f66de9825110d708761b8c156e0324a88f1375f2829ef54df +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2d61d67b7815b0af1fda6668e6bf51e39caa30d --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_32-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b3ee41227d749824bf2f8836f4e9e064a96f491a59333380a67563f116c6457 +size 78696195 diff --git a/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..b009df699c1a3c0ab69e907ba0855765fe310acf --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_34-model_00-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5c7619da287f164016b0617949cb02fbb5ac6e41947cb240cfc6c7b3454383f +size 11459 diff --git a/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt b/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..9bb222f15f288e15a23a7ae5d506c9cbe45e6e11 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/layer_34-model_01-model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:444f7e9be7e3afb420a5b46fec8480f479ae9781327c145791bad20d92241c8d +size 11459 diff --git a/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt b/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..08e8d6c784f92f2abd2628ef733f8bf99887d330 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4781d3479edb7e1c2ccd416a4af79c7ec327c7215fa5563e334f941531f7b402 +size 46899 diff --git a/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt b/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..df61914a5a687f6d6cbf4e31d406f4e23fd5e822 --- /dev/null +++ b/4b284b6bc4subopt/global_step122168/mp_rank_01_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec0689aa79994e7ba196aa7433e0bd281cc0b35f7c282f3b5f93493e919e6a12 +size 46899 diff --git a/4b284b6bc4subopt/latest b/4b284b6bc4subopt/latest new file mode 100644 index 0000000000000000000000000000000000000000..984c815702ac6f6792dd9073d5b325126129d314 --- /dev/null +++ b/4b284b6bc4subopt/latest @@ -0,0 +1 @@ +global_step122168 \ No newline at end of file diff --git a/4b284b6bc4subopt/sbatch_4b284b6bc4subopt.sh b/4b284b6bc4subopt/sbatch_4b284b6bc4subopt.sh new file mode 100644 index 0000000000000000000000000000000000000000..4396d69b9260443434d98623ca3cb0c78fc8d535 --- /dev/null +++ b/4b284b6bc4subopt/sbatch_4b284b6bc4subopt.sh @@ -0,0 +1,164 @@ +#!/bin/bash +#SBATCH --nodes=32 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=40 ############## +#SBATCH --mem=256G ############## +#SBATCH -p standard-g ############## +#SBATCH -t 48:00:00 ############## +#SBATCH --gpus-per-node=mi250:8 ############## +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread ############## +#SBATCH --account=project_462000119 ############## +#SBATCH -o %j.out ############## +#SBATCH -e %j.err ############## + +VARIANT=4b284b6bc4opt + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" + +TRAIN_DATA_PATH=train6b.txt +# "train: 1.0 0:1 gpt2tok_c4_en_6B_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 gpt2tok_c4validation_text_document" + +PP_SIZE=1 +TP_SIZE=2 + +MICRO_BATCH_SIZE=2 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_2639M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=10000 + +# Tokens: 84_000_000_000 +# -> Samples: 41_015_625.0 +# TRAIN_SAMPLES=41_015_625 +TRAIN_SAMPLES=62_550_493 + + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.95 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 625_505 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1000 \ + --eval-iters 100 \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)" diff --git a/4b284b6bc4subopt/tensorboard_4b284b6bc4subopt/events.out.tfevents.1683211721.nid006565.94534.0 b/4b284b6bc4subopt/tensorboard_4b284b6bc4subopt/events.out.tfevents.1683211721.nid006565.94534.0 new file mode 100644 index 0000000000000000000000000000000000000000..d5c01ae54cf36d6cc3f1188537b349e7111a7025 --- /dev/null +++ b/4b284b6bc4subopt/tensorboard_4b284b6bc4subopt/events.out.tfevents.1683211721.nid006565.94534.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56cb912577ae0cd824e4efe21b474b71a96a486daf3bcd2d91dc7499cc5473a2 +size 180820887 diff --git a/4b284b6bc4subopt/tensorboard_4b284b6bc4subopt/events.out.tfevents.1683384533.nid006741.48255.0 b/4b284b6bc4subopt/tensorboard_4b284b6bc4subopt/events.out.tfevents.1683384533.nid006741.48255.0 new file mode 100644 index 0000000000000000000000000000000000000000..864336dae9beddf1ef4c853a2e3b3c8a5465cb93 --- /dev/null +++ b/4b284b6bc4subopt/tensorboard_4b284b6bc4subopt/events.out.tfevents.1683384533.nid006741.48255.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3765a3634c4cba29a6d293d5606701ee3d69a4abf7db5e9fc0bf5611f318ca39 +size 39858510 diff --git a/4b284b6bc4subopt/tensorboard_4b284b6bc4subopt/events.out.tfevents.1683422714.nid006741.22960.0 b/4b284b6bc4subopt/tensorboard_4b284b6bc4subopt/events.out.tfevents.1683422714.nid006741.22960.0 new file mode 100644 index 0000000000000000000000000000000000000000..0ce42aa58ddaed563987f03a258e31eac33d7f32 --- /dev/null +++ b/4b284b6bc4subopt/tensorboard_4b284b6bc4subopt/events.out.tfevents.1683422714.nid006741.22960.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e74aa0fed0dd6624a9d9f870506c131c96759d073dd0399894856c88db2ab03f +size 21490 diff --git a/4b284b6bc4subopt/tensorboard_4b284b6bc4subopt/events.out.tfevents.1683423027.nid006741.28239.0 b/4b284b6bc4subopt/tensorboard_4b284b6bc4subopt/events.out.tfevents.1683423027.nid006741.28239.0 new file mode 100644 index 0000000000000000000000000000000000000000..2b8522f0b39e97cb94e1f3a13400bc53df28de43 --- /dev/null +++ b/4b284b6bc4subopt/tensorboard_4b284b6bc4subopt/events.out.tfevents.1683423027.nid006741.28239.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1ea2fba3e25944fe8e02590bf6b9ea1dc9920acaba31e7cfed77131a1bdf4d17 +size 16424 diff --git a/4b284b6bc4subopt/tensorboard_4b284b6bc4suboptval/events.out.tfevents.1683534303.nid006518.85018.0 b/4b284b6bc4subopt/tensorboard_4b284b6bc4suboptval/events.out.tfevents.1683534303.nid006518.85018.0 new file mode 100644 index 0000000000000000000000000000000000000000..39f5d8bbb4df8a7b2a1cb76362af9e3f5ff0a2ac --- /dev/null +++ b/4b284b6bc4subopt/tensorboard_4b284b6bc4suboptval/events.out.tfevents.1683534303.nid006518.85018.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d84ce5c3e45924254ffec5da49a0c6d7869b51621d9a73d54e8f363d8db611c9 +size 980 diff --git a/sbatch_4b284b1b9c4.sh b/sbatch_4b284b1b9c4.sh new file mode 100644 index 0000000000000000000000000000000000000000..b542cd09b6ef3e7c010c9b3aaef675d42f78fa5e --- /dev/null +++ b/sbatch_4b284b1b9c4.sh @@ -0,0 +1,162 @@ +#!/bin/bash +#SBATCH --nodes=32 +#SBATCH --ntasks-per-node=1 +#SBATCH --cpus-per-task=40 ############## +#SBATCH --mem=256G ############## +#SBATCH -p standard-g ############## +#SBATCH -t 48:00:00 ############## +#SBATCH --gpus-per-node=mi250:8 ############## +#SBATCH --exclusive=user +#SBATCH --hint=nomultithread ############## +#SBATCH --account=project_462000119 ############## +#SBATCH -o %j.out ############## +#SBATCH -e %j.err ############## + +VARIANT=4b284b1b9c4 + +# if run without sbatch, invoke here +if [ -z $SLURM_JOB_ID ]; then + mkdir -p logs + sbatch "$0" + exit +fi + +set -euo pipefail + +# symlink logs/latest.out and logs/latest.err +ln -f -s $SLURM_JOB_ID.out logs/latest.out +ln -f -s $SLURM_JOB_ID.err logs/latest.err + +KILL_SWITCH_PATH=kill-switch-$VARIANT +CHECKPOINT_PATH=checkpoints_$VARIANT +TENSORBOARD_PATH=tensorboard_$VARIANT + +# Data +VOCAB_FILE="gpt2/vocab.json" +MERGE_FILE="gpt2/merges.txt" + +TRAIN_DATA_PATH=train1b9.txt +# "train: 1.0 0:1 gpt2tok_c4_en_1B9_text_document" +VALID_DATA_PATH=val.txt +# "validation: 1.0 0:1 gpt2tok_c4validation_text_document" + +PP_SIZE=1 +TP_SIZE=2 + +MICRO_BATCH_SIZE=2 +GRADIENT_ACCUMULATION_STEPS=1 +WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES)) +GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS)) + +# Model parameters +source model_params.sh +MODEL_PARAM=("${PARAM_4516M[@]}") +NHIDDEN=${MODEL_PARAM[0]} +FFN_HIDDEN_SIZE=${MODEL_PARAM[1]} +KV_SIZE=${MODEL_PARAM[2]} +NHEADS=${MODEL_PARAM[3]} +NLAYERS=${MODEL_PARAM[4]} +SEQ_LEN=2048 + +echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS" + +SAVE_INTERVAL=10000 + +# Tokens: 84_000_000_000 +# -> Samples: 41_015_625.0 +TRAIN_SAMPLES=41_015_625 + +OPTIMIZER_ARGS=" \ + --optimizer adam \ + --adam-beta1 0.9 \ + --adam-beta2 0.95 \ + --adam-eps 1e-8 \ + --lr 2e-4 \ + --min-lr 2e-5 \ + --lr-decay-style cosine \ + --lr-decay-samples $TRAIN_SAMPLES \ + --lr-warmup-samples 410_156 \ + --clip-grad 1.0 \ + --weight-decay 1e-1 \ + " + +GPT_ARGS=" \ + --num-layers $NLAYERS \ + --hidden-size $NHIDDEN \ + --num-attention-heads $NHEADS \ + --kv-channels $KV_SIZE \ + --ffn-hidden-size $FFN_HIDDEN_SIZE \ + --seq-length $SEQ_LEN \ + --max-position-embeddings $SEQ_LEN \ + --micro-batch-size $MICRO_BATCH_SIZE \ + --global-batch-size $GLOBAL_BATCH_SIZE \ + --train-samples $TRAIN_SAMPLES \ + --vocab-file $VOCAB_FILE \ + --merge-file $MERGE_FILE \ + --clip-grad 1.0 \ + --kill-switch-path $KILL_SWITCH_PATH \ + --bf16 \ + $OPTIMIZER_ARGS \ + " + +OUTPUT_ARGS=" \ + --log-interval 10 \ + --save-interval $SAVE_INTERVAL \ + --eval-interval 1000 \ + --eval-iters 100 \ + --tensorboard-dir $TENSORBOARD_PATH \ + --tensorboard-queue-size 5 \ + --log-timers-to-tensorboard \ + --log-batch-size-to-tensorboard \ + --log-validation-ppl-to-tensorboard \ + " + +ZERO_STAGE=0 + +mkdir -p ds_configs +DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json" + +cat < $DS_CONFIG_PATH +{ + "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE, + "train_batch_size": $GLOBAL_BATCH_SIZE, + "gradient_clipping": 1.0, + "zero_optimization": { + "stage": $ZERO_STAGE + }, + "bf16": { + "enabled": true + }, + "steps_per_print": 2000, + "wall_clock_breakdown": false +} +EOF + +DEEPSPEED_ARGS=" \ + --deepspeed \ + --deepspeed_config $DS_CONFIG_PATH \ + --zero-stage $ZERO_STAGE \ + " + +CMD=" \ + Megatron-DeepSpeed/pretrain_gpt.py \ + --tensor-model-parallel-size $TP_SIZE \ + --pipeline-model-parallel-size $PP_SIZE \ + $GPT_ARGS \ + $OUTPUT_ARGS \ + --save $CHECKPOINT_PATH \ + --load $CHECKPOINT_PATH \ + --train-weighted-split-paths-path $TRAIN_DATA_PATH \ + --valid-weighted-split-paths-path $VALID_DATA_PATH \ + --data-impl mmap \ + $DEEPSPEED_ARGS \ + " + +echo $CMD + +echo "START $SLURM_JOBID: $(date)" + +# bash launch_srun.sh $CMD +srun --label launch.sh $CMD + +echo "END $SLURM_JOBID: $(date)"